def save_objects(session, data, region, date): position, track_name, artist_name, url = data artist = get_or_create(session, Artist, name=artist_name) track_vals = { 'title': track_name, 'artist': artist } track = get_or_create(session, Track, **track_vals) link_vals = { 'url': url, 'track': track } link = get_or_create(session, Link, **link_vals) chart_vals = { 'type': 'viral50', 'region': region, 'position': position, 'streams': None, 'date': date, 'track': track } chart = get_or_create(session, Chart, **chart_vals)
def build_db(): ''' Build a small db with some example entries. ''' names = [ 'Oleg', 'Mark', 'William', 'James', 'Harry', 'Amelia', 'Oliver', 'Jack', 'John' ] books = [ 'MySQL for Python', 'MySQL Cookbook', 'Essential SQLAlchemy', 'Learning Python', 'Python for Data Analysis', 'Core Python Applications Programming', 'Python Algorithms' 'Think Python', 'Computer Games with Python', 'MongoDb in action' ] user = {'name': 'user', 'passwd': 'like-books'} counter = session.query(Author).count() if not counter: for i in range(len(names)): author = get_or_create(Author, name=names[i]) author.books.append(get_or_create(Book, title=books[i])) author.books.append(get_or_create(Book, title=random.choice(books))) session.commit() session.add(User(nickname=user['name'], password=user['passwd'])) session.commit()
def _get_written_question_bulletin(): for i in range(48, 54): soup = read_or_dl("http://www.lachambre.be/kvvcr/showpage.cfm?section=/qrva&language=fr&rightmenu=right?legislat=52&cfm=/site/wwwcfm/qrva/qrvaList.cfm?legislat=%i" % i, "bulletin list %i" % i) for b in soup('table')[4]('tr')[1:]: try: if i == 53: get_or_create(WrittenQuestionBulletin, legislature="53", lachambre_id=b('td')[0]('a')[-1].text.split()[-1], date=b('td')[2].text, publication_date=b('td')[3].text, url=b('td')[1].a["href"], pdf_url=b('td')[0].a["href"], ) else: get_or_create(WrittenQuestionBulletin, legislature=str(i), lachambre_id=b('td')[0]('a')[-1].text.split()[-1], publication_date=b('td')[2].text, url=b('td')[1].a["href"] if b('td')[1].a else None, pdf_url=b('td')[0].a["href"], ) logger.debug("%s" % b('td')[0]('a')[-1].text.split()[-1]) except TypeError: continue
def scrape(): for a, url in enumerate( ('http://www.lachambre.be/kvvcr/showpage.cfm?section=none&language=fr&cfm=/site/wwwcfm/rajv/rajvlist.cfm?lastreports=y', 'http://www.lachambre.be/kvvcr/showpage.cfm?section=none&language=fr&cfm=/site/wwwcfm/rajv/rajvlist.cfm?lastreports=n' )): soup, suppe = read_or_dl_with_nl(url, "annual repports %i" % a) for i, j in zip( soup.find('div', id="story")('table')[1]('tr', recursive=False)[::5], suppe.find('div', id="story")('table')[1]('tr', recursive=False)[::5]): get_or_create( AnnualReport, title={ "fr": i('td')[2].text, "nl": j('td')[2].text }, date=i('td')[0].text, law_and_article={ "fr": i('td')[4].text, "nl": j('td')[4].text }, periodicity=re.sub("[^0-9]", "", i('td')[5].text), pdf_url=i('td')[1].a["href"] if i('td')[1].a else "", )
def _save_a_written_question(link): soupsoup, suppesuppe = read_or_dl_with_nl(LACHAMBRE_PREFIX + link.a["href"], "written question %s" % re.search("dossierID=([0-9A-Z-]+).xml", link.a["href"]).groups()[0]) data = AccessControlDict(((x.td.text, x('td')[1]) for x in soupsoup.find('table', 'txt')('tr') if x.td.text)) data_nl = AccessControlDict(((x.td.text, x('td')[1]) for x in suppesuppe.find('table', 'txt')('tr') if x.td.text)) get_or_create(WrittenQuestion, _id="lachambre_id", lachambre_id=re.search("dossierID=([0-9A-Z-]+).xml", link.a["href"]).groups()[0], title={"fr": data["Titre"].text, "nl": data_nl["Titel"].text}, departement={"fr": data[u"Département"].text, "nl": data_nl[u"Departement"].text}, sub_departement={"fr": data[u"Sous-département"].text, "nl": data_nl[u"Sub-departement"].text}, deposition_date=data[u"Date de dépôt"].text, delay_date=dico_get_text(data, u"Date de délai"), publication_date=dico_get_text(data, "Date publication"), # TODO: link to the actual deputy author=data[u"Auteur"].text, language=data[u"Langue"].text, question_status={"fr": dico_get_text(data, "Statut question"), "nl": dico_get_text(data_nl, "Status vraag")}, status={"fr": dico_get_text(data, "Statut"), "nl": dico_get_text(data_nl, "Status")}, question={"fr": u"%s" % data["Question"], "nl": "%s" % data_nl["Vraag"]}, answer={"fr": dico_get_text(data, u"Réponse"), "nl": dico_get_text(data_nl, u"Antwoord")}, publication_reponse_pdf_url=get_href_else_blank(data, u"Publication réponse"), publication_question_pdf_url=get_href_else_blank(data, u"Publication question"), publication_reponse=get_text_else_blank(data, u"Publication réponse"), publication_question=get_text_else_blank(data, u"Publication question"), eurovoc_descriptors={"fr": get_items_list_else_empty_list(data, "Descripteurs Eurovoc"), "nl": get_items_list_else_empty_list(data_nl, "Eurovoc-descriptoren")}, eurovoc_candidats_descriptors={"fr": get_items_list_else_empty_list(data, "Candidats-descripteurs Eurovoc"), "nl": get_items_list_else_empty_list(data_nl, "Eurovoc kandidaat-descriptoren")}, keywords={"fr": get_items_list_else_empty_list(data, u"Mots-clés libres"), "nl": get_items_list_else_empty_list(data_nl, u"Vrije trefwoorden")}, url=link.a["href"], ) data.die_if_got_not_accessed_keys()
def fill_database(): admin_role = get_or_create(db.session, Role, name='admin') dj_role = get_or_create(db.session, Role, name='dj') default_radio = get_or_create(db.session, Radio, name='default') admin_user = get_or_create(db.session, User, login=config.admin_login) admin_user.password = pwd_context.encrypt(config.admin_password) admin_user.roles = [admin_role, dj_role] admin_user.radio = default_radio admin_user.is_active = True admin_user.comment = "admin z config.py, zawsze posiada haslo z config.py"
def scrape(): for a, url in enumerate(('http://www.lachambre.be/kvvcr/showpage.cfm?section=none&language=fr&cfm=/site/wwwcfm/rajv/rajvlist.cfm?lastreports=y', 'http://www.lachambre.be/kvvcr/showpage.cfm?section=none&language=fr&cfm=/site/wwwcfm/rajv/rajvlist.cfm?lastreports=n')): soup, suppe = read_or_dl_with_nl(url, "annual repports %i" % a) for i, j in zip(soup.find('div', id="story")('table')[1].tbody('tr', recursive=False)[::5], suppe.find('div', id="story")('table')[1].tbody('tr', recursive=False)[::5]): get_or_create(AnnualReport, title={"fr": i('td')[2].text, "nl": j('td')[2].text}, date=i('td')[0].text, law_and_article={"fr": i('td')[4].text, "nl": j('td')[4].text}, periodicity=re.sub("[^0-9]", "", i('td')[5].text), pdf_url=i('td')[1].a["href"] if i('td')[1].a else "", )
def post(self): form = AddAuthorForm() if form.validate(): # if session.query(Author).filter_by(name=form.name.data).count() == 0: author = get_or_create(Author, name=form.name.data) for title in str_separator(form.title.data): author.books.append(get_or_create(Book, title=title)) session.commit() return redirect(url_for('add_author')) else: msg = 'Author <strong class=badge>{0}</strong> already \ exists!'.format(form.name.data) flash(msg) return render_template('add_author.haml', form=form)
def test_get_or_create_already_exists(self, m): from utils import get_or_create quiz_id = 5 quiz_title = "Final Exam" course_id = 1 prebuilt_quiz = Quiz( canvas_id=quiz_id, course_id=course_id, title=quiz_title ) views.db.session.add(prebuilt_quiz) views.db.session.commit() quiz, created = get_or_create( views.db.session, Quiz, canvas_id=quiz_id, course_id=course_id ) self.assertFalse(created) self.assertIsInstance(quiz, Quiz) self.assertEqual(quiz.canvas_id, quiz_id) self.assertEqual(quiz.course_id, course_id) self.assertEqual(quiz.title, quiz_title)
def _create_objects(self, object_names): """ Create objects from a list of object names. :param object_names: list, List of object names :return: """ for obj_name in object_names: if self.data.get(obj_name): data = self.data[obj_name] multiple = False if isinstance(data, list) or isinstance(data, dict): multiple = True table_name = TABLE_NAMES[obj_name] object_class = get_class_by_table_name(table_name) if object_class: if multiple: # inject product id into data dictionary for objects that need it as a field. if obj_name in self.product_config[PRODUCT_DEPENDENT]: for key, value in data.items(): value['product_id'] = self.objects['product_id'] self.objects[obj_name] = get_or_create_multiple(object_class, data=data) db_session.add_all(self.objects[obj_name]) else: self.objects[obj_name], _ = get_or_create(object_class, name=self.data[obj_name]) db_session.add(self.objects[obj_name])
def _get_deputie_commissions(soup, deputy): # here we will walk in a list of h4 .. h5 .. div+ .. h5 .. div+ # look at the bottom of each deputies' page membership = soup.find('td', rowspan="1") item = membership.h4 role = None deputy.commissions = [] while item.nextSibling: if hasattr(item, 'tag'): if item.name == 'h5': role = item.text[6:-1] elif item.name == 'div': logger.debug("linking deputy to commission %s" % item.a.text) commission = get_or_create(Commission, url=item.a['href'], lachambre_id=int(re.search("com=(\d+)", item.a["href"]).groups()[0])) deputy.commissions.append(get_or_create(CommissionMembership, commission=commission, role=role)) item = item.nextSibling
def deputies_list(reset=False): soup = read_or_dl( "http://www.lachambre.be/kvvcr/showpage.cfm?section=/depute&language=fr&rightmenu=right_depute&cfm=/site/wwwcfm/depute/cvlist.cfm", "deputies", reset) for dep in soup('table')[4]('tr'): items = dep('td') full_name = re.sub(' +', ' ', items[0].a.text).strip() url = items[0].a['href'] party = get_or_create(Party, name=items[1].a.text, url=dict(items[1].a.attrs)['href']) email = items[2].a.text website = items[3].a['href'] if items[3].a else None # yes, one deputies key contains a O instead of an 0, I'm not joking lachambre_id = re.search('key=([0-9O]+)', url).groups()[0] if not Deputy.objects.filter(lachambre_id=lachambre_id): deputy = Deputy.objects.create( full_name=full_name, party=party, url=url, websites=[website] if website else [], lachambre_id=lachambre_id, emails=[email]) logger.debug( 'adding new deputy %s %s %s %s %s' % (lachambre_id.encode("Utf-8"), full_name.encode("Utf-8"), party, email.encode("Utf-8"), website.encode("Utf-8") if website else '')) logger.info("[NEW] deputy: %s" % deputy)
def scrape(): soup, suppe = read_or_dl_with_nl( "http://www.lachambre.be/kvvcr/showpage.cfm?section=/comm/commissions&language=fr&cfm=/site/wwwcfm/comm/LstCom.cfm&rightmenu=right_cricra", "commissions list") _type = "" for i, j in zip(soup("div", id="story")[1], suppe("div", id="story")[1]): if not isinstance(i, NavigableString) and (i.h4 or i.a): if i.h4: _type = i.h4.text _type_nl = j.h4.text elif i.a: commission = get_or_create(Commission, lachambre_id=int( re.search( "com=(\d+)", i.a["href"]).groups()[0])) commission.type["fr"] = _type commission.type["nl"] = _type_nl commission.name["fr"] = i.a.text commission.name["nl"] = j.a.text commission.url = i.a["href"] commission.save() for com in list(Commission.objects.all()): handle_commission(com)
def handle_commission(commission): soup, suppe = read_or_dl_with_nl(LACHAMBRE_PREFIX + commission.url, "commission %s" % commission.lachambre_id) commission.full_name["fr"] = soup.h1.text commission.full_name["nl"] = suppe.h1.text commission.deputies = [] seats = {"fr": {}, "nl": {}} for i, j in zip(soup('p'), suppe('p')): role = i.b.text[:-1] role_nl = j.b.text[:-1] for dep in i('a'): deputy = Deputy.objects.get(lachambre_id=re.search( "key=([O0-9]+)", dep["href"]).groups()[0]) membership = get_or_create(CommissionMembership, deputy=deputy, commission=commission) membership.role = role membership.save() commission.deputies.append(membership.id) seats["fr"][role] = map( lambda x: (x[0], len(x[1].split(','))), zip(map(lambda x: x.text[:-1], i('b')[1:]), str(i).split("<br />")[1:])) seats["nl"][role_nl] = map( lambda x: (x[0], len(x[1].split(','))), zip(map(lambda x: x.text[:-1], i('b')[1:]), str(i).split("<br />")[1:])) commission.seats = seats commission.save()
def post(self): form = AddBookForm() if form.validate(): if session.query(Book).filter_by(title=form.title.data).count() == 0: print form.title.data book = get_or_create(Book, title=form.title.data) for name in str_separator(form.name.data): book.authors.append(get_or_create(Author, name=name)) session.commit() msg = 'Book <strong class=badge>{0}</strong> added!'.format(form.title.data) flash(msg) return redirect(url_for('add_book')) else: msg = 'Book <strong class=badge>{0}</strong> already \ exists!'.format(form.title.data) flash(msg) return render_template('add_book.haml', form=form)
def add_participant(self, first_name, last_name, email, session): participant = get_or_create(session, Student, first_name=first_name, last_name=last_name, email=email) if participant not in self.participants: self.participants.append(participant)
def _extract_features(task, view, sequence, func_name, func, left, right, do_update): function = get_or_create( Function, keys=dict( task=task, name=func_name, left=left, right=right ) ) labels = Labels.select().where( Labels.task == task, Labels.sequence == sequence ) data = Data.select().where( Data.view == view, Data.sequence == sequence ) df = pd.DataFrame( data=[dd.x for dd in data], index=[dd.t for dd in data], columns=view.columns ) df.index.name = 't' rows = [] for label in labels: inds = ((df.index > label.t + left) & (df.index <= label.t + right)) seq = df[inds] f_of_x = func(seq) f_of_x = parse_feature_type(f_of_x) assert np.all(np.isfinite(f_of_x)), 'Extracted features contain NaNs, with\ndf=\n{}\n\nf(df) = {}'.format( df[inds], f_of_x ) if do_update: Features.update( x=f_of_x ).where( Features.label == label, Features.function == function ).execute() else: rows.append(dict( function=function, label=label, x=f_of_x )) if not do_update: Features.insert_many(rows).execute()
def register_split_definition(self, task, partition_name, split_num, split, tag=None): assert isinstance(split, dict) tag = tag or '' partition = get_or_create( Partition, keys=dict( task=task, name=partition_name ) ) get_or_create( SplitDefinition, keys=dict( tag=tag, split_num=split_num, partition=partition ), values=split )
def _get_deputie_commissions(soup, deputy): # here we will walk in a list of h4 .. h5 .. div+ .. h5 .. div+ # look at the bottom of each deputies' page membership = soup.find('td', rowspan="1") item = membership.h4 role = None deputy.commissions = [] while item.nextSibling: if hasattr(item, 'tag'): if item.name == 'h5': role = item.text[6:-1] elif item.name == 'div': logger.debug("linking deputy to commission %s" % item.a.text) commission = get_or_create( Commission, url=item.a['href'], lachambre_id=int( re.search("com=(\d+)", item.a["href"]).groups()[0])) deputy.commissions.append( get_or_create(CommissionMembership, commission=commission, role=role)) item = item.nextSibling
def log(json_data): logging.info('Received JSON: %s' % json_data) payload = json.loads(json_data) nickname = payload.get('nickname') server = payload.get('server', 'unknown') channel = payload.get('channel') message_type = payload.get('message_type') message_content = payload.get('message_content') timestamp = parse_timestamp(payload.get('timestamp')) user = get_or_create(User, server=server, nickname=nickname) user.last_seen_at = datetime.utcnow() user.put() channel = get_or_create(Channel, server=server, channel=channel) # store the message msg = Message(user=user, channel=channel, message_type=message_type, message_content=message_content, json=json_data, timestamp=timestamp) msg.put() return msg
def push_unsent_payloads(self): # we need to create our own session here as we're running in a new thread local_session = self.Session() while True: # Infinite loop while parent process is working # check if internet access if active_internet_connection(self.ipaddress): unsent_sensordata = local_session.query(SensorData).filter_by(sent=False) for sensordata in unsent_sensordata: for consumer in self.payload_consumers: if not isinstance(consumer, PushDataBase): consumer_obj = consumer() else: consumer_obj = consumer # get SensorDataPushState entry for this payload. If it doesn't exist. Create it consumer_push_state, created = get_or_create(local_session, SensorDataPushState, defaults={'timestamp': str(datetime.utcnow()), 'attempts': 0, 'aborted': False}, sensordata_id=sensordata.id, consumer=str(consumer_obj.name)) # check to see if has been sent already. it's it's just been created then the answer is no if created or (not consumer_push_state.aborted and not consumer_push_state.sent): # we need to check that it was successful. push_success = consumer_obj.push(sensordata) consumer_push_state.attempts += 1 # add one to the number of attempts consumer_push_state.timestamp = str(datetime.utcnow()) if push_success: consumer_push_state.sent = True else: if consumer_push_state.attempts > consumer_obj.max_retries: logging.error('%s push aborted after too many retries' % consumer_push_state) consumer_push_state.aborted = True # check to see if there are any more consumers of this data left to push. # if not then we can now mark it as sent unsent_count = local_session.query(func.count(SensorDataPushState.id))\ .filter_by(sensordata_id=sensordata.id, sent=False, aborted=False).scalar() if unsent_count == 0: sensordata.sent = True local_session.commit()
def test_get_or_create_created(self, m): from utils import get_or_create quiz_id = 5 course_id = 1 quiz, created = get_or_create( views.db.session, Quiz, canvas_id=quiz_id, course_id=course_id, ) self.assertTrue(created) self.assertIsInstance(quiz, Quiz) self.assertEqual(quiz.canvas_id, quiz_id) self.assertEqual(quiz.course_id, course_id)
def _get_deputy_analysis(url, deputy, type, reset=False): soupsoup = read_or_dl(LACHAMBRE_PREFIX + lame_url(url), '%s %s' % (deputy.full_name, type), reset) setattr(deputy, "analysis_%s_url" % type, url) setattr(deputy, "analysis_%s_list" % type, []) for i in soupsoup('table')[3]('tr', valign="top"): logger.debug("add", type, i.tr('td')[1].text.strip()) dico = table2dic(i.table('td')) logger.debug("%s" % dico) getattr(deputy, "analysis_%s_list" % type).\ append(get_or_create(Analysis, _id="lachambre_id", lachambre_id=re.search("dossierID=([0-9A-Za-z-]+)", i.a["href"]).groups()[0], title=dico["Titre"], descriptor=dico["Descripteurs"], url=i.a['href'], type=type))
def extract_features(connection, task, view, func, func_name, left, right, force): assert left < right; task = connection.tasks[task] view = connection.views[view] function = get_or_create( Function, keys=dict( task=task, name=func_name, left=left, right=right ) ) num_features = Features.select().where( Features.function == function ).count() num_labels = Labels.select().where( Labels.task == task ).count() if not force: if num_features == num_labels: return do_update = num_features == num_labels if num_features != num_labels: Features.delete().where( Features.function == function ).execute() for _, sequence in connection.itersequences(): print task, view, sequence, function _extract_features( task=task, view=view, sequence=sequence, func_name=func_name, func=func, left=left, right=right, do_update=do_update )
def _get_competences(dico, dico_nl, document): # FIXME: meh, DRY if dico.get(u"Compétence") and dico_nl.get(u"Bevoegdheid"): document.timeline = [] for (_date, _title), (_, _title_nl) in zip([clean_text(x).split(u" \xa0 ", 1) for x in dico[u"Compétence"]["head"].contents[::2]], [clean_text(x).split(u" \xa0 ", 1) for x in dico_nl[u"Bevoegdheid"]["head"].contents[::2]]): logger.debug("append time line %s %s %s" % (_date, _title, _title_nl)) document.timeline.append(DocumentTimeLine.objects.create(title={"fr": _title, "nl": _title_nl}, date=_date)) elif dico.get(u"Compétence"): document.timeline = [] for (_date, _title) in [clean_text(x).split(u" \xa0 ", 1) for x in dico[u"Compétence"]["head"].contents[::2]]: logger.debug("append time line %s %s %s" % (_date, _title, "")) document.timeline.append(DocumentTimeLine.objects.create(title={"fr": _title, "nl": ""}, date=_date)) elif dico_nl.get(u"Bevoegdheid"): document.timeline = [] for (_date, _title_nl) in [clean_text(x).split(u" \xa0 ", 1) for x in dico_nl[u"Bevoegdheid"]["head"].contents[::2]]: logger.debug("append time line %s %s %s" % (_date, "", _title_nl)) document.timeline.append(DocumentTimeLine.objects.create(title={"fr": "", "nl": _title_nl}, date=_date)) if dico.get("Analyse des interventions"): document.analysis = get_or_create(Analysis, _id="lachambre_id", lachambre_id=dico["Analyse des interventions"]["head"].a.text, url=dico["Analyse des interventions"]["head"].a["href"])
def handle_commission(commission): soup, suppe = read_or_dl_with_nl(LACHAMBRE_PREFIX + commission.url, "commission %s" % commission.lachambre_id) commission.full_name["fr"] = soup.h1.text commission.full_name["nl"] = suppe.h1.text commission.deputies = [] seats = {"fr": {}, "nl": {}} for i, j in zip(soup('p'), suppe('p')): role = i.b.text[:-1] role_nl = j.b.text[:-1] for dep in i('a'): deputy = Deputy.objects.get(lachambre_id=re.search("key=([O0-9]+)", dep["href"]).groups()[0]) membership = get_or_create(CommissionMembership, deputy=deputy, commission=commission) membership.role = role membership.save() commission.deputies.append(membership.id) seats["fr"][role] = map(lambda x: (x[0], len(x[1].split(','))), zip(map(lambda x: x.text[:-1], i('b')[1:]), str(i).split("<br />")[1:])) seats["nl"][role_nl] = map(lambda x: (x[0], len(x[1].split(','))), zip(map(lambda x: x.text[:-1], i('b')[1:]), str(i).split("<br />")[1:])) commission.seats = seats commission.save()
def scrape(): soup, suppe = read_or_dl_with_nl("http://www.lachambre.be/kvvcr/showpage.cfm?section=/comm/commissions&language=fr&cfm=/site/wwwcfm/comm/LstCom.cfm&rightmenu=right_cricra", "commissions list") _type = "" for i, j in zip(soup("div", id="story")[1], suppe("div", id="story")[1]): if not isinstance(i, NavigableString) and (i.h4 or i.a): if i.h4: _type = i.h4.text _type_nl = j.h4.text elif i.a: commission = get_or_create(Commission, lachambre_id=int(re.search("com=(\d+)", i.a["href"]).groups()[0])) commission.type["fr"] = _type commission.type["nl"] = _type_nl commission.name["fr"] = i.a.text commission.name["nl"] = j.a.text commission.url = i.a["href"] commission.save() for com in list(Commission.objects.all()): handle_commission(com)
def register_partition(self, task, partition_name, key_func, force=False): assert isinstance(task, Task) partition = get_or_create( Partition, keys=dict( task=task, name=partition_name ) ) label_query = Labels.select().where( Labels.task == task ) group_query = GroupDefinition.select().where( GroupDefinition.partition == partition ) update = group_query.count() == label_query.count() if update and not force: return if update: print 'updating' for label in label_query: GroupDefinition.update( fold=key_func(label) ).where( SplitDefinition.partition == partition, SplitDefinition.label == label ).execute() else: print 'saving' rows = [dict( partition=partition, label=label, key=key_func(label) ) for label in label_query] GroupDefinition.insert_many(rows).execute()
def deputies_list(reset=False): soup = read_or_dl("http://www.lachambre.be/kvvcr/showpage.cfm?section=/depute&language=fr&rightmenu=right_depute&cfm=/site/wwwcfm/depute/cvlist.cfm", "deputies", reset) for dep in soup('table')[4]('tr'): items = dep('td') full_name = re.sub(' +', ' ', items[0].a.text).strip() url = items[0].a['href'] party = get_or_create(Party, name=items[1].a.text, url=dict(items[1].a.attrs)['href']) email = items[2].a.text website = items[3].a['href'] if items[3].a else None # yes, one deputies key contains a O instead of an 0, I'm not joking lachambre_id = re.search('key=([0-9O]+)', url).groups()[0] if not Deputy.objects.filter(lachambre_id=lachambre_id): deputy = Deputy.objects.create(full_name=full_name, party=party, url=url, websites=[website] if website else [], lachambre_id=lachambre_id, emails=[email]) logger.debug('adding new deputy %s %s %s %s %s' % (lachambre_id.encode("Utf-8"), full_name.encode("Utf-8"), party, email.encode("Utf-8"), website.encode("Utf-8") if website else '')) logger.info("[NEW] deputy: %s" % deputy)
def _get_deputy_questions(url, deputy, type, reset=False): soupsoup = read_or_dl(LACHAMBRE_PREFIX + lame_url(url), '%s %s' % (deputy.full_name, type), reset) setattr(deputy, "questions_%s_url" % type, url) setattr(deputy, "questions_%s_list" % type, []) for i in soupsoup('table')[3]('tr', valign="top"): logger.debug("add", type, i.tr('td')[1].text.strip()) dico = table2dic(i.table('td')) logger.debug("%s" % dico) getattr(deputy, "questions_%s_list" % type).\ append(get_or_create(Question, _id="lachambre_id", title=dico["Titre"], lachambre_id=re.search("dossierID=([0-9A-Za-z-]+)", i.a["href"]).groups()[0], reunion_type=dico.get(u"Réunion"), reunion_date=dico.get("Date discussion"), session_id=dico.get("Session"), pdf_url=dico.get(u"Compte rendu intégral", {"href": None})["href"], eurovoc_descriptors=map(lambda x: x.strip(), dico.get("Descripteurs Eurovoc", "").split('|')), keywords=map(lambda x: x.strip(), dico.get(u"Mots-clés libres", "").split("|")), url=i.a['href'], type=type))
def _create_base_product(self): """ Creates base product without relationships that require its existence formerly. Needs objects from INDEPENDENT object names to exist first. Product is then flushed to the database but not committed. :return: product, instance of a product """ product_class = get_class_by_table_name(f'{self.product_type}_product') if product_class: # build kwargs for product product_kwargs = {} for field in self.product_config[BASE_FIELDS]: if field in self.product_config[SINGLE_RELATIONS]: product_kwargs[field] = self.objects[field] else: product_kwargs[field] = self.data[field] product, _ = get_or_create(product_class, **product_kwargs) db_session.add(product) db_session.flush() self.objects['product_id'] = product.id return product raise ValueError('Unknown product class')
def post(self, name): print name form = EditAuthorForm() author = get_or_404(Author, name=name) if form.validate(): if author.name != form.name.data: author.name = form.name.data try: session.commit() except: session.rollback() msg = 'Author <strong class=badge>{0}</strong> already \ exists!'.format(form.name.data) flash(msg) # return render_template('edit_author.haml', form=form, name=name) return self.get(name) new_books = str_separator(form.title.data) old_books = get_books(author, format='list') # get what to delete from exists books list and what to add, to bring # to edited field books rm_lst, append_lst = diff_list(old_books, new_books) print(rm_lst, append_lst) # remove not related books for title in rm_lst: b = get_or_404(Book, title=title) author.books.remove(b) if len(b.authors) == 1: session.delete(b) # relate new books with exist author for title in append_lst: b = get_or_create(Book, title=title) author.books.append(b) session.commit() return redirect(url_for('index')) return render_template('edit_author.haml', form=form, name=name)
def post(self, title): book = get_or_404(Book, title=title) form = EditBookForm() if form.validate(): if book.title != form.title.data: book.title = form.title.data try: session.commit() except: session.rollback() msg = 'Book <strong class=badge>{0}</strong> already \ exists!'.format(form.title.data) flash(msg) # return render_template('edit_book.haml', form=form, title=title) return self.get(title) # Get current and changed list of authors. new_authors = str_separator(form.name.data) old_authors = get_authors(book, format='list') # We get authors' names which have to be deleted from an original list # and also authors' names which have to be added. rm_lst, append_lst = diff_list(old_authors, new_authors) # remove not related authors for name in rm_lst: a = get_or_404(Author, name=name) book.authors.remove(a) if len(a.books) == 1: session.delete(a) # relate new authors with exist book for name in append_lst: a = get_or_create(Author, name=name) book.authors.append(a) session.commit() return redirect(url_for('index')) return render_template('edit_book.haml', form=form, title=title)
def login(): credentials = request.get_json(force=True) email = credentials['email'] app_id = credentials['app_id'] password = credentials['password'] user = db.session.query(UserAccount).filter_by(email=email).first() if user and user.check_password(password): if db.session.query(Application).filter_by(id=app_id).first(): token = generate_token() user_app, created = get_or_create(UserApplication, application_id=app_id, user_id=user.id) if not created: user_app.last_login = datetime.datetime.now() user_app.token = token else: user_app.token = token db.session.add(user_app) db.session.commit() return jsonify({"code": 0, "token": token}) else: return jsonify({"code": 2, "token": None}) else: return jsonify({"code": 2, "token": None})
def add_grade(): """ Запрос на добавление оценки студенту по дисциплине :param student_id: - int - идентификатор студента :param class_id: - int - идентификатор дисциплины :param score: - int - оценка(от 0 до 5) :return: Если дисциплинна уже проставленна, оценка заменятеся """ data = request.get_json(force=True) student = Student.query.get_or_404(data['student_id']) c_class = Class.query.get_or_404(data['class_id']) grade = get_or_create( db.session, Grade, student_id=student.id, class_id=c_class.id ) grade.score = data['score'] db.session.commit() return jsonify({"status": "ok"})
def __init__(self, connection_path, meta_path): self.connection_params = json.load(open(connection_path, 'r')) db.init( self.connection_params['database'], **self.connection_params['credentials'] ) db.create_tables(all_tables, safe=True) self.db = db self.meta = Metadata(meta_path) # Load the dataset object self.dataset, created = Dataset.get_or_create( name=self.meta.name ) # Load the tasks self.tasks = {} for task in self.meta.tasks: self.tasks[task['name']] = get_or_create( Task, keys=dict( dataset=self.dataset, name=task['name'] ), values=dict( columns=task['columns'], num_columns=len(task['columns']) ) ) # Load the views self.views = {} for view in self.meta.views: self.views[view['name']] = get_or_create( View, keys=dict( dataset=self.dataset, name=view['name'] ), values=dict( columns=view['columns'], num_columns=len(view['columns']) ) ) # Load the sequences self.sequences = {} for sequence in self.meta.sequences: self.sequences[sequence['name']] = get_or_create( Sequence, keys=dict( dataset=self.dataset, name=sequence['name'] ), values=dict( t_min=sequence['t_min'], t_max=sequence['t_max'] ) ) # Load the partitions self.splits = {} self.partitions = {} self.classifiers = {} self.feature_unions = {} for task_name, task in self.tasks.iteritems(): pp = {pp.name: pp for pp in task.partitions} self.partitions[task_name] = pp self.partitions[task] = pp ss = {pp.name: list(pp.splits.execute()) for pp in task.partitions} self.splits[task_name] = ss self.splits[task] = ss cc = {} for classifier in task.classifiers: cc[classifier.name] = classifier self.classifiers[task] = cc self.classifiers[task_name] = cc ff = {} for feature_union in task.feature_unions: ff[feature_union.name] = feature_union self.feature_unions[task] = ff self.feature_unions[task_name] = ff
def setTags(self, tags): self.tags = [get_or_create(Tag, title=tag) for tag in tags]
def refresh_background(course_id): """ Look up existing extensions and apply them to new quizzes. :param course_id: The Canvas ID of the Course. :type course_id: int :rtype: dict :returns: A dictionary containing two parts: - success `bool` False if there was an error, True otherwise. - message `str` A long description of success or failure. """ job = get_current_job() update_job(job, 0, 'Starting...', 'started') with app.app_context(): course, created = get_or_create( db.session, Course, canvas_id=course_id ) try: course_name = get_course(course_id).get( 'name', '<UNNAMED COURSE>' ) course.course_name = course_name db.session.commit() except requests.exceptions.HTTPError: update_job( job, 0, 'Course not found.', 'failed', error=True ) logger.exception('Unable to find course #{}'.format(course_id)) return job.meta # quiz stuff quizzes = missing_quizzes(course_id) num_quizzes = len(quizzes) if num_quizzes < 1: update_job( job, 100, 'Complete. No quizzes required updates.', 'complete', error=False ) return job.meta percent_user_map = defaultdict(list) inactive_list = [] update_job(job, 0, 'Getting past extensions.', 'processing', False) for extension in course.extensions: # If extension is inactive, ignore. if not extension.active: inactive_list.append(extension.user.sortable_name) logger.debug('Extension #{} is inactive.'.format( extension.id )) continue user_canvas_id = User.query.filter_by( id=extension.user_id ).first().canvas_id # Check if user is in course. If not, deactivate extension. try: canvas_user = get_user(course_id, user_canvas_id) # Skip user if not a student. Fixes an edge case where a # student that previously recieved an extension changes roles. enrolls = canvas_user.get('enrollments', []) type_list = [e['type'] for e in enrolls if e['enrollment_state'] == 'active'] if not any(t == 'StudentEnrollment' for t in type_list): logger.info(( "User #{} was found in course #{}, but is not an " "active student. Deactivating extension #{}. Roles " "found: {}" ).format( user_canvas_id, course_id, extension.id, ", ".join(type_list) if len(enrolls) > 0 else None )) extension.active = False db.session.commit() inactive_list.append(extension.user.sortable_name) continue except requests.exceptions.HTTPError: log_str = ( 'User #{} not in course #{}. Deactivating extension #{}.' ) logger.info( log_str.format(user_canvas_id, course_id, extension.id) ) extension.active = False db.session.commit() inactive_list.append(extension.user.sortable_name) continue percent_user_map[extension.percent].append(user_canvas_id) if len(percent_user_map) < 1: msg_str = 'No active extensions were found.<br>' if len(inactive_list) > 0: msg_str += ' Extensions for the following students are inactive:<br>{}' msg_str = msg_str.format("<br>".join(inactive_list)) update_job( job, 100, msg_str, 'complete', error=False ) return job.meta for index, quiz in enumerate(quizzes): quiz_id = quiz.get('id', None) quiz_title = quiz.get('title', '[UNTITLED QUIZ]') comp_perc = int(((float(index)) / float(num_quizzes)) * 100) refreshing_str = 'Refreshing quiz #{} - {} [{} of {}]' update_job( job, comp_perc, refreshing_str.format( quiz_id, quiz_title, index + 1, num_quizzes ), 'processing', error=False ) for percent, user_list in percent_user_map.iteritems(): extension_response = extend_quiz( course_id, quiz, percent, user_list ) if extension_response.get('success', False) is True: # add/update quiz quiz_obj, created = get_or_create( db.session, Quiz, canvas_id=quiz_id, course_id=course.id ) quiz_obj.title = quiz_title db.session.commit() else: error_message = 'Some quizzes couldn\'t be updated. ' error_message += extension_response.get('message', '') update_job( job, comp_perc, error_message, 'failed', error=True, ) return job.meta msg = '{} quizzes have been updated.'.format(len(quizzes)) update_job(job, 100, msg, 'complete', error=False) return job.meta
def get_new_documents(): for document_page in read_or_dl("http://www.lachambre.be/kvvcr/showpage.cfm?section=/flwb&language=fr&rightmenu=right&cfm=ListDocument.cfm", "all documents")('div', **{'class': re.compile("linklist_[01]")}): soup, suppe = read_or_dl_with_nl(LACHAMBRE_PREFIX + document_page.a["href"], "document %s" % document_page.a.text) for soup, suppe in zip(soup('table')[4]('tr', valign="top"), suppe('table')[4]('tr', valign="top")): get_or_create(Document, _id="lachambre_id", title={"fr": soup('div')[1].text, "nl": suppe('div')[1].text}, lachambre_id=soup.div.text, url=soup.a["href"])
def update_background(course_id, extension_dict): """ Update time on selected students' quizzes to a specified percentage. :param course_id: The Canvas ID of the Course to update in :type course_id: int :param extension_dict: A dictionary that includes the percent of time and a list of canvas user ids. Example: { 'percent': '300', 'user_ids': [ '0123456', '1234567', '9867543', '5555555' ] } :type extension_dict: dict """ job = get_current_job() update_job(job, 0, 'Starting...', 'started') with app.app_context(): if not extension_dict: update_job( job, 0, 'Invalid Request', 'failed', error=True ) logger.warning('Invalid Request: {}'.format(extension_dict)) return job.meta try: course_json = get_course(course_id) except requests.exceptions.HTTPError: update_job( job, 0, 'Course not found.', 'failed', error=True ) logger.exception('Unable to find course #{}'.format(course_id)) return job.meta course_name = course_json.get('name', '<UNNAMED COURSE>') user_ids = extension_dict.get('user_ids', []) percent = extension_dict.get('percent', None) if not percent: update_job( job, 0, '`percent` field required.', 'failed', error=True ) logger.warning('Percent field not provided. Request: {}'.format( extension_dict )) return job.meta course, created = get_or_create(db.session, Course, canvas_id=course_id) course.course_name = course_name db.session.commit() for user_id in user_ids: try: canvas_user = get_user(course_id, user_id) sortable_name = canvas_user.get('sortable_name', '<MISSING NAME>') sis_id = canvas_user.get('sis_user_id') except requests.exceptions.HTTPError: # Unable to find user. Log and skip them. logger.warning( "Unable to find user #{} in course #{}".format( user_id, course_id ) ) continue user, created = get_or_create(db.session, User, canvas_id=user_id) user.sortable_name = sortable_name user.sis_id = sis_id db.session.commit() # create/update extension extension, created = get_or_create( db.session, Extension, course_id=course.id, user_id=user.id ) extension.percent = percent db.session.commit() quizzes = get_quizzes(course_id) num_quizzes = len(quizzes) quiz_time_list = [] unchanged_quiz_time_list = [] if num_quizzes < 1: update_job( job, 0, 'Sorry, there are no quizzes for this course.', 'failed', error=True ) logger.warning( "No quizzes found for course {}. Unable to update.".format( course_id ) ) return job.meta for index, quiz in enumerate(quizzes): quiz_id = quiz.get('id', None) quiz_title = quiz.get('title', "[UNTITLED QUIZ]") comp_perc = int(((float(index)) / float(num_quizzes)) * 100) updating_str = 'Updating quiz #{} - {} [{} of {}]' update_job( job, comp_perc, updating_str.format(quiz_id, quiz_title, index + 1, num_quizzes), 'processing', error=False ) extension_response = extend_quiz(course_id, quiz, percent, user_ids) if extension_response.get('success', False) is True: # add/update quiz quiz_obj, created = get_or_create( db.session, Quiz, canvas_id=quiz_id, course_id=course.id ) quiz_obj.title = quiz_title db.session.commit() added_time = extension_response.get('added_time', None) if added_time is not None: quiz_time_list.append({ "title": quiz_title, "added_time": added_time }) else: unchanged_quiz_time_list.append({"title": quiz_title}) else: update_job( job, comp_perc, extension_response.get( 'message', 'An unknown error occured.' ), 'failed', error=True ) logger.error("Extension failed: {}".format(extension_response)) return job.meta msg_str = ( 'Success! {} {} been updated for {} student(s) to have {}% time. ' '{} {} no time limit and were left unchanged.' ) message = msg_str.format( len(quiz_time_list), "quizzes have" if len(quiz_time_list) != 1 else "quiz has", len(user_ids), percent, len(unchanged_quiz_time_list), "quizzes have" if len(unchanged_quiz_time_list) != 1 else "quiz has" ) update_job(job, 100, message, 'complete', error=False) job.meta['quiz_list'] = quiz_time_list job.meta['unchanged_list'] = unchanged_quiz_time_list job.save() return job.meta
def _createTags(self, tags): return [get_or_create(Tag, title=tag) for tag in tags]
forks = determine_capacity(v2) if forks is None: raise RuntimeException("Capactiy not determinable") forks = forks - 1 print("Forks set to {}".format(forks)) cancel_all(v2.jobs) cancel_all(v2.workflow_jobs) delete_all(v2.jobs) delete_all(v2.workflow_jobs) delete_all(v2.workflow_job_templates) delete_all(v2.inventory) delete_all(v2.job_templates) org = get_or_create(v2.organizations, name='cmurders murder inc.') inv = get_or_create(org.related.inventories, name='cmurders murdertown', organization=org, variables=inventory_vars) for i in xrange(0, forks + 5): h = get_or_create(inv.related.hosts, name='high-capacity-host-{}'.format(i), inventory=inv) proj = get_or_create( v2.projects, name='Murder Tools', organization=org, scm_url='https://github.com/chrismeyersfsu/ansible-examples.git',
import time import sys, os sys.path.append(os.path.join(os.getcwd(), '..', '..')) from utils import get_or_create, delete_all host_vars = """ansible_connection: local""" sleep_interval_vars = """--- sleep_min: 240 sleep_max: 480 """ delete_all(v2.jobs) org = get_or_create(v2.organizations, name='cmurders murder inc.') inv = get_or_create(org.related.inventories, name='cmurders murdertown', organization=org) h = get_or_create(inv.related.hosts, name='cmurder', variables=host_vars, inventory=inv) proj = get_or_create(v2.projects, name='Murder Tools', organization=org, scm_url='https://github.com/chrismeyersfsu/ansible-examples.git', wait=True) jt = get_or_create(v2.job_templates, name="Kill Number 1", organization=org, inventory=inv, project=proj, extra_vars=sleep_interval_vars, playbook='sleep.yml', allow_simultaneous=True) wfjt = get_or_create(v2.workflow_job_templates, name="Mass Murder", organization=org) parallel_count = 100 current_node_count = wfjt.related.workflow_nodes.get()['count'] if current_node_count < parallel_count: for i in xrange(0, parallel_count): wfjt.related.workflow_nodes.post(dict(unified_job_template=jt.id)) wfj = None #wfj = wfjt.launch().wait_until_completed()
def on_event(): """Handles an event from Hangouts Chat.""" engine = create_engine(DATABASE_URL) if not engine.dialect.has_table(engine, USER_TABLE_NAME): Base.metadata.create_all(engine) Base.metadata.bind = engine DBSession = sessionmaker(bind=engine) session = DBSession() event = request.get_json() email = event['message']['sender']['email'] text = event['message']['text'] if event['type'] == 'ADDED_TO_SPACE' and event['space']['type'] == 'ROOM': text = 'Thanks for adding me to "{0}"!'.format( event['space']['displayName']) elif event['type'] == 'MESSAGE': message = event['message']['text'] message = message.replace("@visualtime", "") if message.startswith('/login'): if len(message.split()) >= 2: password = message.split()[1] get_or_create(session, User, email=email, password=password) text = 'Usuario {0} ha sido iniciado correctamente'.format( email) if message.startswith('/info'): users = session.query(User).filter(User.email == email) if users.count() == 0: text = "Oppp!! usa /login <password> para iniciar tu usuario" else: user = users.first() visualtime_client = VisualTimeHelper(user.email, user.password) visualtime_client.login() visualtime_info = visualtime_client.get_output_time() text = 'Hola {0}: \n' \ 'Llevas trabajando: {1} \n' \ 'Hoy te vas a casa a las: {2} \n' \ 'El porcentaje de avance de tu jornada es: {3} % \n' \ 'La jornada de hoy tiene: {4} horas \n' \ 'Ahora mismo te encuentras: {5}'.format(user.email, visualtime_info['working_time'], visualtime_info['output_time'].split()[1], visualtime_info['percent'], visualtime_info['day'], "trabajando" if int(visualtime_info['direction']) == 1 else "descansando") elif message.startswith("/push"): users = session.query(User).filter(User.email == email) if users.count() == 0: text = u"Oppp!! usa /login <password> para iniciar tu usuario" else: user = users.first() visualtime_client = VisualTimeHelper(user.email, user.password) visualtime_client.login() visualtime_info = visualtime_client.push() if visualtime_info: text = u"Ok!! Fichaje creado con éxito." else: text = u"Upps! Hay un error creado el fichaje." elif message.startswith("/logout"): users = session.query(User).filter(User.email == email) if users.count() == 0: text = u"Oppp!! usa /login <password> para iniciar tu usuario" else: users.delete(synchronize_session=False) text = u"Ha cerrado sesión con éxito." else: return return jsonify({'text': text})
forks = determine_capacity(v2) if forks is None: raise RuntimeException("Capactiy not determinable") forks = forks - 1 print("Forks set to {}".format(forks)) cancel_all(v2.jobs) cancel_all(v2.workflow_jobs) delete_all(v2.jobs) delete_all(v2.workflow_jobs) delete_all(v2.workflow_job_templates) delete_all(v2.inventory) delete_all(v2.job_templates) org = get_or_create(v2.organizations, name='cmurders murder inc.') inv = get_or_create(org.related.inventories, name='cmurders murdertown', organization=org, variables=inventory_vars) for i in xrange(0, forks+5): h = get_or_create(inv.related.hosts, name='high-capacity-host-{}'.format(i), inventory=inv) proj = get_or_create(v2.projects, name='Murder Tools', organization=org, scm_url='https://github.com/chrismeyersfsu/ansible-examples.git', wait=True) jt = get_or_create(v2.job_templates, name="High Capacity JT - {} forks".format(forks), organization=org, inventory=inv, project=proj, extra_vars=sleep_interval_vars, playbook='sleep.yml', allow_simultaneous=True, forks=forks) wfjt = get_or_create(v2.workflow_job_templates, name="High Capacity Workflow", organization=org) parallel_count = 100 current_node_count = wfjt.related.workflow_nodes.get()['count'] if current_node_count < parallel_count: for i in xrange(0, parallel_count):
def setTags(self, tags): self.tags = [get_or_create(Tag, title = tag) for tag in tags]
def _createTags(self, tags): return [get_or_create(Tag, title = tag) for tag in tags]