Ejemplo n.º 1
0
def save_objects(session, data, region, date):
    position, track_name, artist_name, url = data
    artist = get_or_create(session, Artist, name=artist_name)

    track_vals = {
        'title': track_name,
        'artist': artist
    }
    track = get_or_create(session, Track, **track_vals)

    link_vals = {
        'url': url,
        'track': track
    }
    link = get_or_create(session, Link, **link_vals)

    chart_vals = {
        'type': 'viral50',
        'region': region,
        'position': position,
        'streams': None,
        'date': date,
        'track': track
    }
    chart = get_or_create(session, Chart, **chart_vals)
Ejemplo n.º 2
0
def build_db():
    '''
    Build a small db with some example entries.

    '''

    names = [
        'Oleg', 'Mark', 'William', 'James', 'Harry', 'Amelia', 'Oliver', 'Jack', 'John'
    ]

    books = [
        'MySQL for Python', 'MySQL Cookbook', 'Essential SQLAlchemy', 'Learning Python',
        'Python for Data Analysis', 'Core Python Applications Programming', 'Python Algorithms'
        'Think Python', 'Computer Games with Python', 'MongoDb in action'
    ]

    user = {'name': 'user', 'passwd': 'like-books'}

    counter = session.query(Author).count()

    if not counter:
        for i in range(len(names)):
            author = get_or_create(Author, name=names[i])
            author.books.append(get_or_create(Book, title=books[i]))
            author.books.append(get_or_create(Book, title=random.choice(books)))
            session.commit()

        session.add(User(nickname=user['name'], password=user['passwd']))
        session.commit()
Ejemplo n.º 3
0
def _get_written_question_bulletin():
    for i in range(48, 54):
        soup = read_or_dl("http://www.lachambre.be/kvvcr/showpage.cfm?section=/qrva&language=fr&rightmenu=right?legislat=52&cfm=/site/wwwcfm/qrva/qrvaList.cfm?legislat=%i" % i, "bulletin list %i" % i)
        for b in soup('table')[4]('tr')[1:]:
            try:
                if i == 53:
                    get_or_create(WrittenQuestionBulletin,
                                  legislature="53",
                                  lachambre_id=b('td')[0]('a')[-1].text.split()[-1],
                                  date=b('td')[2].text,
                                  publication_date=b('td')[3].text,
                                  url=b('td')[1].a["href"],
                                  pdf_url=b('td')[0].a["href"],
                                 )
                else:
                    get_or_create(WrittenQuestionBulletin,
                                  legislature=str(i),
                                  lachambre_id=b('td')[0]('a')[-1].text.split()[-1],
                                  publication_date=b('td')[2].text,
                                  url=b('td')[1].a["href"] if b('td')[1].a else None,
                                  pdf_url=b('td')[0].a["href"],
                                 )
                    logger.debug("%s" % b('td')[0]('a')[-1].text.split()[-1])
            except TypeError:
                continue
Ejemplo n.º 4
0
def scrape():
    for a, url in enumerate(
        ('http://www.lachambre.be/kvvcr/showpage.cfm?section=none&language=fr&cfm=/site/wwwcfm/rajv/rajvlist.cfm?lastreports=y',
         'http://www.lachambre.be/kvvcr/showpage.cfm?section=none&language=fr&cfm=/site/wwwcfm/rajv/rajvlist.cfm?lastreports=n'
         )):
        soup, suppe = read_or_dl_with_nl(url, "annual repports %i" % a)

        for i, j in zip(
                soup.find('div', id="story")('table')[1]('tr',
                                                         recursive=False)[::5],
                suppe.find('div',
                           id="story")('table')[1]('tr',
                                                   recursive=False)[::5]):
            get_or_create(
                AnnualReport,
                title={
                    "fr": i('td')[2].text,
                    "nl": j('td')[2].text
                },
                date=i('td')[0].text,
                law_and_article={
                    "fr": i('td')[4].text,
                    "nl": j('td')[4].text
                },
                periodicity=re.sub("[^0-9]", "",
                                   i('td')[5].text),
                pdf_url=i('td')[1].a["href"] if i('td')[1].a else "",
            )
Ejemplo n.º 5
0
def _save_a_written_question(link):
    soupsoup, suppesuppe = read_or_dl_with_nl(LACHAMBRE_PREFIX + link.a["href"], "written question %s" % re.search("dossierID=([0-9A-Z-]+).xml", link.a["href"]).groups()[0])
    data = AccessControlDict(((x.td.text, x('td')[1]) for x in soupsoup.find('table', 'txt')('tr') if x.td.text))
    data_nl = AccessControlDict(((x.td.text, x('td')[1]) for x in suppesuppe.find('table', 'txt')('tr') if x.td.text))
    get_or_create(WrittenQuestion,
                  _id="lachambre_id",
                  lachambre_id=re.search("dossierID=([0-9A-Z-]+).xml", link.a["href"]).groups()[0],
                  title={"fr": data["Titre"].text, "nl": data_nl["Titel"].text},
                  departement={"fr": data[u"Département"].text, "nl": data_nl[u"Departement"].text},
                  sub_departement={"fr": data[u"Sous-département"].text, "nl": data_nl[u"Sub-departement"].text},
                  deposition_date=data[u"Date de dépôt"].text,
                  delay_date=dico_get_text(data, u"Date de délai"),
                  publication_date=dico_get_text(data, "Date publication"),
                  # TODO: link to the actual deputy
                  author=data[u"Auteur"].text,
                  language=data[u"Langue"].text,
                  question_status={"fr": dico_get_text(data, "Statut question"), "nl": dico_get_text(data_nl, "Status vraag")},
                  status={"fr": dico_get_text(data, "Statut"), "nl": dico_get_text(data_nl, "Status")},
                  question={"fr": u"%s" % data["Question"], "nl": "%s" % data_nl["Vraag"]},
                  answer={"fr": dico_get_text(data, u"Réponse"), "nl": dico_get_text(data_nl, u"Antwoord")},
                  publication_reponse_pdf_url=get_href_else_blank(data, u"Publication réponse"),
                  publication_question_pdf_url=get_href_else_blank(data, u"Publication question"),
                  publication_reponse=get_text_else_blank(data, u"Publication réponse"),
                  publication_question=get_text_else_blank(data, u"Publication question"),
                  eurovoc_descriptors={"fr": get_items_list_else_empty_list(data, "Descripteurs Eurovoc"),
                                       "nl": get_items_list_else_empty_list(data_nl, "Eurovoc-descriptoren")},
                  eurovoc_candidats_descriptors={"fr": get_items_list_else_empty_list(data, "Candidats-descripteurs Eurovoc"),
                                                 "nl": get_items_list_else_empty_list(data_nl, "Eurovoc kandidaat-descriptoren")},
                  keywords={"fr": get_items_list_else_empty_list(data, u"Mots-clés libres"),
                            "nl": get_items_list_else_empty_list(data_nl, u"Vrije trefwoorden")},
                  url=link.a["href"],
                 )

    data.die_if_got_not_accessed_keys()
Ejemplo n.º 6
0
def fill_database():
    admin_role = get_or_create(db.session, Role, name='admin')
    dj_role = get_or_create(db.session, Role, name='dj')
    default_radio = get_or_create(db.session, Radio, name='default')
    admin_user = get_or_create(db.session, User, login=config.admin_login)
    admin_user.password = pwd_context.encrypt(config.admin_password)
    admin_user.roles = [admin_role, dj_role]
    admin_user.radio = default_radio
    admin_user.is_active = True
    admin_user.comment = "admin z config.py, zawsze posiada haslo z config.py"
Ejemplo n.º 7
0
def scrape():
    for a, url in enumerate(('http://www.lachambre.be/kvvcr/showpage.cfm?section=none&language=fr&cfm=/site/wwwcfm/rajv/rajvlist.cfm?lastreports=y',
                         'http://www.lachambre.be/kvvcr/showpage.cfm?section=none&language=fr&cfm=/site/wwwcfm/rajv/rajvlist.cfm?lastreports=n')):
        soup, suppe = read_or_dl_with_nl(url, "annual repports %i" % a)

        for i, j in zip(soup.find('div', id="story")('table')[1].tbody('tr', recursive=False)[::5], suppe.find('div', id="story")('table')[1].tbody('tr', recursive=False)[::5]):
            get_or_create(AnnualReport,
                          title={"fr": i('td')[2].text, "nl": j('td')[2].text},
                          date=i('td')[0].text,
                          law_and_article={"fr": i('td')[4].text, "nl": j('td')[4].text},
                          periodicity=re.sub("[^0-9]", "", i('td')[5].text),
                          pdf_url=i('td')[1].a["href"] if i('td')[1].a else "",
                          )
Ejemplo n.º 8
0
 def post(self):
     form = AddAuthorForm()
     if form.validate():
         # if session.query(Author).filter_by(name=form.name.data).count() == 0:
         author = get_or_create(Author, name=form.name.data)
         for title in str_separator(form.title.data):
             author.books.append(get_or_create(Book, title=title))
         session.commit()
         return redirect(url_for('add_author'))
     else:
         msg = 'Author <strong class=badge>{0}</strong> already \
                exists!'.format(form.name.data)
         flash(msg)
     return render_template('add_author.haml', form=form)
Ejemplo n.º 9
0
    def test_get_or_create_already_exists(self, m):
        from utils import get_or_create

        quiz_id = 5
        quiz_title = "Final Exam"
        course_id = 1

        prebuilt_quiz = Quiz(
            canvas_id=quiz_id,
            course_id=course_id,
            title=quiz_title
        )
        views.db.session.add(prebuilt_quiz)
        views.db.session.commit()

        quiz, created = get_or_create(
            views.db.session,
            Quiz,
            canvas_id=quiz_id,
            course_id=course_id
        )
        self.assertFalse(created)
        self.assertIsInstance(quiz, Quiz)
        self.assertEqual(quiz.canvas_id, quiz_id)
        self.assertEqual(quiz.course_id, course_id)
        self.assertEqual(quiz.title, quiz_title)
Ejemplo n.º 10
0
    def _create_objects(self, object_names):
        """
        Create objects from a list of object names.
        :param object_names: list, List of object names
        :return:
        """
        for obj_name in object_names:
            if self.data.get(obj_name):
                data = self.data[obj_name]
                multiple = False
                if isinstance(data, list) or isinstance(data, dict):
                    multiple = True
                table_name = TABLE_NAMES[obj_name]
                object_class = get_class_by_table_name(table_name)

                if object_class:
                    if multiple:
                        # inject product id into data dictionary for objects that need it as a field.
                        if obj_name in self.product_config[PRODUCT_DEPENDENT]:
                            for key, value in data.items():
                                value['product_id'] = self.objects['product_id']
                        self.objects[obj_name] = get_or_create_multiple(object_class, data=data)
                        db_session.add_all(self.objects[obj_name])
                    else:
                        self.objects[obj_name], _ = get_or_create(object_class, name=self.data[obj_name])
                        db_session.add(self.objects[obj_name])
Ejemplo n.º 11
0
def _get_deputie_commissions(soup, deputy):
    # here we will walk in a list of h4 .. h5 .. div+ .. h5 .. div+
    # look at the bottom of each deputies' page
    membership = soup.find('td', rowspan="1")
    item = membership.h4
    role = None
    deputy.commissions = []
    while item.nextSibling:
        if hasattr(item, 'tag'):
            if item.name == 'h5':
                role = item.text[6:-1]
            elif item.name == 'div':
                logger.debug("linking deputy to commission %s" % item.a.text)
                commission = get_or_create(Commission, url=item.a['href'], lachambre_id=int(re.search("com=(\d+)", item.a["href"]).groups()[0]))
                deputy.commissions.append(get_or_create(CommissionMembership, commission=commission, role=role))
        item = item.nextSibling
Ejemplo n.º 12
0
def deputies_list(reset=False):
    soup = read_or_dl(
        "http://www.lachambre.be/kvvcr/showpage.cfm?section=/depute&language=fr&rightmenu=right_depute&cfm=/site/wwwcfm/depute/cvlist.cfm",
        "deputies", reset)

    for dep in soup('table')[4]('tr'):
        items = dep('td')
        full_name = re.sub('  +', ' ', items[0].a.text).strip()
        url = items[0].a['href']
        party = get_or_create(Party,
                              name=items[1].a.text,
                              url=dict(items[1].a.attrs)['href'])
        email = items[2].a.text
        website = items[3].a['href'] if items[3].a else None
        # yes, one deputies key contains a O instead of an 0, I'm not joking
        lachambre_id = re.search('key=([0-9O]+)', url).groups()[0]
        if not Deputy.objects.filter(lachambre_id=lachambre_id):
            deputy = Deputy.objects.create(
                full_name=full_name,
                party=party,
                url=url,
                websites=[website] if website else [],
                lachambre_id=lachambre_id,
                emails=[email])
            logger.debug(
                'adding new deputy %s %s %s %s %s' %
                (lachambre_id.encode("Utf-8"),
                 full_name.encode("Utf-8"), party, email.encode("Utf-8"),
                 website.encode("Utf-8") if website else ''))
            logger.info("[NEW] deputy: %s" % deputy)
Ejemplo n.º 13
0
def scrape():
    soup, suppe = read_or_dl_with_nl(
        "http://www.lachambre.be/kvvcr/showpage.cfm?section=/comm/commissions&language=fr&cfm=/site/wwwcfm/comm/LstCom.cfm&rightmenu=right_cricra",
        "commissions list")
    _type = ""
    for i, j in zip(soup("div", id="story")[1], suppe("div", id="story")[1]):
        if not isinstance(i, NavigableString) and (i.h4 or i.a):
            if i.h4:
                _type = i.h4.text
                _type_nl = j.h4.text
            elif i.a:
                commission = get_or_create(Commission,
                                           lachambre_id=int(
                                               re.search(
                                                   "com=(\d+)",
                                                   i.a["href"]).groups()[0]))
                commission.type["fr"] = _type
                commission.type["nl"] = _type_nl
                commission.name["fr"] = i.a.text
                commission.name["nl"] = j.a.text
                commission.url = i.a["href"]

                commission.save()

    for com in list(Commission.objects.all()):
        handle_commission(com)
Ejemplo n.º 14
0
def handle_commission(commission):
    soup, suppe = read_or_dl_with_nl(LACHAMBRE_PREFIX + commission.url,
                                     "commission %s" % commission.lachambre_id)
    commission.full_name["fr"] = soup.h1.text
    commission.full_name["nl"] = suppe.h1.text
    commission.deputies = []
    seats = {"fr": {}, "nl": {}}
    for i, j in zip(soup('p'), suppe('p')):
        role = i.b.text[:-1]
        role_nl = j.b.text[:-1]
        for dep in i('a'):
            deputy = Deputy.objects.get(lachambre_id=re.search(
                "key=([O0-9]+)", dep["href"]).groups()[0])
            membership = get_or_create(CommissionMembership,
                                       deputy=deputy,
                                       commission=commission)
            membership.role = role
            membership.save()
            commission.deputies.append(membership.id)
        seats["fr"][role] = map(
            lambda x: (x[0], len(x[1].split(','))),
            zip(map(lambda x: x.text[:-1],
                    i('b')[1:]),
                str(i).split("<br />")[1:]))
        seats["nl"][role_nl] = map(
            lambda x: (x[0], len(x[1].split(','))),
            zip(map(lambda x: x.text[:-1],
                    i('b')[1:]),
                str(i).split("<br />")[1:]))

    commission.seats = seats
    commission.save()
Ejemplo n.º 15
0
 def post(self):
     form = AddBookForm()
     if form.validate():
         if session.query(Book).filter_by(title=form.title.data).count() == 0:
             print form.title.data
             book = get_or_create(Book, title=form.title.data)
             for name in str_separator(form.name.data):
                 book.authors.append(get_or_create(Author, name=name))
             session.commit()
             msg = 'Book <strong class=badge>{0}</strong> added!'.format(form.title.data)
             flash(msg)
             return redirect(url_for('add_book'))
         else:
             msg = 'Book <strong class=badge>{0}</strong> already \
                    exists!'.format(form.title.data)
             flash(msg)
     return render_template('add_book.haml', form=form)
Ejemplo n.º 16
0
 def add_participant(self, first_name, last_name, email, session):
     participant = get_or_create(session,
                                 Student,
                                 first_name=first_name,
                                 last_name=last_name,
                                 email=email)
     if participant not in self.participants:
         self.participants.append(participant)
Ejemplo n.º 17
0
def _extract_features(task, view, sequence, func_name, func, left, right, do_update):
    function = get_or_create(
        Function,
        keys=dict(
            task=task,
            name=func_name,
            left=left,
            right=right
        )
    )
    
    labels = Labels.select().where(
        Labels.task == task,
        Labels.sequence == sequence
    )
    
    data = Data.select().where(
        Data.view == view,
        Data.sequence == sequence
    )
    
    df = pd.DataFrame(
        data=[dd.x for dd in data],
        index=[dd.t for dd in data],
        columns=view.columns
    )
    df.index.name = 't' 

    rows = []
    for label in labels:
        inds = ((df.index > label.t + left) &
                (df.index <= label.t + right))
        seq = df[inds]
        f_of_x = func(seq)
        f_of_x = parse_feature_type(f_of_x)
        
        assert np.all(np.isfinite(f_of_x)), 'Extracted features contain NaNs, with\ndf=\n{}\n\nf(df) = {}'.format(
            df[inds], f_of_x
        )
        
        if do_update:
            Features.update(
                x=f_of_x
            ).where(
                Features.label == label,
                Features.function == function
            ).execute()
        
        else:
            rows.append(dict(
                function=function,
                label=label,
                x=f_of_x
            ))
    
    if not do_update:
        Features.insert_many(rows).execute()
Ejemplo n.º 18
0
 def register_split_definition(self, task, partition_name, split_num, split, tag=None):
     assert isinstance(split, dict)
     
     tag = tag or ''
     
     partition = get_or_create(
         Partition,
         keys=dict(
             task=task,
             name=partition_name
         )
     )
     
     get_or_create(
         SplitDefinition,
         keys=dict(
             tag=tag,
             split_num=split_num,
             partition=partition
         ),
         values=split
     )
Ejemplo n.º 19
0
def _get_deputie_commissions(soup, deputy):
    # here we will walk in a list of h4 .. h5 .. div+ .. h5 .. div+
    # look at the bottom of each deputies' page
    membership = soup.find('td', rowspan="1")
    item = membership.h4
    role = None
    deputy.commissions = []
    while item.nextSibling:
        if hasattr(item, 'tag'):
            if item.name == 'h5':
                role = item.text[6:-1]
            elif item.name == 'div':
                logger.debug("linking deputy to commission %s" % item.a.text)
                commission = get_or_create(
                    Commission,
                    url=item.a['href'],
                    lachambre_id=int(
                        re.search("com=(\d+)", item.a["href"]).groups()[0]))
                deputy.commissions.append(
                    get_or_create(CommissionMembership,
                                  commission=commission,
                                  role=role))
        item = item.nextSibling
Ejemplo n.º 20
0
 def log(json_data):
     logging.info('Received JSON: %s' % json_data)
     
     payload = json.loads(json_data)
     
     nickname = payload.get('nickname')
     server = payload.get('server', 'unknown')
     channel = payload.get('channel')
     message_type = payload.get('message_type')
     message_content = payload.get('message_content')
     timestamp = parse_timestamp(payload.get('timestamp'))
     
     user = get_or_create(User, server=server, nickname=nickname)
     user.last_seen_at = datetime.utcnow()
     user.put()
     
     channel = get_or_create(Channel, server=server, channel=channel)
     # store the message
     msg = Message(user=user, channel=channel, message_type=message_type,
         message_content=message_content, json=json_data, 
         timestamp=timestamp)
     msg.put()
     return msg
     
Ejemplo n.º 21
0
    def push_unsent_payloads(self):
        # we need to create our own session here as we're running in a new thread
        local_session = self.Session()
        while True:  # Infinite loop while parent process is working
            # check if internet access
            if active_internet_connection(self.ipaddress):

                unsent_sensordata = local_session.query(SensorData).filter_by(sent=False)

                for sensordata in unsent_sensordata:
                    for consumer in self.payload_consumers:
                        if not isinstance(consumer, PushDataBase):
                            consumer_obj = consumer()
                        else:
                            consumer_obj = consumer

                        # get SensorDataPushState entry for this payload. If it doesn't exist. Create it
                        consumer_push_state, created = get_or_create(local_session, SensorDataPushState,
                                                                     defaults={'timestamp': str(datetime.utcnow()),
                                                                               'attempts': 0,
                                                                               'aborted': False},
                                                                     sensordata_id=sensordata.id,
                                                                     consumer=str(consumer_obj.name))
                        # check to see if has been sent already. it's it's just been created then the answer is no
                        if created or (not consumer_push_state.aborted and not consumer_push_state.sent):
                            # we need to check that it was successful.
                            push_success = consumer_obj.push(sensordata)

                            consumer_push_state.attempts += 1  # add one to the number of attempts
                            consumer_push_state.timestamp = str(datetime.utcnow())

                            if push_success:
                                consumer_push_state.sent = True
                            else:
                                if consumer_push_state.attempts > consumer_obj.max_retries:
                                    logging.error('%s push aborted after too many retries' % consumer_push_state)
                                    consumer_push_state.aborted = True

                            # check to see if there are any more consumers of this data left to push.
                            # if not then we can now mark it as sent
                            unsent_count = local_session.query(func.count(SensorDataPushState.id))\
                                .filter_by(sensordata_id=sensordata.id,
                                           sent=False,
                                           aborted=False).scalar()
                            if unsent_count == 0:
                                sensordata.sent = True

                        local_session.commit()
Ejemplo n.º 22
0
    def test_get_or_create_created(self, m):
        from utils import get_or_create

        quiz_id = 5
        course_id = 1

        quiz, created = get_or_create(
            views.db.session,
            Quiz,
            canvas_id=quiz_id,
            course_id=course_id,
        )
        self.assertTrue(created)
        self.assertIsInstance(quiz, Quiz)
        self.assertEqual(quiz.canvas_id, quiz_id)
        self.assertEqual(quiz.course_id, course_id)
Ejemplo n.º 23
0
def _get_deputy_analysis(url, deputy, type, reset=False):
    soupsoup = read_or_dl(LACHAMBRE_PREFIX + lame_url(url), '%s %s' % (deputy.full_name, type), reset)
    setattr(deputy, "analysis_%s_url" % type, url)
    setattr(deputy, "analysis_%s_list" % type, [])
    for i in soupsoup('table')[3]('tr', valign="top"):
        logger.debug("add", type, i.tr('td')[1].text.strip())
        dico = table2dic(i.table('td'))
        logger.debug("%s" % dico)
        getattr(deputy, "analysis_%s_list" % type).\
            append(get_or_create(Analysis,
            _id="lachambre_id",
            lachambre_id=re.search("dossierID=([0-9A-Za-z-]+)", i.a["href"]).groups()[0],
            title=dico["Titre"],
                   descriptor=dico["Descripteurs"],
            url=i.a['href'],
            type=type))
Ejemplo n.º 24
0
def extract_features(connection, task, view, func, func_name, left, right, force):
    assert left < right;
    
    task = connection.tasks[task]
    view = connection.views[view]
    
    function = get_or_create(
        Function,
        keys=dict(
            task=task,
            name=func_name,
            left=left,
            right=right
        )
    )
    
    num_features = Features.select().where(
        Features.function == function
    ).count()
    
    num_labels = Labels.select().where(
        Labels.task == task
    ).count()
    
    if not force:
        if num_features == num_labels:
            return
    
    do_update = num_features == num_labels
    if num_features != num_labels:
        Features.delete().where(
            Features.function == function
        ).execute()
    
    for _, sequence in connection.itersequences():
        print task, view, sequence, function
        
        _extract_features(
            task=task,
            view=view,
            sequence=sequence,
            func_name=func_name,
            func=func,
            left=left,
            right=right,
            do_update=do_update
        )
Ejemplo n.º 25
0
def _get_deputy_analysis(url, deputy, type, reset=False):
    soupsoup = read_or_dl(LACHAMBRE_PREFIX + lame_url(url),
                          '%s %s' % (deputy.full_name, type), reset)
    setattr(deputy, "analysis_%s_url" % type, url)
    setattr(deputy, "analysis_%s_list" % type, [])
    for i in soupsoup('table')[3]('tr', valign="top"):
        logger.debug("add", type, i.tr('td')[1].text.strip())
        dico = table2dic(i.table('td'))
        logger.debug("%s" % dico)
        getattr(deputy, "analysis_%s_list" % type).\
                append(get_or_create(Analysis,
                                     _id="lachambre_id",
                                     lachambre_id=re.search("dossierID=([0-9A-Za-z-]+)", i.a["href"]).groups()[0],
                                     title=dico["Titre"],
                                     descriptor=dico["Descripteurs"],
                                     url=i.a['href'],
                                     type=type))
Ejemplo n.º 26
0
def _get_competences(dico, dico_nl, document):
    # FIXME: meh, DRY
    if dico.get(u"Compétence") and dico_nl.get(u"Bevoegdheid"):
        document.timeline = []
        for (_date, _title), (_, _title_nl) in zip([clean_text(x).split(u" \xa0 ", 1) for x in dico[u"Compétence"]["head"].contents[::2]],
                                                   [clean_text(x).split(u" \xa0 ", 1) for x in dico_nl[u"Bevoegdheid"]["head"].contents[::2]]):
            logger.debug("append time line %s %s %s" % (_date, _title, _title_nl))
            document.timeline.append(DocumentTimeLine.objects.create(title={"fr": _title, "nl": _title_nl}, date=_date))
    elif dico.get(u"Compétence"):
        document.timeline = []
        for (_date, _title) in [clean_text(x).split(u" \xa0 ", 1) for x in dico[u"Compétence"]["head"].contents[::2]]:
            logger.debug("append time line %s %s %s" % (_date, _title, ""))
            document.timeline.append(DocumentTimeLine.objects.create(title={"fr": _title, "nl": ""}, date=_date))
    elif dico_nl.get(u"Bevoegdheid"):
        document.timeline = []
        for (_date, _title_nl) in [clean_text(x).split(u" \xa0 ", 1) for x in dico_nl[u"Bevoegdheid"]["head"].contents[::2]]:
            logger.debug("append time line %s %s %s" % (_date, "", _title_nl))
            document.timeline.append(DocumentTimeLine.objects.create(title={"fr": "", "nl": _title_nl}, date=_date))
    if dico.get("Analyse des interventions"):
        document.analysis = get_or_create(Analysis, _id="lachambre_id", lachambre_id=dico["Analyse des interventions"]["head"].a.text, url=dico["Analyse des interventions"]["head"].a["href"])
Ejemplo n.º 27
0
def _get_competences(dico, dico_nl, document):
    # FIXME: meh, DRY
    if dico.get(u"Compétence") and dico_nl.get(u"Bevoegdheid"):
        document.timeline = []
        for (_date, _title), (_, _title_nl) in zip([clean_text(x).split(u" \xa0 ", 1) for x in dico[u"Compétence"]["head"].contents[::2]],
                                                   [clean_text(x).split(u" \xa0 ", 1) for x in dico_nl[u"Bevoegdheid"]["head"].contents[::2]]):
            logger.debug("append time line %s %s %s" % (_date, _title, _title_nl))
            document.timeline.append(DocumentTimeLine.objects.create(title={"fr": _title, "nl": _title_nl}, date=_date))
    elif dico.get(u"Compétence"):
        document.timeline = []
        for (_date, _title) in [clean_text(x).split(u" \xa0 ", 1) for x in dico[u"Compétence"]["head"].contents[::2]]:
            logger.debug("append time line %s %s %s" % (_date, _title, ""))
            document.timeline.append(DocumentTimeLine.objects.create(title={"fr": _title, "nl": ""}, date=_date))
    elif dico_nl.get(u"Bevoegdheid"):
        document.timeline = []
        for (_date, _title_nl) in [clean_text(x).split(u" \xa0 ", 1) for x in dico_nl[u"Bevoegdheid"]["head"].contents[::2]]:
            logger.debug("append time line %s %s %s" % (_date, "", _title_nl))
            document.timeline.append(DocumentTimeLine.objects.create(title={"fr": "", "nl": _title_nl}, date=_date))
    if dico.get("Analyse des interventions"):
        document.analysis = get_or_create(Analysis, _id="lachambre_id", lachambre_id=dico["Analyse des interventions"]["head"].a.text, url=dico["Analyse des interventions"]["head"].a["href"])
Ejemplo n.º 28
0
def handle_commission(commission):
    soup, suppe = read_or_dl_with_nl(LACHAMBRE_PREFIX + commission.url, "commission %s" % commission.lachambre_id)
    commission.full_name["fr"] = soup.h1.text
    commission.full_name["nl"] = suppe.h1.text
    commission.deputies = []
    seats = {"fr": {}, "nl": {}}
    for i, j in zip(soup('p'), suppe('p')):
        role = i.b.text[:-1]
        role_nl = j.b.text[:-1]
        for dep in i('a'):
            deputy = Deputy.objects.get(lachambre_id=re.search("key=([O0-9]+)", dep["href"]).groups()[0])
            membership = get_or_create(CommissionMembership, deputy=deputy, commission=commission)
            membership.role = role
            membership.save()
            commission.deputies.append(membership.id)
        seats["fr"][role] = map(lambda x: (x[0], len(x[1].split(','))), zip(map(lambda x: x.text[:-1], i('b')[1:]), str(i).split("<br />")[1:]))
        seats["nl"][role_nl] = map(lambda x: (x[0], len(x[1].split(','))), zip(map(lambda x: x.text[:-1], i('b')[1:]), str(i).split("<br />")[1:]))

    commission.seats = seats
    commission.save()
Ejemplo n.º 29
0
def scrape():
    soup, suppe = read_or_dl_with_nl("http://www.lachambre.be/kvvcr/showpage.cfm?section=/comm/commissions&language=fr&cfm=/site/wwwcfm/comm/LstCom.cfm&rightmenu=right_cricra", "commissions list")
    _type = ""
    for i, j in zip(soup("div", id="story")[1], suppe("div", id="story")[1]):
        if not isinstance(i, NavigableString) and (i.h4 or i.a):
            if i.h4:
                _type = i.h4.text
                _type_nl = j.h4.text
            elif i.a:
                commission = get_or_create(Commission, lachambre_id=int(re.search("com=(\d+)", i.a["href"]).groups()[0]))
                commission.type["fr"] = _type
                commission.type["nl"] = _type_nl
                commission.name["fr"] = i.a.text
                commission.name["nl"] = j.a.text
                commission.url = i.a["href"]

                commission.save()

    for com in list(Commission.objects.all()):
        handle_commission(com)
Ejemplo n.º 30
0
 def register_partition(self, task, partition_name, key_func, force=False):
     assert isinstance(task, Task)
     
     partition = get_or_create(
         Partition,
         keys=dict(
             task=task,
             name=partition_name
         )
     )
     
     label_query = Labels.select().where(
         Labels.task == task
     )
     
     group_query = GroupDefinition.select().where(
         GroupDefinition.partition == partition
     )
     
     update = group_query.count() == label_query.count()
     if update and not force:
         return
     
     if update:
         print 'updating'
         for label in label_query:
             GroupDefinition.update(
                 fold=key_func(label)
             ).where(
                 SplitDefinition.partition == partition,
                 SplitDefinition.label == label
             ).execute()
     
     else:
         print 'saving'
         rows = [dict(
             partition=partition,
             label=label,
             key=key_func(label)
         ) for label in label_query]
         GroupDefinition.insert_many(rows).execute()
Ejemplo n.º 31
0
def deputies_list(reset=False):
    soup = read_or_dl("http://www.lachambre.be/kvvcr/showpage.cfm?section=/depute&language=fr&rightmenu=right_depute&cfm=/site/wwwcfm/depute/cvlist.cfm", "deputies", reset)

    for dep in soup('table')[4]('tr'):
        items = dep('td')
        full_name = re.sub('  +', ' ', items[0].a.text).strip()
        url = items[0].a['href']
        party = get_or_create(Party, name=items[1].a.text, url=dict(items[1].a.attrs)['href'])
        email = items[2].a.text
        website = items[3].a['href'] if items[3].a else None
        # yes, one deputies key contains a O instead of an 0, I'm not joking
        lachambre_id = re.search('key=([0-9O]+)', url).groups()[0]
        if not Deputy.objects.filter(lachambre_id=lachambre_id):
            deputy = Deputy.objects.create(full_name=full_name,
                                           party=party,
                                           url=url,
                                           websites=[website] if website else [],
                                           lachambre_id=lachambre_id,
                                           emails=[email])
            logger.debug('adding new deputy %s %s %s %s %s' % (lachambre_id.encode("Utf-8"), full_name.encode("Utf-8"), party, email.encode("Utf-8"), website.encode("Utf-8") if website else ''))
            logger.info("[NEW] deputy: %s" % deputy)
Ejemplo n.º 32
0
def _get_deputy_questions(url, deputy, type, reset=False):
    soupsoup = read_or_dl(LACHAMBRE_PREFIX + lame_url(url), '%s %s' % (deputy.full_name, type), reset)
    setattr(deputy, "questions_%s_url" % type, url)
    setattr(deputy, "questions_%s_list" % type, [])
    for i in soupsoup('table')[3]('tr', valign="top"):
        logger.debug("add", type, i.tr('td')[1].text.strip())
        dico = table2dic(i.table('td'))
        logger.debug("%s" % dico)
        getattr(deputy, "questions_%s_list" % type).\
            append(get_or_create(Question,
            _id="lachambre_id",
            title=dico["Titre"],
                   lachambre_id=re.search("dossierID=([0-9A-Za-z-]+)", i.a["href"]).groups()[0],
            reunion_type=dico.get(u"Réunion"),
            reunion_date=dico.get("Date discussion"),
            session_id=dico.get("Session"),
            pdf_url=dico.get(u"Compte rendu intégral", {"href": None})["href"],
            eurovoc_descriptors=map(lambda x: x.strip(), dico.get("Descripteurs Eurovoc", "").split('|')),
            keywords=map(lambda x: x.strip(), dico.get(u"Mots-clés libres", "").split("|")),
            url=i.a['href'],
            type=type))
Ejemplo n.º 33
0
 def _create_base_product(self):
     """
     Creates base product without relationships that require its existence formerly. Needs objects from
     INDEPENDENT object names to exist first. Product is then flushed to the database but not committed.
     :return: product, instance of a product
     """
     product_class = get_class_by_table_name(f'{self.product_type}_product')
     if product_class:
         # build kwargs for product
         product_kwargs = {}
         for field in self.product_config[BASE_FIELDS]:
             if field in self.product_config[SINGLE_RELATIONS]:
                 product_kwargs[field] = self.objects[field]
             else:
                 product_kwargs[field] = self.data[field]
         product, _ = get_or_create(product_class, **product_kwargs)
         db_session.add(product)
         db_session.flush()
         self.objects['product_id'] = product.id
         return product
     raise ValueError('Unknown product class')
Ejemplo n.º 34
0
    def post(self, name):
        print name
        form = EditAuthorForm()
        author = get_or_404(Author, name=name)
        if form.validate():
            if author.name != form.name.data:
                author.name = form.name.data
                try:
                    session.commit()
                except:
                    session.rollback()
                    msg = 'Author <strong class=badge>{0}</strong> already \
                           exists!'.format(form.name.data)
                    flash(msg)
                    # return render_template('edit_author.haml', form=form, name=name)
                    return self.get(name)
            
            new_books = str_separator(form.title.data)
            old_books = get_books(author, format='list')

            # get what to delete from exists books list and what to add, to bring
            # to edited field books
            rm_lst, append_lst = diff_list(old_books, new_books)
            print(rm_lst, append_lst)

            # remove not related books
            for title in rm_lst:
                b = get_or_404(Book, title=title)
                author.books.remove(b)
                if len(b.authors) == 1:
                    session.delete(b)
            
            # relate new books with exist author
            for title in append_lst:
                b = get_or_create(Book, title=title)
                author.books.append(b)
            session.commit()

            return redirect(url_for('index'))
        return render_template('edit_author.haml', form=form, name=name)
Ejemplo n.º 35
0
    def post(self, title):
        book = get_or_404(Book, title=title)
        form = EditBookForm()
        if form.validate():
            if book.title != form.title.data:
                book.title = form.title.data
                try:
                    session.commit()
                except:
                    session.rollback()
                    msg = 'Book <strong class=badge>{0}</strong> already \
                           exists!'.format(form.title.data)
                    flash(msg)
                    # return render_template('edit_book.haml', form=form, title=title)
                    return self.get(title)

            # Get current and changed list of authors.
            new_authors = str_separator(form.name.data)
            old_authors = get_authors(book, format='list')

            # We get authors' names which have to be deleted from an original list 
            # and also authors' names which have to be added.
            rm_lst, append_lst = diff_list(old_authors, new_authors)

            # remove not related authors
            for name in rm_lst:
                a = get_or_404(Author, name=name)
                book.authors.remove(a)
                if len(a.books) == 1:
                    session.delete(a)
            
            # relate new authors with exist book
            for name in append_lst:
                a = get_or_create(Author, name=name)
                book.authors.append(a)
            session.commit()

            return redirect(url_for('index'))
        return render_template('edit_book.haml', form=form, title=title)
Ejemplo n.º 36
0
def _get_deputy_questions(url, deputy, type, reset=False):
    soupsoup = read_or_dl(LACHAMBRE_PREFIX + lame_url(url),
                          '%s %s' % (deputy.full_name, type), reset)
    setattr(deputy, "questions_%s_url" % type, url)
    setattr(deputy, "questions_%s_list" % type, [])
    for i in soupsoup('table')[3]('tr', valign="top"):
        logger.debug("add", type, i.tr('td')[1].text.strip())
        dico = table2dic(i.table('td'))
        logger.debug("%s" % dico)
        getattr(deputy, "questions_%s_list" % type).\
                append(get_or_create(Question,
                                     _id="lachambre_id",
                                     title=dico["Titre"],
                                     lachambre_id=re.search("dossierID=([0-9A-Za-z-]+)", i.a["href"]).groups()[0],
                                     reunion_type=dico.get(u"Réunion"),
                                     reunion_date=dico.get("Date discussion"),
                                     session_id=dico.get("Session"),
                                     pdf_url=dico.get(u"Compte rendu intégral", {"href": None})["href"],
                                     eurovoc_descriptors=map(lambda x: x.strip(), dico.get("Descripteurs Eurovoc", "").split('|')),
                                     keywords=map(lambda x: x.strip(), dico.get(u"Mots-clés libres", "").split("|")),
                                     url=i.a['href'],
                                     type=type))
Ejemplo n.º 37
0
def login():
    credentials = request.get_json(force=True)
    email = credentials['email']
    app_id = credentials['app_id']
    password = credentials['password']
    user = db.session.query(UserAccount).filter_by(email=email).first()
    if user and user.check_password(password):
        if db.session.query(Application).filter_by(id=app_id).first():
            token = generate_token()
            user_app, created = get_or_create(UserApplication, application_id=app_id, 
                                            user_id=user.id)
            if not created:
                user_app.last_login = datetime.datetime.now()
                user_app.token = token
            else:
                user_app.token = token
                db.session.add(user_app)
            db.session.commit()
            return jsonify({"code": 0, "token": token})
        else:
            return jsonify({"code": 2, "token": None})
    else:
        return jsonify({"code": 2, "token": None})
Ejemplo n.º 38
0
def add_grade():
    """
    Запрос на добавление оценки студенту по дисциплине
    :param student_id: - int - идентификатор студента
    :param class_id: - int - идентификатор дисциплины
    :param score: - int - оценка(от 0 до 5)
    :return:
    Если дисциплинна уже проставленна, оценка заменятеся
    """
    data = request.get_json(force=True)

    student = Student.query.get_or_404(data['student_id'])
    c_class = Class.query.get_or_404(data['class_id'])

    grade = get_or_create(
        db.session,
        Grade,
        student_id=student.id,
        class_id=c_class.id
    )
    grade.score = data['score']
    db.session.commit()

    return jsonify({"status": "ok"})
Ejemplo n.º 39
0
 def __init__(self, connection_path, meta_path):
     self.connection_params = json.load(open(connection_path, 'r'))
     
     db.init(
         self.connection_params['database'],
         **self.connection_params['credentials']
     )
     db.create_tables(all_tables, safe=True)
     
     self.db = db
     self.meta = Metadata(meta_path)
     
     # Load the dataset object
     self.dataset, created = Dataset.get_or_create(
         name=self.meta.name
     )
     
     # Load the tasks
     self.tasks = {}
     for task in self.meta.tasks:
         self.tasks[task['name']] = get_or_create(
             Task,
             keys=dict(
                 dataset=self.dataset,
                 name=task['name']
             ),
             values=dict(
                 columns=task['columns'],
                 num_columns=len(task['columns'])
             )
         )
     
     # Load the views
     self.views = {}
     for view in self.meta.views:
         self.views[view['name']] = get_or_create(
             View,
             keys=dict(
                 dataset=self.dataset,
                 name=view['name']
             ),
             values=dict(
                 columns=view['columns'],
                 num_columns=len(view['columns'])
             )
         )
     
     # Load the sequences
     self.sequences = {}
     for sequence in self.meta.sequences:
         self.sequences[sequence['name']] = get_or_create(
             Sequence,
             keys=dict(
                 dataset=self.dataset,
                 name=sequence['name']
             ),
             values=dict(
                 t_min=sequence['t_min'],
                 t_max=sequence['t_max']
             )
         )
     
     # Load the partitions
     self.splits = {}
     self.partitions = {}
     self.classifiers = {}
     self.feature_unions = {}
     for task_name, task in self.tasks.iteritems():
         pp = {pp.name: pp for pp in task.partitions}
         self.partitions[task_name] = pp
         self.partitions[task] = pp
         
         ss = {pp.name: list(pp.splits.execute()) for pp in task.partitions}
         self.splits[task_name] = ss
         self.splits[task] = ss
         
         cc = {}
         for classifier in task.classifiers:
             cc[classifier.name] = classifier
         self.classifiers[task] = cc
         self.classifiers[task_name] = cc
         
         ff = {}
         for feature_union in task.feature_unions:
             ff[feature_union.name] = feature_union
         self.feature_unions[task] = ff
         self.feature_unions[task_name] = ff
Ejemplo n.º 40
0
 def setTags(self, tags):
     self.tags = [get_or_create(Tag, title=tag) for tag in tags]
Ejemplo n.º 41
0
def refresh_background(course_id):
    """
    Look up existing extensions and apply them to new quizzes.

    :param course_id: The Canvas ID of the Course.
    :type course_id: int
    :rtype: dict
    :returns: A dictionary containing two parts:

        - success `bool` False if there was an error, True otherwise.
        - message `str` A long description of success or failure.
    """
    job = get_current_job()

    update_job(job, 0, 'Starting...', 'started')

    with app.app_context():
        course, created = get_or_create(
            db.session,
            Course,
            canvas_id=course_id
        )

        try:
            course_name = get_course(course_id).get(
                'name',
                '<UNNAMED COURSE>'
            )
            course.course_name = course_name
            db.session.commit()
        except requests.exceptions.HTTPError:
            update_job(
                job,
                0,
                'Course not found.',
                'failed',
                error=True
            )
            logger.exception('Unable to find course #{}'.format(course_id))

            return job.meta

        # quiz stuff
        quizzes = missing_quizzes(course_id)

        num_quizzes = len(quizzes)

        if num_quizzes < 1:
            update_job(
                job,
                100,
                'Complete. No quizzes required updates.',
                'complete',
                error=False
            )

            return job.meta

        percent_user_map = defaultdict(list)

        inactive_list = []

        update_job(job, 0, 'Getting past extensions.', 'processing', False)
        for extension in course.extensions:
            # If extension is inactive, ignore.
            if not extension.active:
                inactive_list.append(extension.user.sortable_name)
                logger.debug('Extension #{} is inactive.'.format(
                    extension.id
                ))
                continue

            user_canvas_id = User.query.filter_by(
                id=extension.user_id
            ).first().canvas_id

            # Check if user is in course. If not, deactivate extension.
            try:
                canvas_user = get_user(course_id, user_canvas_id)

                # Skip user if not a student. Fixes an edge case where a
                # student that previously recieved an extension changes roles.
                enrolls = canvas_user.get('enrollments', [])
                type_list = [e['type'] for e in enrolls if e['enrollment_state'] == 'active']
                if not any(t == 'StudentEnrollment' for t in type_list):
                    logger.info((
                        "User #{} was found in course #{}, but is not an "
                        "active student. Deactivating extension #{}. Roles "
                        "found: {}"
                    ).format(
                        user_canvas_id,
                        course_id,
                        extension.id,
                        ", ".join(type_list) if len(enrolls) > 0 else None
                    ))
                    extension.active = False
                    db.session.commit()
                    inactive_list.append(extension.user.sortable_name)
                    continue

            except requests.exceptions.HTTPError:
                log_str = (
                    'User #{} not in course #{}. Deactivating extension #{}.'
                )
                logger.info(
                    log_str.format(user_canvas_id, course_id, extension.id)
                )
                extension.active = False
                db.session.commit()
                inactive_list.append(extension.user.sortable_name)
                continue

            percent_user_map[extension.percent].append(user_canvas_id)

        if len(percent_user_map) < 1:
            msg_str = 'No active extensions were found.<br>'

            if len(inactive_list) > 0:
                msg_str += ' Extensions for the following students are inactive:<br>{}'
                msg_str = msg_str.format("<br>".join(inactive_list))

            update_job(
                job,
                100,
                msg_str,
                'complete',
                error=False
            )
            return job.meta

        for index, quiz in enumerate(quizzes):
            quiz_id = quiz.get('id', None)
            quiz_title = quiz.get('title', '[UNTITLED QUIZ]')

            comp_perc = int(((float(index)) / float(num_quizzes)) * 100)
            refreshing_str = 'Refreshing quiz #{} - {} [{} of {}]'
            update_job(
                job,
                comp_perc,
                refreshing_str.format(
                    quiz_id,
                    quiz_title,
                    index + 1,
                    num_quizzes
                ),
                'processing',
                error=False
            )

            for percent, user_list in percent_user_map.iteritems():
                extension_response = extend_quiz(
                    course_id,
                    quiz,
                    percent,
                    user_list
                )

                if extension_response.get('success', False) is True:
                    # add/update quiz
                    quiz_obj, created = get_or_create(
                        db.session,
                        Quiz,
                        canvas_id=quiz_id,
                        course_id=course.id
                    )
                    quiz_obj.title = quiz_title

                    db.session.commit()
                else:
                    error_message = 'Some quizzes couldn\'t be updated. '
                    error_message += extension_response.get('message', '')
                    update_job(
                        job,
                        comp_perc,
                        error_message,
                        'failed',
                        error=True,
                    )
                    return job.meta

        msg = '{} quizzes have been updated.'.format(len(quizzes))
        update_job(job, 100, msg, 'complete', error=False)
        return job.meta
Ejemplo n.º 42
0
def get_new_documents():
    for document_page in read_or_dl("http://www.lachambre.be/kvvcr/showpage.cfm?section=/flwb&language=fr&rightmenu=right&cfm=ListDocument.cfm", "all documents")('div', **{'class': re.compile("linklist_[01]")}):
        soup, suppe = read_or_dl_with_nl(LACHAMBRE_PREFIX + document_page.a["href"], "document %s" % document_page.a.text)
        for soup, suppe in zip(soup('table')[4]('tr', valign="top"), suppe('table')[4]('tr', valign="top")):
            get_or_create(Document, _id="lachambre_id", title={"fr": soup('div')[1].text, "nl": suppe('div')[1].text}, lachambre_id=soup.div.text, url=soup.a["href"])
Ejemplo n.º 43
0
def update_background(course_id, extension_dict):
    """
    Update time on selected students' quizzes to a specified percentage.

    :param course_id: The Canvas ID of the Course to update in
    :type course_id: int
    :param extension_dict: A dictionary that includes the percent of
        time and a list of canvas user ids.

        Example:
        {
            'percent': '300',
            'user_ids': [
                '0123456',
                '1234567',
                '9867543',
                '5555555'
            ]
        }
    :type extension_dict: dict
    """
    job = get_current_job()

    update_job(job, 0, 'Starting...', 'started')

    with app.app_context():
        if not extension_dict:
            update_job(
                job,
                0,
                'Invalid Request',
                'failed',
                error=True
            )
            logger.warning('Invalid Request: {}'.format(extension_dict))
            return job.meta

        try:
            course_json = get_course(course_id)
        except requests.exceptions.HTTPError:
            update_job(
                job,
                0,
                'Course not found.',
                'failed',
                error=True
            )
            logger.exception('Unable to find course #{}'.format(course_id))
            return job.meta

        course_name = course_json.get('name', '<UNNAMED COURSE>')

        user_ids = extension_dict.get('user_ids', [])
        percent = extension_dict.get('percent', None)

        if not percent:
            update_job(
                job,
                0,
                '`percent` field required.',
                'failed',
                error=True
            )
            logger.warning('Percent field not provided. Request: {}'.format(
                extension_dict
            ))
            return job.meta

        course, created = get_or_create(db.session, Course, canvas_id=course_id)
        course.course_name = course_name
        db.session.commit()

        for user_id in user_ids:
            try:
                canvas_user = get_user(course_id, user_id)

                sortable_name = canvas_user.get('sortable_name', '<MISSING NAME>')
                sis_id = canvas_user.get('sis_user_id')

            except requests.exceptions.HTTPError:
                # Unable to find user. Log and skip them.
                logger.warning(
                    "Unable to find user #{} in course #{}".format(
                        user_id,
                        course_id
                    )
                )
                continue

            user, created = get_or_create(db.session, User, canvas_id=user_id)

            user.sortable_name = sortable_name
            user.sis_id = sis_id

            db.session.commit()

            # create/update extension
            extension, created = get_or_create(
                db.session,
                Extension,
                course_id=course.id,
                user_id=user.id
            )
            extension.percent = percent

            db.session.commit()

        quizzes = get_quizzes(course_id)
        num_quizzes = len(quizzes)
        quiz_time_list = []
        unchanged_quiz_time_list = []

        if num_quizzes < 1:
            update_job(
                job,
                0,
                'Sorry, there are no quizzes for this course.',
                'failed',
                error=True
            )
            logger.warning(
                "No quizzes found for course {}. Unable to update.".format(
                    course_id
                )
            )
            return job.meta

        for index, quiz in enumerate(quizzes):
            quiz_id = quiz.get('id', None)
            quiz_title = quiz.get('title', "[UNTITLED QUIZ]")

            comp_perc = int(((float(index)) / float(num_quizzes)) * 100)
            updating_str = 'Updating quiz #{} - {} [{} of {}]'
            update_job(
                job,
                comp_perc,
                updating_str.format(quiz_id, quiz_title, index + 1, num_quizzes),
                'processing',
                error=False
            )

            extension_response = extend_quiz(course_id, quiz, percent, user_ids)

            if extension_response.get('success', False) is True:
                # add/update quiz
                quiz_obj, created = get_or_create(
                    db.session,
                    Quiz,
                    canvas_id=quiz_id,
                    course_id=course.id
                )
                quiz_obj.title = quiz_title

                db.session.commit()

                added_time = extension_response.get('added_time', None)
                if added_time is not None:
                    quiz_time_list.append({
                        "title": quiz_title,
                        "added_time": added_time
                    })
                else:
                    unchanged_quiz_time_list.append({"title": quiz_title})
            else:
                update_job(
                    job,
                    comp_perc,
                    extension_response.get(
                        'message',
                        'An unknown error occured.'
                    ),
                    'failed',
                    error=True
                )
                logger.error("Extension failed: {}".format(extension_response))
                return job.meta

        msg_str = (
            'Success! {} {} been updated for {} student(s) to have {}% time. '
            '{} {} no time limit and were left unchanged.'
        )

        message = msg_str.format(
            len(quiz_time_list),
            "quizzes have" if len(quiz_time_list) != 1 else "quiz has",
            len(user_ids),
            percent,
            len(unchanged_quiz_time_list),
            "quizzes have" if len(unchanged_quiz_time_list) != 1 else "quiz has"
        )

        update_job(job, 100, message, 'complete', error=False)
        job.meta['quiz_list'] = quiz_time_list
        job.meta['unchanged_list'] = unchanged_quiz_time_list
        job.save()

        return job.meta
Ejemplo n.º 44
0
 def _createTags(self, tags):
     return [get_or_create(Tag, title=tag) for tag in tags]
Ejemplo n.º 45
0
forks = determine_capacity(v2)
if forks is None:
    raise RuntimeException("Capactiy not determinable")
forks = forks - 1
print("Forks set to {}".format(forks))

cancel_all(v2.jobs)
cancel_all(v2.workflow_jobs)

delete_all(v2.jobs)
delete_all(v2.workflow_jobs)
delete_all(v2.workflow_job_templates)
delete_all(v2.inventory)
delete_all(v2.job_templates)

org = get_or_create(v2.organizations, name='cmurders murder inc.')
inv = get_or_create(org.related.inventories,
                    name='cmurders murdertown',
                    organization=org,
                    variables=inventory_vars)

for i in xrange(0, forks + 5):
    h = get_or_create(inv.related.hosts,
                      name='high-capacity-host-{}'.format(i),
                      inventory=inv)

proj = get_or_create(
    v2.projects,
    name='Murder Tools',
    organization=org,
    scm_url='https://github.com/chrismeyersfsu/ansible-examples.git',
Ejemplo n.º 46
0
def get_new_documents():
    for document_page in read_or_dl("http://www.lachambre.be/kvvcr/showpage.cfm?section=/flwb&language=fr&rightmenu=right&cfm=ListDocument.cfm", "all documents")('div', **{'class': re.compile("linklist_[01]")}):
        soup, suppe = read_or_dl_with_nl(LACHAMBRE_PREFIX + document_page.a["href"], "document %s" % document_page.a.text)
        for soup, suppe in zip(soup('table')[4]('tr', valign="top"), suppe('table')[4]('tr', valign="top")):
            get_or_create(Document, _id="lachambre_id", title={"fr": soup('div')[1].text, "nl": suppe('div')[1].text}, lachambre_id=soup.div.text, url=soup.a["href"])
Ejemplo n.º 47
0
import time
import sys, os
sys.path.append(os.path.join(os.getcwd(), '..', '..'))
from utils import get_or_create, delete_all

host_vars = """ansible_connection: local"""
sleep_interval_vars = """---
sleep_min: 240
sleep_max: 480
"""

delete_all(v2.jobs)

org = get_or_create(v2.organizations, name='cmurders murder inc.')
inv = get_or_create(org.related.inventories, name='cmurders murdertown', organization=org)
h = get_or_create(inv.related.hosts, name='cmurder', variables=host_vars, inventory=inv)

proj = get_or_create(v2.projects, name='Murder Tools', organization=org, scm_url='https://github.com/chrismeyersfsu/ansible-examples.git', wait=True)

jt = get_or_create(v2.job_templates, name="Kill Number 1", organization=org, inventory=inv, project=proj, extra_vars=sleep_interval_vars, playbook='sleep.yml', allow_simultaneous=True)

wfjt = get_or_create(v2.workflow_job_templates, name="Mass Murder", organization=org)

parallel_count = 100
current_node_count = wfjt.related.workflow_nodes.get()['count']
if current_node_count < parallel_count:
    for i in xrange(0, parallel_count):
        wfjt.related.workflow_nodes.post(dict(unified_job_template=jt.id))

wfj = None
#wfj = wfjt.launch().wait_until_completed()
Ejemplo n.º 48
0
def on_event():
    """Handles an event from Hangouts Chat."""

    engine = create_engine(DATABASE_URL)
    if not engine.dialect.has_table(engine, USER_TABLE_NAME):
        Base.metadata.create_all(engine)

    Base.metadata.bind = engine
    DBSession = sessionmaker(bind=engine)
    session = DBSession()

    event = request.get_json()
    email = event['message']['sender']['email']
    text = event['message']['text']

    if event['type'] == 'ADDED_TO_SPACE' and event['space']['type'] == 'ROOM':
        text = 'Thanks for adding me to "{0}"!'.format(
            event['space']['displayName'])

    elif event['type'] == 'MESSAGE':
        message = event['message']['text']
        message = message.replace("@visualtime", "")

        if message.startswith('/login'):
            if len(message.split()) >= 2:
                password = message.split()[1]
                get_or_create(session, User, email=email, password=password)
                text = 'Usuario {0} ha sido iniciado correctamente'.format(
                    email)

        if message.startswith('/info'):
            users = session.query(User).filter(User.email == email)

            if users.count() == 0:
                text = "Oppp!! usa /login <password> para iniciar tu usuario"

            else:
                user = users.first()
                visualtime_client = VisualTimeHelper(user.email, user.password)
                visualtime_client.login()
                visualtime_info = visualtime_client.get_output_time()


                text = 'Hola {0}: \n' \
                       'Llevas trabajando: {1} \n' \
                       'Hoy te vas a casa a las: {2} \n' \
                       'El porcentaje de avance de tu jornada es: {3} % \n' \
                       'La jornada de hoy tiene: {4} horas \n' \
                       'Ahora mismo te encuentras: {5}'.format(user.email,
                                                            visualtime_info['working_time'],
                                                            visualtime_info['output_time'].split()[1],
                                                            visualtime_info['percent'],
                                                            visualtime_info['day'],
                                                            "trabajando" if int(visualtime_info['direction']) == 1 else "descansando")

        elif message.startswith("/push"):

            users = session.query(User).filter(User.email == email)

            if users.count() == 0:
                text = u"Oppp!! usa /login <password> para iniciar tu usuario"

            else:
                user = users.first()
                visualtime_client = VisualTimeHelper(user.email, user.password)
                visualtime_client.login()
                visualtime_info = visualtime_client.push()

                if visualtime_info:
                    text = u"Ok!! Fichaje creado con éxito."
                else:
                    text = u"Upps! Hay un error creado el fichaje."

        elif message.startswith("/logout"):
            users = session.query(User).filter(User.email == email)

            if users.count() == 0:
                text = u"Oppp!! usa /login <password> para iniciar tu usuario"
            else:
                users.delete(synchronize_session=False)
                text = u"Ha cerrado sesión con éxito."

    else:
        return

    return jsonify({'text': text})
Ejemplo n.º 49
0
forks = determine_capacity(v2)
if forks is None:
    raise RuntimeException("Capactiy not determinable")
forks = forks - 1
print("Forks set to {}".format(forks))

cancel_all(v2.jobs)
cancel_all(v2.workflow_jobs)

delete_all(v2.jobs)
delete_all(v2.workflow_jobs)
delete_all(v2.workflow_job_templates)
delete_all(v2.inventory)
delete_all(v2.job_templates)

org = get_or_create(v2.organizations, name='cmurders murder inc.')
inv = get_or_create(org.related.inventories, name='cmurders murdertown', organization=org, variables=inventory_vars)

for i in xrange(0, forks+5):
    h = get_or_create(inv.related.hosts, name='high-capacity-host-{}'.format(i), inventory=inv)

proj = get_or_create(v2.projects, name='Murder Tools', organization=org, scm_url='https://github.com/chrismeyersfsu/ansible-examples.git', wait=True)

jt = get_or_create(v2.job_templates, name="High Capacity JT - {} forks".format(forks), organization=org, inventory=inv, project=proj, extra_vars=sleep_interval_vars, playbook='sleep.yml', allow_simultaneous=True, forks=forks)

wfjt = get_or_create(v2.workflow_job_templates, name="High Capacity Workflow", organization=org)

parallel_count = 100
current_node_count = wfjt.related.workflow_nodes.get()['count']
if current_node_count < parallel_count:
    for i in xrange(0, parallel_count):
Ejemplo n.º 50
0
 def setTags(self, tags):
     self.tags = [get_or_create(Tag, title = tag) for tag in tags]
Ejemplo n.º 51
0
 def _createTags(self, tags):
     return [get_or_create(Tag, title = tag) for tag in tags]