コード例 #1
0
    specialization    text,
    program           text,
    parse_status      text,
    parse_date        date,
    parse_who         text,
    parse_what        text,
    lock_version      text,
    requirement_text  text,
    -- Added Values
    requirement_html  text,
    parse_tree        jsonb default '{}'::jsonb,
    hexdigest         text,
    PRIMARY KEY (institution, requirement_id));
  delete from program_requirements;
  """)
    conn.commit()

    print('done \nStart CSV file')
    fields = (
        """institution, requirement_id, block_type, block_value, title, period_start,
  period_stop, school, degree, college, major1, major2, concentration, minor, liberal_learning,
  specialization, program, parse_status, parse_date, parse_who, parse_what, lock_version,
  requirement_text, requirement_html, parse_tree, hexdigest""")
    csv.field_size_limit(sys.maxsize)
    with open(latest, newline='', errors='replace') as csvfile:
        csv_reader = csv.reader(csvfile)
        for line in csv_reader:
            if csv_reader.line_num == 1:
                cols = [col.lower().replace(' ', '_') for col in line]
                Row = namedtuple('Row', cols)
            else:
コード例 #2
0
ファイル: review_rules.py プロジェクト: cvickery/transfer-app
def do_form_3(request, session):
    if DEBUG:
        print(f'*** do_form_3({session})')
    reviews = json.loads(request.form['reviews'])
    kept_reviews = [e for e in reviews if e['include']]
    email = session['email']
    if len(kept_reviews) == 0:
        result = '<h1>There are no reviews to confirm.</h1>'
    else:
        message_tail = 'review'
        if len(kept_reviews) > 1:
            num_reviews = len(kept_reviews)
            if num_reviews < 13:
                num_reviews = [
                    '', 'two', 'three', 'four', 'five', 'six', 'seven',
                    'eight', 'nine', 'ten', 'eleven', 'twelve'
                ][num_reviews - 1]
            message_tail = '{} reviews'.format(num_reviews)

        # Insert these reviews into the pending_reviews table of the db.
        conn = PgConnection()
        cursor = conn.cursor()
        token = str(uuid.uuid4())
        reviews = json.dumps(kept_reviews)
        q = "insert into pending_reviews (token, email, reviews) values(%s, %s, %s)"
        cursor.execute(q, (token, email, reviews))
        conn.commit()
        conn.close()

        # Description message templates
        review_dict = dict()
        review_dict['ok'] = '{}: OK'
        review_dict['not-ok'] = '{}: {}'
        review_dict['other'] = 'Other: {}'

        # Generate description messages
        style_str = ' style="border:1px solid #666;vertical-align:middle; padding:0.5em;"'
        suffix = 's'
        if len(kept_reviews) == 1:
            suffix = ''
        review_rows = f"""
                    <table style="border-collapse:collapse;">
                      <tr>
                        <th colspan="5"{style_str}>Rule</th>
                        <th{style_str}>Current Status<br><em>(links show review history)</em></th>
                        <th colspan="2"{style_str}>Your Review{suffix}</th>
                      </tr>
                      """
        for review in kept_reviews:
            review_rows += review['rule_str']
        review_rows += '</table>'

        # Send the email
        url = request.url_root + 'confirmation/' + token

        response = send_token(email, url, review_rows)
        if response.status_code != 202:
            result = f'Error sending email: {response.body}'
        else:
            result = f"""
      {header(title='Review Rules: Respond to Email',
              nav_items = [
              {'type': 'link',
               'href': '/',
               'text': 'Main Menu'},
              {'type': 'link',
               'href': '/review_rules',
               'text':'Review More Rules'}])}
      <details>
        <summary>Check your email at {email}</summary>
        <hr>
        <p>
          Click on the 'activate these reviews' button in that email
          to confirm that you actually wish to have your {message_tail} recorded.
        </p>
      </details>
      <h2>
        Thank you for your work!
      </h2>
      """
    return render_template('review_rules.html', result=Markup(result))
コード例 #3
0
ファイル: dgw_driver.py プロジェクト: cvickery/dgw_processor
def dgw_parser(institution,
               block_type,
               block_value,
               period='all',
               do_parse=False):
    """ For each matching Scribe Block, create a DGW_Processor to hold the info about it; the
      constructor parses the block and extracts information objects from it, creating a HTML
      representation of the Scribe Block and lists of dicts of the extracted objects, one for the
      head and one for the body of the block.

      Update/replace the HTML Scribe Block and the lists of object in the requirement_blocks table.

       The period argument can be 'current', 'latest', or 'all', which will be picked out of the
       result set for 'all'
  """
    if DEBUG:
        print(
            f'*** dgw_parser({institution}, {block_type}, {block_value}. {period})',
            file=sys.stderr)
    if do_parse:
        operation = 'Parsed'
    else:
        operation = 'Updated'
    conn = PgConnection()
    fetch_cursor = conn.cursor()
    update_cursor = conn.cursor()
    query = """
    select requirement_id, title, period_start, period_stop, requirement_text
    from requirement_blocks
    where institution = %s
      and block_type = %s
      and block_value = %s
    order by period_stop desc
  """
    fetch_cursor.execute(query, (institution, block_type, block_value))
    # Sanity Check
    assert fetch_cursor.rowcount > 0, f'No Requirements Found\n{fetch_cursor.query}'
    num_rows = fetch_cursor.rowcount
    num_updates = 0
    for row in fetch_cursor.fetchall():
        if period == 'current' and row.period_stop != '99999999':
            return f"""<h1 class="error">“{row.title}” is not a currently offered {block_type}
                 at {institution}.</h1>
              """
        # Filter out everything after END.
        # For parsing, also filter out "hide" things, but leave them in for display purposes.
        text_to_parse = dgw_filter(row.requirement_text)
        text_to_show = dgw_filter(row.requirement_text, remove_hide=False)
        processor = DGW_Processor(institution, row.requirement_id, block_type,
                                  block_value, row.title, row.period_start,
                                  row.period_stop, text_to_show)

        # Default behavior is just to show the scribe block(s), and not to try parsing them in real
        # time. (But during development, that can be useful for catching coding errors.)
        if do_parse:
            if DEBUG:
                print('Parsing ...', file=sys.stderr)
            dgw_logger = DGW_Logger(institution, block_type, block_value,
                                    row.period_stop)

            input_stream = InputStream(text_to_parse)
            lexer = ReqBlockLexer(input_stream)
            lexer.removeErrorListeners()
            lexer.addErrorListener(dgw_logger)
            token_stream = CommonTokenStream(lexer)
            parser = ReqBlockParser(token_stream)
            parser.removeErrorListeners()
            parser.addErrorListener(dgw_logger)
            tree = parser.req_block()

            try:
                if DEBUG:
                    print('Walking ...', file=sys.stderr)
                walker = ParseTreeWalker()
                walker.walk(processor, tree)
            except Exception as e:
                exc_type, exc_value, exc_traceback = sys.exc_info()
                print(f'{exc_type.__name__}: {exc_value}', file=sys.stderr)
                traceback.print_tb(exc_traceback, limit=30, file=sys.stderr)
                # msg_body = f"""College: {processor.institution}
                #                Block Type: {processor.block_type}
                #                Block Value: {processor.block_value}
                #                Catalog: {processor.catalog_years.catalog_type}
                #                Catalog Years: {processor.catalog_years.text}
                #                Error: {e}"""
        requirement_html = re.sub(r'\n\s*', r'\n',
                                  processor.html().replace("'", '’'))
        head_objects = json.dumps(processor.sections[1])
        body_objects = json.dumps(processor.sections[2])
        # Add the info to the db
        update_query = f""" update requirement_blocks
                          set requirement_html = '{requirement_html}',
                              head_objects = '{head_objects}',
                              body_objects = '{body_objects}'
                        where institution = '{institution}'
                          and requirement_id = '{row.requirement_id}'
                    """
        update_cursor.execute(update_query)
        num_updates += update_cursor.rowcount
        if DEBUG:
            print(f'\r{operation} {institution} {row.requirement_id}', end='')

        if period == 'current' or period == 'latest':
            break
    conn.commit()
    conn.close()
    if DEBUG:
        print()
    return (num_updates, num_rows)
コード例 #4
0
def process_pending(row):
    """ Look up the token and generate events. Return as status message.
  """
    token = row.token
    reviews = json.loads(row.reviews)
    email = row.email
    when_entered = row.when_entered
    summaries = ''

    conn = PgConnection()
    cursor = conn.cursor()

    institutions = set()
    for review in reviews:
        key = RuleKey._make(review['rule_key'].split(':'))
        institutions.add(key.source_institution)
        institutions.add(key.destination_institution)
        cursor.execute(
            """
      select id, review_status
        from transfer_rules
       where source_institution = %s
         and destination_institution = %s
         and subject_area = %s
         and group_number = %s
      """, key)
        rule_id, old_status = cursor.fetchone()
        # Generate an event for this review
        q = """
    insert into events (rule_id, event_type,
                        who, what, event_time)
                       values (%s, %s, %s, %s, %s)"""
        cursor.execute(q, (rule_id, review['event_type'], email,
                           review['comment_text'], when_entered))

        # Update the review state for this rule.
        new_status = old_status | abbr_to_bitmask[review['event_type']]
        q = 'update transfer_rules set review_status = %s where id = %s'
        cursor.execute(q, (new_status, rule_id))

        # Generate a summary of this review
        old_status_str = status_string(old_status)
        new_status_str = status_string(new_status)
        # Convert to event-history link for the rule
        new_status_str = f"""
    <a href="/history/{review['rule_key']}"
       target="_blank"
       rel="noopener noreferrer">{new_status_str}</a>"""
        summaries += f"""
    <tr>
      {review['rule_str']}
    </tr>
    """

    # Remove record from pending_reviews
    cursor.execute('delete from pending_reviews where token = %s', (token, ))
    conn.commit()
    conn.close()

    suffix = 's'
    have_has = 'were'
    num_reviews = len(reviews)
    if num_reviews == 1:
        suffix = ''
        have_has = 'was'
    if num_reviews < 13:
        num_reviews = [
            '', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight',
            'nine', 'ten', 'eleven', 'twelve'
        ][num_reviews - 1]

    # Return summary as an html table, and the set of institutions affected.
    return f"""
  <p class="instructions">
    The following {num_reviews} transfer rule review{suffix} {have_has} submitted by
    <em>{email}</em> on {when_entered.strftime('%B %d, %Y at %I:%M %p')}.
  </p>
  <table>
    <tr>
      <th colspan="5">Rule</th>
      <th>Previous Review Status<br/><em>(links show review history)</em></th>
      <th colspan="2">New Review Status</th>
    </tr>
    {summaries}
  </table>
    """, institutions
コード例 #5
0
def dgw_interpreter(institution: str,
                    block_type: str,
                    block_value: str,
                    period='all',
                    update_db=True,
                    verbose=False) -> tuple:
    """ For each matching Scribe Block, parse the block and generate lists of JSON objects from it.

       The period argument can be 'all', 'current', or 'latest', with the latter two being picked
       out of the result set for 'all'
  """
    if DEBUG:
        print(
            f'*** dgw_interpreter({institution}, {block_type}, {block_value}, {period})'
        )

    conn = PgConnection()
    fetch_cursor = conn.cursor()
    update_cursor = conn.cursor()
    query = """
    select institution, requirement_id, title, period_start, period_stop, requirement_text
    from requirement_blocks
    where institution = %s
      and block_type = %s
      and block_value = %s
    order by period_stop desc
  """
    fetch_cursor.execute(query, (institution, block_type, block_value))
    # Sanity Check
    if fetch_cursor.rowcount < 1:
        print(f'No Requirements Found\n{fetch_cursor.query}', file=sys.stderr)
        return (None, None)

    num_rows = fetch_cursor.rowcount
    num_updates = 0
    for row in fetch_cursor.fetchall():
        if verbose:
            print(
                f'{institution} {row.requirement_id} {block_type} {block_value} ',
                end='',
                file=sys.stderr)
            if period == 'current' and row.period_stop != '99999999':
                print(f'Not currently offered.', end='', file=sys.stderr)
            else:
                print(catalog_years(row.period_start, row.period_stop).text,
                      end='',
                      file=sys.stderr)
            print(file=sys.stderr)

        # Filter out everything after END, plus hide-related tokens (but not hidden content).
        text_to_parse = dgw_filter(row.requirement_text)

        # Generate the parse tree from the Antlr4 parser generator.
        input_stream = InputStream(text_to_parse)
        lexer = ReqBlockLexer(input_stream)
        token_stream = CommonTokenStream(lexer)
        parser = ReqBlockParser(token_stream)
        parse_tree = parser.req_block()

        # Walk the head and body parts of the parse tree, interpreting the parts to be saved.
        header_list = []
        head_ctx = parse_tree.head()
        if head_ctx:
            for child in head_ctx.getChildren():
                obj = dispatch(child, institution, row.requirement_id)
                if obj != {}:
                    header_list.append(obj)

        body_list = []
        body_ctx = parse_tree.body()
        if body_ctx:
            for child in body_ctx.getChildren():
                obj = dispatch(child, institution, row.requirement_id)
                if obj != {}:
                    body_list.append(obj)

        if update_db:
            update_cursor.execute(
                f"""
update requirement_blocks set header_list = %s, body_list = %s
where institution = '{row.institution}'
and requirement_id = '{row.requirement_id}'
""", (json.dumps(header_list), json.dumps(body_list)))
        if period == 'current' or period == 'latest':
            break
    conn.commit()
    conn.close()
    return (header_list, body_list)