specialization text,
program text,
parse_status text,
parse_date date,
parse_who text,
parse_what text,
lock_version text,
requirement_text text,
-- Added Values
requirement_html text,
parse_tree jsonb default '{}'::jsonb,
hexdigest text,
PRIMARY KEY (institution, requirement_id));
delete from program_requirements;
""")
conn.commit()
print('done \nStart CSV file')
fields = (
"""institution, requirement_id, block_type, block_value, title, period_start,
period_stop, school, degree, college, major1, major2, concentration, minor, liberal_learning,
specialization, program, parse_status, parse_date, parse_who, parse_what, lock_version,
requirement_text, requirement_html, parse_tree, hexdigest""")
csv.field_size_limit(sys.maxsize)
with open(latest, newline='', errors='replace') as csvfile:
csv_reader = csv.reader(csvfile)
for line in csv_reader:
if csv_reader.line_num == 1:
cols = [col.lower().replace(' ', '_') for col in line]
Row = namedtuple('Row', cols)
else:
def do_form_3(request, session):
if DEBUG:
print(f'*** do_form_3({session})')
reviews = json.loads(request.form['reviews'])
kept_reviews = [e for e in reviews if e['include']]
email = session['email']
if len(kept_reviews) == 0:
result = '<h1>There are no reviews to confirm.</h1>'
else:
message_tail = 'review'
if len(kept_reviews) > 1:
num_reviews = len(kept_reviews)
if num_reviews < 13:
num_reviews = [
'', 'two', 'three', 'four', 'five', 'six', 'seven',
'eight', 'nine', 'ten', 'eleven', 'twelve'
][num_reviews - 1]
message_tail = '{} reviews'.format(num_reviews)
# Insert these reviews into the pending_reviews table of the db.
conn = PgConnection()
cursor = conn.cursor()
token = str(uuid.uuid4())
reviews = json.dumps(kept_reviews)
q = "insert into pending_reviews (token, email, reviews) values(%s, %s, %s)"
cursor.execute(q, (token, email, reviews))
conn.commit()
conn.close()
# Description message templates
review_dict = dict()
review_dict['ok'] = '{}: OK'
review_dict['not-ok'] = '{}: {}'
review_dict['other'] = 'Other: {}'
# Generate description messages
style_str = ' style="border:1px solid #666;vertical-align:middle; padding:0.5em;"'
suffix = 's'
if len(kept_reviews) == 1:
suffix = ''
review_rows = f"""
<table style="border-collapse:collapse;">
<tr>
<th colspan="5"{style_str}>Rule</th>
<th{style_str}>Current Status<br><em>(links show review history)</em></th>
<th colspan="2"{style_str}>Your Review{suffix}</th>
</tr>
"""
for review in kept_reviews:
review_rows += review['rule_str']
review_rows += '</table>'
# Send the email
url = request.url_root + 'confirmation/' + token
response = send_token(email, url, review_rows)
if response.status_code != 202:
result = f'Error sending email: {response.body}'
else:
result = f"""
{header(title='Review Rules: Respond to Email',
nav_items = [
{'type': 'link',
'href': '/',
'text': 'Main Menu'},
{'type': 'link',
'href': '/review_rules',
'text':'Review More Rules'}])}
<details>
<summary>Check your email at {email}</summary>
<hr>
<p>
Click on the 'activate these reviews' button in that email
to confirm that you actually wish to have your {message_tail} recorded.
</p>
</details>
<h2>
Thank you for your work!
</h2>
"""
return render_template('review_rules.html', result=Markup(result))
def dgw_parser(institution,
block_type,
block_value,
period='all',
do_parse=False):
""" For each matching Scribe Block, create a DGW_Processor to hold the info about it; the
constructor parses the block and extracts information objects from it, creating a HTML
representation of the Scribe Block and lists of dicts of the extracted objects, one for the
head and one for the body of the block.
Update/replace the HTML Scribe Block and the lists of object in the requirement_blocks table.
The period argument can be 'current', 'latest', or 'all', which will be picked out of the
result set for 'all'
"""
if DEBUG:
print(
f'*** dgw_parser({institution}, {block_type}, {block_value}. {period})',
file=sys.stderr)
if do_parse:
operation = 'Parsed'
else:
operation = 'Updated'
conn = PgConnection()
fetch_cursor = conn.cursor()
update_cursor = conn.cursor()
query = """
select requirement_id, title, period_start, period_stop, requirement_text
from requirement_blocks
where institution = %s
and block_type = %s
and block_value = %s
order by period_stop desc
"""
fetch_cursor.execute(query, (institution, block_type, block_value))
# Sanity Check
assert fetch_cursor.rowcount > 0, f'No Requirements Found\n{fetch_cursor.query}'
num_rows = fetch_cursor.rowcount
num_updates = 0
for row in fetch_cursor.fetchall():
if period == 'current' and row.period_stop != '99999999':
return f"""<h1 class="error">“{row.title}” is not a currently offered {block_type}
at {institution}.</h1>
"""
# Filter out everything after END.
# For parsing, also filter out "hide" things, but leave them in for display purposes.
text_to_parse = dgw_filter(row.requirement_text)
text_to_show = dgw_filter(row.requirement_text, remove_hide=False)
processor = DGW_Processor(institution, row.requirement_id, block_type,
block_value, row.title, row.period_start,
row.period_stop, text_to_show)
# Default behavior is just to show the scribe block(s), and not to try parsing them in real
# time. (But during development, that can be useful for catching coding errors.)
if do_parse:
if DEBUG:
print('Parsing ...', file=sys.stderr)
dgw_logger = DGW_Logger(institution, block_type, block_value,
row.period_stop)
input_stream = InputStream(text_to_parse)
lexer = ReqBlockLexer(input_stream)
lexer.removeErrorListeners()
lexer.addErrorListener(dgw_logger)
token_stream = CommonTokenStream(lexer)
parser = ReqBlockParser(token_stream)
parser.removeErrorListeners()
parser.addErrorListener(dgw_logger)
tree = parser.req_block()
try:
if DEBUG:
print('Walking ...', file=sys.stderr)
walker = ParseTreeWalker()
walker.walk(processor, tree)
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
print(f'{exc_type.__name__}: {exc_value}', file=sys.stderr)
traceback.print_tb(exc_traceback, limit=30, file=sys.stderr)
# msg_body = f"""College: {processor.institution}
# Block Type: {processor.block_type}
# Block Value: {processor.block_value}
# Catalog: {processor.catalog_years.catalog_type}
# Catalog Years: {processor.catalog_years.text}
# Error: {e}"""
requirement_html = re.sub(r'\n\s*', r'\n',
processor.html().replace("'", '’'))
head_objects = json.dumps(processor.sections[1])
body_objects = json.dumps(processor.sections[2])
# Add the info to the db
update_query = f""" update requirement_blocks
set requirement_html = '{requirement_html}',
head_objects = '{head_objects}',
body_objects = '{body_objects}'
where institution = '{institution}'
and requirement_id = '{row.requirement_id}'
"""
update_cursor.execute(update_query)
num_updates += update_cursor.rowcount
if DEBUG:
print(f'\r{operation} {institution} {row.requirement_id}', end='')
if period == 'current' or period == 'latest':
break
conn.commit()
conn.close()
if DEBUG:
print()
return (num_updates, num_rows)
def process_pending(row):
""" Look up the token and generate events. Return as status message.
"""
token = row.token
reviews = json.loads(row.reviews)
email = row.email
when_entered = row.when_entered
summaries = ''
conn = PgConnection()
cursor = conn.cursor()
institutions = set()
for review in reviews:
key = RuleKey._make(review['rule_key'].split(':'))
institutions.add(key.source_institution)
institutions.add(key.destination_institution)
cursor.execute(
"""
select id, review_status
from transfer_rules
where source_institution = %s
and destination_institution = %s
and subject_area = %s
and group_number = %s
""", key)
rule_id, old_status = cursor.fetchone()
# Generate an event for this review
q = """
insert into events (rule_id, event_type,
who, what, event_time)
values (%s, %s, %s, %s, %s)"""
cursor.execute(q, (rule_id, review['event_type'], email,
review['comment_text'], when_entered))
# Update the review state for this rule.
new_status = old_status | abbr_to_bitmask[review['event_type']]
q = 'update transfer_rules set review_status = %s where id = %s'
cursor.execute(q, (new_status, rule_id))
# Generate a summary of this review
old_status_str = status_string(old_status)
new_status_str = status_string(new_status)
# Convert to event-history link for the rule
new_status_str = f"""
<a href="/history/{review['rule_key']}"
target="_blank"
rel="noopener noreferrer">{new_status_str}</a>"""
summaries += f"""
<tr>
{review['rule_str']}
</tr>
"""
# Remove record from pending_reviews
cursor.execute('delete from pending_reviews where token = %s', (token, ))
conn.commit()
conn.close()
suffix = 's'
have_has = 'were'
num_reviews = len(reviews)
if num_reviews == 1:
suffix = ''
have_has = 'was'
if num_reviews < 13:
num_reviews = [
'', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight',
'nine', 'ten', 'eleven', 'twelve'
][num_reviews - 1]
# Return summary as an html table, and the set of institutions affected.
return f"""
<p class="instructions">
The following {num_reviews} transfer rule review{suffix} {have_has} submitted by
<em>{email}</em> on {when_entered.strftime('%B %d, %Y at %I:%M %p')}.
</p>
<table>
<tr>
<th colspan="5">Rule</th>
<th>Previous Review Status<br/><em>(links show review history)</em></th>
<th colspan="2">New Review Status</th>
</tr>
{summaries}
</table>
""", institutions
def dgw_interpreter(institution: str,
block_type: str,
block_value: str,
period='all',
update_db=True,
verbose=False) -> tuple:
""" For each matching Scribe Block, parse the block and generate lists of JSON objects from it.
The period argument can be 'all', 'current', or 'latest', with the latter two being picked
out of the result set for 'all'
"""
if DEBUG:
print(
f'*** dgw_interpreter({institution}, {block_type}, {block_value}, {period})'
)
conn = PgConnection()
fetch_cursor = conn.cursor()
update_cursor = conn.cursor()
query = """
select institution, requirement_id, title, period_start, period_stop, requirement_text
from requirement_blocks
where institution = %s
and block_type = %s
and block_value = %s
order by period_stop desc
"""
fetch_cursor.execute(query, (institution, block_type, block_value))
# Sanity Check
if fetch_cursor.rowcount < 1:
print(f'No Requirements Found\n{fetch_cursor.query}', file=sys.stderr)
return (None, None)
num_rows = fetch_cursor.rowcount
num_updates = 0
for row in fetch_cursor.fetchall():
if verbose:
print(
f'{institution} {row.requirement_id} {block_type} {block_value} ',
end='',
file=sys.stderr)
if period == 'current' and row.period_stop != '99999999':
print(f'Not currently offered.', end='', file=sys.stderr)
else:
print(catalog_years(row.period_start, row.period_stop).text,
end='',
file=sys.stderr)
print(file=sys.stderr)
# Filter out everything after END, plus hide-related tokens (but not hidden content).
text_to_parse = dgw_filter(row.requirement_text)
# Generate the parse tree from the Antlr4 parser generator.
input_stream = InputStream(text_to_parse)
lexer = ReqBlockLexer(input_stream)
token_stream = CommonTokenStream(lexer)
parser = ReqBlockParser(token_stream)
parse_tree = parser.req_block()
# Walk the head and body parts of the parse tree, interpreting the parts to be saved.
header_list = []
head_ctx = parse_tree.head()
if head_ctx:
for child in head_ctx.getChildren():
obj = dispatch(child, institution, row.requirement_id)
if obj != {}:
header_list.append(obj)
body_list = []
body_ctx = parse_tree.body()
if body_ctx:
for child in body_ctx.getChildren():
obj = dispatch(child, institution, row.requirement_id)
if obj != {}:
body_list.append(obj)
if update_db:
update_cursor.execute(
f"""
update requirement_blocks set header_list = %s, body_list = %s
where institution = '{row.institution}'
and requirement_id = '{row.requirement_id}'
""", (json.dumps(header_list), json.dumps(body_list)))
if period == 'current' or period == 'latest':
break
conn.commit()
conn.close()
return (header_list, body_list)