def assign_servers_test_output(df_train, df_test, percentile, confidence,
apps_server):
df_train['hour'] = None
df_train['hour'] = pd.DatetimeIndex(df_train['Date']).hour
data_l = list(df_train['pairs'])
pairs_count = (df_train.groupby('pairs2').agg({
'Date': 'count',
'norm_latency': 'mean',
'Duration': 'sum',
'Packets': 'sum'
}).reset_index())
pairs_count.columns = [
'pairs', 'frequency', 'avg_norm_latency', 'total_duration',
'total_packets'
]
pairs_count['norm_latency'] = (
pairs_count['total_duration'] / pairs_count['total_packets'].sum()
) * 100 #sum of all duration time divided by sum of all packets transfered for that pair
per_n = (pairs_count['frequency'].quantile(percentile))
patterns = pyfpgrowth.find_frequent_patterns(data_l, per_n)
rules = pyfpgrowth.generate_association_rules(patterns, confidence)
#format the rules, bring back in the other info on latency rank
formated_rules = format_rules(rules, df_train, apps_server)
#now we make the server assignments based on the training rules applied to the test data
server_df, server_assignments, total_latency, total_latency_model, avg_latency, avg_latency_model = server_association(
formated_rules, df_test, apps_server) #this function loaded fr
#return(formated_rules)
return (server_df, server_assignments, total_latency, total_latency_model,
avg_latency, avg_latency_model)
def do_form_2(request, session):
"""
Process CUNY Subject list from form 2.
Generate form_3: the selected transfer rules for review
"""
if DEBUG:
print(f'*** do_form_2({session})')
for k, v in session.items():
print(f'{k}: {v}')
conn = PgConnection()
cursor = conn.cursor()
# Look up transfer rules where the sending course belongs to a sending institution and is one of
# the source disciplines and the receiving course belongs to a receiving institution and is one of
# the receiving disciplines.
try:
source_institution_params = ', '.join(
'%s' for i in session['source_institutions'])
destination_institution_params = ', '.join(
'%s' for i in session['destination_institutions'])
except KeyError:
# the session is expired or invalid. Go back to Step 1.
return render_template('review_rules.html',
result=Markup("""
<h1>Session Expired</h1>
<p>
<a href="/" class="button">
Main Menu</a>
<a href="/review_rules"
class="restart button">Restart
</a>
</p>
"""))
# Be sure there is the possibility there will be some rules
source_subject_list = request.form.getlist('source_subject')
destination_subject_list = request.form.getlist('destination_subject')
if len(source_subject_list) < 1:
return render_template(
'review_rules.html',
result=Markup(
'<h1 class="error">No sending disciplines selected.</h1>'))
if len(destination_subject_list) < 1:
return render_template(
'review_rules.html',
result=Markup(
'<h1 class="error">No receiving disciplines selected.</h1>'))
# Prepare the query to get the set of rules that match the institutions and cuny_subjects
# selected.
if request.form.get('all-source-subjects'):
source_subjects_clause = ''
else:
source_subjects_str = '|'.join(f':{s}:' for s in source_subject_list)
source_subjects_clause = f" and '{source_subjects_str}' ~ source_subjects"
source_subjects = ', '.join(f"'{s}'" for s in source_subject_list)
source_subjects_clause = f"""
and id in (select rule_id from subject_rule_map where subject in ({source_subjects}))"""
# Get all the rules where,
# - The source and destination institutions have been selected
# and
# - The source_subjects have been selected
q = f"""
select *
from transfer_rules
where source_institution in ({source_institution_params})
and destination_institution in ({destination_institution_params})
{source_subjects_clause}
order by source_institution, destination_institution, subject_area, group_number"""
cursor.execute(
q,
(session['source_institutions'] + session['destination_institutions']))
if cursor.rowcount < 1:
return render_template(
'review_rules.html',
result=Markup(
'<h1 class="error">There are no matching rules.</h1>'))
all_rules = cursor.fetchall()
selected_rules = []
# Get the source and destination course lists from the above set of rules where the destination
# subject was selected. It's possible to have selected rules that don’t transfer to any of the
# selected destination subjects, so those rules are dropped while building the selected-rules
# list.
if request.form.get('all-destination-subjects'):
destination_subjects_clause = ''
else:
# Create a clause that makes sure the destination course has one of the destination subjects
destination_subject_list = request.form.getlist('destination_subject')
destination_subject_params = ', '.join(
f"'{s}'" for s in destination_subject_list)
destination_subjects_clause = f" and dc.cuny_subject in ({destination_subject_params})"
for rule in all_rules:
# It’s possible some of the selected rules don’t have destination courses in any of the selected
# disciplines, so that has to be checked first.
cursor.execute(
f"""
select dc.course_id,
dc.offer_nbr,
dc.offer_count,
dc.discipline,
dc.catalog_number,
dn.discipline_name,
dc.cuny_subject,
dc.cat_num,
dc.transfer_credits,
dc.credit_source,
dc.is_mesg,
dc.is_bkcr
from destination_courses dc, cuny_disciplines dn
where dc.rule_id = %s
and dn.institution = %s
and dn.discipline = dc.discipline
{destination_subjects_clause}
order by discipline, cat_num
""", (rule.id, rule.destination_institution))
if cursor.rowcount > 0:
destination_courses = [
Destination_Course._make(c) for c in cursor.fetchall()
]
cursor.execute(
"""
select sc.course_id,
sc.offer_nbr,
sc.offer_count,
sc.discipline,
sc.catalog_number,
dn.discipline_name,
sc.cuny_subject,
sc.cat_num,
sc.min_credits,
sc.max_credits,
sc.min_gpa,
sc.max_gpa
from source_courses sc, cuny_disciplines dn
where sc.rule_id = %s
and dn.institution = %s
and dn.discipline = sc.discipline
order by discipline, cat_num
""", (rule.id, rule.source_institution))
if cursor.rowcount > 0:
source_courses = [
Source_Course._make(c) for c in cursor.fetchall()
]
# Create the Transfer_Rule tuple suitable for passing to format_rules, and add it to the
# list of rules to pass.
selected_rules.append(
Transfer_Rule._make([
rule.id, rule.source_institution,
rule.destination_institution, rule.subject_area,
rule.group_number, rule.source_disciplines,
rule.source_subjects, rule.review_status, source_courses,
destination_courses
]))
cursor.close()
conn.close()
if len(selected_rules) == 0:
num_rules = 'No matching transfer rules found.'
if len(selected_rules) == 1:
num_rules = 'There is one matching transfer rule.'
if len(selected_rules) > 1:
num_rules = f'There are {len(selected_rules):,} matching transfer rules.'
rules_table = format_rules(selected_rules, scrollable=True)
result = f"""
{header(title='Review Rules: Review Selected Rules',
nav_items=[{'type': 'link',
'href': '/',
'text': 'Main Menu'},
{'type': 'link',
'href': '/review_rules',
'text': 'Change Colleges'},
{'type': 'button',
'class': 'back-button',
'text': 'Change Subjects'
}])}
<details open>
<summary>Instructions (click to open/close)</summary>
<hr>
{num_rules}
<p>
Blanket Credit courses are <span class="blanket">highlighted like this</span>.
</p>
<p>
Rules that are <span class="credit-mismatch">highlighted like this</span> have a different
number of credits taken from the number of credits transferred.
Hover over the “=>” to see the numbers of credits.
</p>
<p>
Credits in parentheses give the number of credits transferred where that does not match the
nominal number of credits for a course.
</p>
<p>
Rules that are <span class="evaluated">highlighted like this</span> are ones that you have
reviewed but not yet submitted.
</p>
<p class="call-to-action">
Click on a rule to review it
</p>
</details>
<fieldset id="verification-fieldset"><legend>Review Reviews</legend>
<p id="num-pending">You have not reviewed any transfer rules yet.</p>
<button type="text" id="send-email" disabled="disabled">
Review Your Reviews
</button>
<form method="post" action="#" id="review-form">
Waiting for rules to finish loading ...
</form>
</fieldset>
<div id="rules-table-div" class="selection-table-div table-height">
{rules_table}
</div>
"""
return render_template('review_rules.html', result=Markup(result))
pairs_count.columns = [
'pairs', 'frequency', 'avg_norm_latency', 'total_duration', 'total_packets'
]
pairs_count['norm_latency'] = (
pairs_count['total_duration'] / pairs_count['total_packets'].sum()
) * 100 #sum of all duration time divided by sum of all packets transfered for that pair
#we only want a list of all the individual pairs at each timestamp. Think of this where each timestamp is a 'transaction' and we chose to buy which 2 items (IP addresses)
data_l = list(df['pairs'])
per_40 = np.percentile(pairs_count['frequency'], [40])[0]
per_80 = np.percentile(pairs_count['frequency'], [80])[0]
#patterns40 = pyfpgrowth.find_frequent_patterns(data_l, per_40)
patterns80 = pyfpgrowth.find_frequent_patterns(
data_l,
per_80) #generates less patterns b/c the min threshold is set higher
confidence = 0.7 #this means the rule is likely to be true 70% of the time, it is a high threshold, used for testing
#rules40 = pyfpgrowth.generate_association_rules(patterns40, confidence)
rules80 = pyfpgrowth.generate_association_rules(patterns80, confidence)
#the input 'rules' is the result of the pyfgrowth function run above. You must have specified the thresholds you want to use
#the orig_df is the original data, with
#the apps_server is the number of apps that can fit on a server
#rules_df_40per_70con_all=format_rules(rules40, df, 20)
rules_df_80per_70con = format_rules(rules80, df, 20)
#test that the rule lenght is as expected
print(len(rules_df_80per_70con))
#head=rules_df_80per_70con.head()
print(rules_df_80per_70con)
def assign_servers_hourly(df, percentile, confidence, apps_server):
df['hour'] = None
df['hour'] = pd.DatetimeIndex(df['Date']).hour
data_groups = []
for i in range(0, df['hour'].nunique()):
data = df[df['hour'] == i]
data_groups.append(data)
pairs_list = []
patterns_list = []
rules_list = []
for i in data_groups:
data_l = list(i['pairs'])
pairs_count = (i.groupby('pairs2').agg({
'Date': 'count',
'norm_latency': 'mean',
'Duration': 'sum',
'Packets': 'sum'
}).reset_index())
pairs_count.columns = [
'pairs', 'frequency', 'avg_norm_latency', 'total_duration',
'total_packets'
]
pairs_count['norm_latency'] = (
pairs_count['total_duration'] / pairs_count['total_packets'].sum()
) * 100 #sum of all duration time divided by sum of all packets transfered for that pair
pairs_list.append(pairs_count)
per_n = (pairs_count['frequency'].quantile(percentile))
patterns = pyfpgrowth.find_frequent_patterns(data_l, per_n)
patterns_list.append(patterns)
rules = pyfpgrowth.generate_association_rules(patterns, confidence)
rules_list.append(rules)
#format the rules, bring back in the other info on latency rank
while {} in rules_list:
rules_list.remove(
{}) #lremove empty items if they have less than 24 hours
formated_rules = []
for i in rules_list:
formatrule = format_rules(i, df, apps_server)
formated_rules.append(formatrule)
#assign IPs to the servers at each hour
server_assign_list = []
total_latency_list = []
total_latency_model_list = []
avg_latency_list = []
avg_latency_model_list = []
#now we use the training model on the unseen test data
data_groups_test = []
for i in range(0, df['hour'].nunique()):
data = df_test[df['hour'] == i]
data_groups_test.append(data)
for i, j in zip(formated_rules, data_groups_test):
server_df, server_assignments, total_latency, total_latency_model, avg_latency, avg_latency_model = server_association(
i, j, apps_server) #this function loaded fr
server_assign_list.append(server_assignments)
total_latency_list.append(total_latency)
total_latency_model_list.append(total_latency_model)
avg_latency_list.append(avg_latency)
avg_latency_model_list.append(avg_latency_model)
#bring together all the durations for the actual data and the model
hours = range(0, df['hour'].nunique())
model_output = pd.DataFrame({
'hours':
hours,
'total_latency_list':
total_latency_list,
'total_latency_model_list':
total_latency_model_list,
'avg_latency_list':
avg_latency_list,
'avg_latency_model_list':
avg_latency_model_list
})
model_output.columns = [
'hours', 'total_latency', 'total_latency_model', 'avg_latency',
'avg_latency_model'
]
model_output['avg_latency_per_reduction'] = (
(model_output['avg_latency'] - model_output['avg_latency_model']) /
model_output['avg_latency']) * 100
#return(formated_rules)
return (server_assign_list, model_output,
model_output['total_latency'].sum(),
model_output['total_latency_model'].sum(),
model_output['avg_latency'].mean(),
model_output['avg_latency_model'].mean())
pairs_count.columns = [
'pairs', 'frequency', 'avg_norm_latency', 'total_duration', 'total_packets'
]
pairs_count['norm_latency'] = (
pairs_count['total_duration'] / pairs_count['total_packets'].sum()
) * 100 #sum of all duration time divided by sum of all packets transfered for that pair
#we only want a list of all the individual pairs at each timestamp. Think of this where each timestamp is a 'transaction' and we chose to buy which 2 items (IP addresses)
data_l = list(df['pairs'])
per_40 = np.percentile(pairs_count['frequency'], [40])[0]
per_80 = np.percentile(pairs_count['frequency'], [80])[0]
patterns40 = pyfpgrowth.find_frequent_patterns(data_l, per_40)
patterns80 = pyfpgrowth.find_frequent_patterns(
data_l,
per_80) #generates less patterns b/c the min threshold is set higher
confidence = 0.7 #this means the rule is likely to be true 70% of the time, it is a high threshold, used for testing
rules40 = pyfpgrowth.generate_association_rules(patterns40, confidence)
rules80 = pyfpgrowth.generate_association_rules(patterns80, confidence)
#the input 'rules' is the result of the pyfgrowth function run above. You must have specified the thresholds you want to use
#the orig_df is the original d
rules_df_80per_70con = format_rules(
rules80, df,
20) #this function was loaded above from the format_rules.py file
server_assignments80, total_latency, total_latency80, avg_latency, avg_latency80 = server_association(
rules_df_80per_70con, df,
20) #this function loaded from server_assocation.py file