def json_out(request, utm_campaign):
utm_campaign = MySQLdb._mysql.escape_string(str(utm_campaign))
dl = DL.DataLoader(db='db1025')
lptl = DL.LandingPageTableLoader(db='db1008')
start_time = lptl.get_earliest_campaign_view(utm_campaign)
end_time = lptl.get_latest_campaign_view(utm_campaign)
"""
Get the views from the given campaign for each banner
=====================================================
"""
logging.info('Determining views for campaign %s' % utm_campaign)
sql = "select utm_source, count(*) as views from landing_page_requests where utm_campaign = '%s' and request_time >= %s and request_time <= %s group by 1" % (
utm_campaign, start_time, end_time)
results = dl.execute_SQL(str(sql))
""" builf the condition string for banners to be used in SQL to retrieve impressions"""
views = dict()
banner_str = ''
for row in results:
views[str(row[0])] = int(row[1])
banner_str_piece = "utm_source = '%s' or " % row[0]
banner_str = banner_str + banner_str_piece
banner_str = banner_str[:-4]
"""
Get the impressions from the given campaign for each banner
===========================================================
"""
logging.info('Determining impressions for campaign %s' % utm_campaign)
sql = "select utm_source, sum(counts) from banner_impressions where (%s) and on_minute >= '%s' and on_minute <= '%s' group by 1" % (
banner_str, start_time, end_time)
results = dl.execute_SQL(str(sql))
""" Build JSON, compute click rates """
click_rate = dict()
json = 'insertStatistics({ '
err_str = ''
for row in results:
try:
utm_source = row[0]
click_rate = float(views[utm_source]) / float(int(row[1]))
item = '"%s" : %s , ' % (utm_source, click_rate)
json = json + item
except:
err_str = err_str + utm_source + ' '
json = json[:-2] + '});'
return render_to_response('live_results/json_out.html', {'html': json},
context_instance=RequestContext(request))
def test_summaries(request):
""" Initialize TestTableLoader """
ttl = DL.TestTableLoader()
test_rows = ttl.get_all_test_rows()
""" Process test info / write html """
html = ''
for row in test_rows:
test_name = ttl.get_test_field(row, 'test_name')
test_type = ttl.get_test_field(row, 'test_type')
utm_campaign = ttl.get_test_field(row, 'utm_campaign')
start_time = ttl.get_test_field(row, 'start_time')
end_time = ttl.get_test_field(row, 'end_time')
try:
test_type = FDH._TESTTYPE_VERBOSE_[test_type]
except:
test_type = 'unknown'
pass
try:
summary_table = ttl.get_test_field(
row, 'html_report').split('<!-- SUMMARY TABLE MARKER -->')[1]
html = html + '<h1><u>' + test_name + ' -- ' + utm_campaign + '</u></h1><div class="spacer"></div>' \
+ '<font size="4"><u>Test Type:</u> ' + test_type + '</font><div class="spacer_small"></div>' \
+ '<font size="4"><u>Running from</u> ' + start_time + ' <u>to</u> ' + end_time + '</font><div class="spacer"></div>' \
+ summary_table + '<div class="spacer"></div><div class="spacer"></div>'
except:
pass
return render_to_response('tests/test_summaries.html', {'template': html},
context_instance=RequestContext(request))
def add_comment(request, utm_campaign):
try:
comments = MySQLdb._mysql.escape_string(request.POST['comments'])
except:
return HttpResponseRedirect(reverse('tests.views.index'))
""" Retrieve the report """
ttl = DL.TestTableLoader()
row = ttl.get_test_row(utm_campaign)
html_string = ttl.get_test_field(row, 'html_report')
""" Insert comment into the page html """
new_html = ''
lines = html_string.split('\n')
now = datetime.datetime.utcnow().__str__()
for line in lines:
if line == '<!-- Cend -->':
line = '<div class="spacer"></div>\n<div class="spacer"></div>\n' + comments + '\n<div class="spacer"></div> --' + now + '\n<!-- Cend -->'
new_html = new_html + line + '\n'
html_string = new_html
html_string = html_string.replace('"', '\\"')
# parse the html for <!-- Cbegin --> <!-- Cend -->
# add the comment above this
""" Update the report """
ttl.update_test_row(test_name=ttl.get_test_field(row, 'test_name'), test_type=ttl.get_test_field(row, 'test_type'), utm_campaign=ttl.get_test_field(row, 'utm_campaign'), start_time=ttl.get_test_field(row, 'start_time'), \
end_time=ttl.get_test_field(row, 'end_time'), winner=ttl.get_test_field(row, 'winner'), is_conclusive=ttl.get_test_field(row, 'is_conclusive'), html_report=html_string)
return HttpResponse(new_html)
def mining_patterns_view(request):
mptl = DL.MiningPatternsTableLoader()
banner_patterns, lp_patterns = mptl.get_pattern_lists()
return render_to_response('LML/mining_patterns.html', {'banner_patterns' : banner_patterns, 'lp_patterns' : lp_patterns}, context_instance=RequestContext(request))
def index(request):
err_msg, earliest_utc_ts_var, latest_utc_ts_var = process_filter_data(request)
sltl = DL.SquidLogTableLoader()
""" Show the squid log table """
squid_table = sltl.get_all_rows_unique_start_time()
filtered_squid_table = list()
for row in squid_table:
log_start_time = sltl.get_squid_log_record_field(row, 'start_time')
""" Ensure the timestamp is properly formatted """
if TP.is_timestamp(log_start_time, 2):
log_start_time = TP.timestamp_convert_format(log_start_time, 2, 1)
if int(log_start_time) > int(earliest_utc_ts_var) and int(log_start_time) < int(latest_utc_ts_var):
filtered_squid_table.append(row)
squid_table = filtered_squid_table
squid_table.reverse()
column_names = sltl.get_column_names()
new_column_names = list()
for name in column_names:
new_column_names.append(sltl.get_verbose_column(name))
squid_table = DR.DataReporting()._write_html_table(squid_table, new_column_names)
""" Show the latest log that has been or is loading and its progress """
completion_rate = sltl.get_completion_rate_of_latest_log()
return render_to_response('LML/index.html', {'err_msg' : err_msg, 'squid_table' : squid_table, 'completion_rate' : completion_rate}, context_instance=RequestContext(request))
def show_report(request, utm_campaign):
ttl = DL.TestTableLoader()
row = ttl.get_test_row(utm_campaign)
try:
html = row[7]
except:
html = '<br><br><center><p><b>Was unable to retrieve report</b></p><br><br><a href="/">Home</a></center>'
return HttpResponse(html)
def index(request):
""" Process POST """
start_time, end_time, min_donation, view_order = process_post_vars(request)
logging.info('Finding live landing pages from %s to %s.' %
(start_time, end_time))
live_lps, columns = DL.CampaignReportingLoader(
query_type='').query_live_landing_pages(start_time,
end_time,
min_donation=min_donation,
view_order=view_order)
if len(live_lps) == 0:
html_table = '<br><p color="red"><b>No landing page data found.<b></p><br>'
else:
html_table = DR.DataReporting()._write_html_table(live_lps, columns)
return render_to_response('live_lps/index.html',
{'html_table': html_table},
context_instance=RequestContext(request))
def mining_patterns_add(request):
err_msg = ''
mptl = DL.MiningPatternsTableLoader()
type = 'banner'
""" Extract Post data """
try:
regexp = MySQLdb._mysql.escape_string(request.POST['regexp_pattern'])
type = MySQLdb._mysql.escape_string(request.POST['pattern_type'])
except:
err_msg = 'Fields to add mining pattern incorrect.'
pass
mptl.insert_row(pattern_type=type, pattern=regexp)
banner_patterns, lp_patterns = mptl.get_pattern_lists()
return render_to_response('LML/mining_patterns.html', {'err_msg' : err_msg , 'banner_patterns' : banner_patterns, 'lp_patterns' : lp_patterns}, context_instance=RequestContext(request))
def mining_patterns_delete(request):
mptl = DL.MiningPatternsTableLoader()
try:
banner_patterns = request.POST.getlist('banner_patterns')
""" Escape POST input """
for index in range(len(banner_patterns)):
banner_patterns[index] = MySQLdb._mysql.escape_string(str(banner_patterns[index]))
except:
banner_patterns = list()
pass
try:
lp_patterns = request.POST.getlist('lp_patterns')
""" Escape POST input """
for index in range(len(lp_patterns)):
lp_patterns[index] = MySQLdb._mysql.escape_string(str(lp_patterns[index]))
except:
lp_patterns = list()
pass
logging.debug(banner_patterns)
logging.debug(lp_patterns)
""" Remove selected patterns """
for elem in banner_patterns:
mptl.delete_row(pattern=elem,pattern_type='banner')
for elem in lp_patterns:
mptl.delete_row(pattern=elem,pattern_type='lp')
banner_patterns, lp_patterns = mptl.get_pattern_lists()
return render_to_response('LML/mining_patterns.html', {'banner_patterns' : banner_patterns, 'lp_patterns' : lp_patterns}, context_instance=RequestContext(request))
def test(request):
try:
"""
PROCESS POST DATA
=================
Escape all user input that can be entered in text fields
"""
test_name_var = MySQLdb._mysql.escape_string(
request.POST['test_name'].strip())
utm_campaign_var = MySQLdb._mysql.escape_string(
request.POST['utm_campaign'].strip())
start_time_var = MySQLdb._mysql.escape_string(
request.POST['start_time'].strip())
end_time_var = MySQLdb._mysql.escape_string(
request.POST['end_time'].strip())
one_step_var = MySQLdb._mysql.escape_string(
request.POST['one_step'].strip())
country = MySQLdb._mysql.escape_string(request.POST['iso_filter'])
""" Convert timestamp format if necessary """
if TP.is_timestamp(start_time_var, 2):
start_time_var = TP.timestamp_convert_format(start_time_var, 2, 1)
if TP.is_timestamp(end_time_var, 2):
end_time_var = TP.timestamp_convert_format(end_time_var, 2, 1)
if cmp(one_step_var, 'True') == 0:
one_step_var = True
else:
one_step_var = False
try:
test_type_var = MySQLdb._mysql.escape_string(
request.POST['test_type'])
labels = request.POST['artifacts']
except KeyError:
test_type_var, labels = FDH.get_test_type(
utm_campaign_var, start_time_var, end_time_var,
DL.CampaignReportingLoader(
query_type='')) # submit an empty query type
labels = labels.__str__()
label_dict = dict()
label_dict_full = dict()
labels = labels[1:-1].split(',')
""" Parse the labels """
for i in range(len(labels)):
labels[i] = labels[i]
label = labels[i].split('\'')[1]
label = label.strip()
pieces = label.split(' ')
label = pieces[0]
for j in range(len(pieces) - 1):
label = label + '_' + pieces[j + 1]
""" Escape the label parameters """
label = MySQLdb._mysql.escape_string(label)
label_dict_full[label] = label
""" Look at the artifact names and map them into a dict() - Determine if artifacts were chosen by the user """
if request.POST.__contains__('artifacts_chosen'):
artifacts_chosen = request.POST.getlist('artifacts_chosen')
""" Ensure that only two items are selected """
if len(artifacts_chosen) > 2:
raise Exception(
'Please select (checkboxes) exactly two items to test')
for elem in artifacts_chosen:
esc_elem = MySQLdb._mysql.escape_string(str(elem))
label_dict[esc_elem] = esc_elem
else:
label_dict = label_dict_full
""" Parse the added labels IF they are not empty """
for key in label_dict.keys():
try:
if not (request.POST[key] == ''):
label_dict[key] = MySQLdb._mysql.escape_string(
str(request.POST[key]))
else:
label_dict[key] = key
except:
logging.error('Could not find %s in the POST QueryDict.' % key)
for key in label_dict_full.keys():
try:
if not (request.POST[key] == ''):
label_dict_full[key] = MySQLdb._mysql.escape_string(
str(request.POST[key]))
else:
label_dict_full[key] = key
except:
logging.error('Could not find %s in the POST QueryDict.' % key)
"""
EXECUTE REPORT GENERATION
=========================
setup time parameters
determine test metrics
execute queries
"""
sample_interval = 1
start_time_obj = TP.timestamp_to_obj(start_time_var, 1)
end_time_obj = TP.timestamp_to_obj(end_time_var, 1)
time_diff = end_time_obj - start_time_obj
time_diff_min = time_diff.seconds / 60.0
test_interval = int(math.floor(time_diff_min /
sample_interval)) # 2 is the interval
metric_types = FDH.get_test_type_metrics(test_type_var)
metric_types_full = dict()
""" Get the full (descriptive) version of the metric names
!! FIXME / TODO -- order these properly !! """
for i in range(len(metric_types)):
metric_types_full[metric_types[i]] = QD.get_metric_full_name(
metric_types[i])
""" USE generate_reporting_objects() TO GENERATE THE REPORT DATA - dependent on test type """
measured_metric, winner, loser, percent_win, confidence, html_table_pm_banner, html_table_pm_lp, html_table_language, html_table \
= generate_reporting_objects(test_name_var, start_time_var, end_time_var, utm_campaign_var, label_dict, label_dict_full, \
sample_interval, test_interval, test_type_var, metric_types, one_step_var, country)
winner_var = winner[0]
results = list()
for index in range(len(winner)):
results.append({
'metric': measured_metric[index],
'winner': winner[index],
'loser': loser[index],
'percent_win': percent_win[index],
'confidence': confidence[index]
})
template_var_dict = {'results' : results, \
'utm_campaign' : utm_campaign_var, 'metric_names_full' : metric_types_full, \
'summary_table': html_table, 'sample_interval' : sample_interval, \
'banner_pm_table' : html_table_pm_banner, 'lp_pm_table' : html_table_pm_lp, 'html_table_language' : html_table_language, \
'start_time' : TP.timestamp_convert_format(start_time_var, 1, 2) , 'end_time' : TP.timestamp_convert_format(end_time_var, 1, 2)}
html = render_to_response('tests/results_' + test_type_var + '.html',
template_var_dict,
context_instance=RequestContext(request))
"""
WRITE TO TEST TABLE
===================
"""
ttl = DL.TestTableLoader()
""" Format the html string """
html_string = html.__str__()
html_string = html_string.replace('"', '\\"')
if ttl.record_exists(utm_campaign=utm_campaign_var):
ttl.update_test_row(test_name=test_name_var,
test_type=test_type_var,
utm_campaign=utm_campaign_var,
start_time=start_time_var,
end_time=end_time_var,
html_report=html_string,
winner=winner_var)
else:
ttl.insert_row(test_name=test_name_var,
test_type=test_type_var,
utm_campaign=utm_campaign_var,
start_time=start_time_var,
end_time=end_time_var,
html_report=html_string,
winner=winner_var)
return html
except Exception as inst:
logging.error('Failed to correctly generate test report.')
logging.error(type(inst))
logging.error(inst.args)
logging.error(inst)
""" Return to the index page with an error """
try:
err_msg = 'Test Generation failed for: %s. Check the fields submitted for generation. <br><br>ERROR:<br><br>%s' % (
utm_campaign_var, inst.__str__())
except:
err_msg = 'Test Generation failed. Check the fields submitted for generation. <br><br>ERROR:<br><br>%s' % inst.__str__(
)
return campaigns_index(request, kwargs={'err_msg': err_msg})
return show_campaigns(request,
utm_campaign_var,
kwargs={'err_msg': err_msg})
def show_campaigns(request, utm_campaign, **kwargs):
"""
PROCESS POST KWARGS
===================
"""
err_msg = ''
try:
err_msg = str(kwargs['kwargs']['err_msg'])
except:
pass
test_type_override = ''
try:
test_type_override = MySQLdb._mysql.escape_string(
request.POST['test_type_override'])
if test_type_override == 'Banner':
test_type_var = FDH._TESTTYPE_BANNER_
elif test_type_override == 'Landing Page':
test_type_var = FDH._TESTTYPE_LP_
elif test_type_override == 'Banner and LP':
test_type_var = FDH._TESTTYPE_BANNER_LP_
except:
test_type_var = ''
pass
try:
""" Find the earliest and latest page views for a given campaign """
lptl = DL.LandingPageTableLoader()
ccrml = DL.CiviCRMLoader()
start_time = ccrml.get_earliest_donation(utm_campaign)
end_time = ccrml.get_latest_donation(utm_campaign)
one_step = lptl.is_one_step(start_time, end_time, utm_campaign)
if not (one_step):
start_time = lptl.get_earliest_campaign_view(utm_campaign)
end_time = lptl.get_latest_campaign_view(utm_campaign)
interval = 1
""" Create reporting object to retrieve campaign data and write plots to image repo on disk """
ir = DR.IntervalReporting(was_run=False,
use_labels=False,
font_size=20,
plot_type='line',
query_type='campaign',
file_path=projSet.__web_home__ +
'campaigns/static/images/')
""" Produce analysis on the campaign view data """
ir.run(start_time,
end_time,
interval,
'views',
utm_campaign, {},
one_step=one_step)
"""
ESTIMATE THE START AND END TIME OF THE CAMPAIGN
===============================================
Search for the first instance when more than 10 views are observed over a sampling period
"""
col_names = ir._data_loader_.get_column_names()
views_index = col_names.index('views')
ts_index = col_names.index('ts')
row_list = list(ir._data_loader_._results_) # copy the query results
for row in row_list:
if row[views_index] > 100:
start_time_est = row[ts_index]
break
row_list.reverse()
for row in row_list:
if row[views_index] > 100:
end_time_est = row[ts_index]
break
"""
BUILD THE VISUALIZATION FOR THE TEST VIEWS OF THIS CAMAPAIGN
============================================================
"""
""" Read the test name """
ttl = DL.TestTableLoader()
row = ttl.get_test_row(utm_campaign)
test_name = ttl.get_test_field(row, 'test_name')
""" Regenerate the data using the estimated start and end times """
ir = DR.IntervalReporting(was_run=False,
use_labels=False,
font_size=20,
plot_type='line',
query_type='campaign',
file_path=projSet.__web_home__ +
'campaigns/static/images/')
ir.run(start_time_est,
end_time_est,
interval,
'views',
utm_campaign, {},
one_step=one_step)
""" Determine the type of test (if not overridden) and retrieve the artifacts """
test_type, artifact_name_list = FDH.get_test_type(
utm_campaign, start_time, end_time,
DL.CampaignReportingLoader(query_type=''), test_type_var)
return render_to_response('campaigns/show_campaigns.html', {'utm_campaign' : utm_campaign, 'test_name' : test_name, 'start_time' : start_time_est, 'end_time' : end_time_est, 'one_step' : one_step, \
'artifacts' : artifact_name_list, 'test_type' : test_type, 'err_msg' : err_msg}, context_instance=RequestContext(request))
except Exception as inst:
logging.error('Failed to correctly produce campaign diagnostics.')
logging.error(type(inst))
logging.error(inst.args)
logging.error(inst)
""" Return to the index page with an error """
err_msg = 'There is insufficient data to analyze this campaign: %s. Check to see if the <a href="/LML/">impressions have been loaded</a>. <br><br>ERROR:<br><br>%s' % (
utm_campaign, inst.__str__())
return index(request, kwargs={'err_msg': err_msg})
def daily_totals(request):
err_msg = ''
start_day_ts = TP.timestamp_from_obj(
datetime.datetime.utcnow() + datetime.timedelta(days=-1), 1, 0)
end_day_ts = TP.timestamp_from_obj(datetime.datetime.utcnow(), 1, 0)
country = '.{2}'
min_donation = 0
order_str = 'order by 1 desc,3 desc'
"""
PROCESS POST
"""
if 'start_day_ts' in request.POST:
if cmp(request.POST['start_day_ts'], '') != 0:
start_day_ts = MySQLdb._mysql.escape_string(
request.POST['start_day_ts'].strip())
format = TP.getTimestampFormat(start_day_ts)
if format == 2:
start_day_ts = TP.timestamp_convert_format(start_day_ts, 2, 1)
# start_day_ts = start_day_ts[:8] + '000000'
elif format == -1:
err_msg = err_msg + 'Start timestamp is formatted incorrectly\n'
if 'end_day_ts' in request.POST:
if cmp(request.POST['end_day_ts'], '') != 0:
end_day_ts = MySQLdb._mysql.escape_string(
request.POST['end_day_ts'].strip())
format = TP.getTimestampFormat(start_day_ts)
if format == 2:
end_day_ts = TP.timestamp_convert_format(end_day_ts, 2, 1)
# end_day_ts = end_day_ts[:8] + '000000'
elif format == -1:
err_msg = err_msg + 'End timestamp is formatted incorrectly\n'
if 'country' in request.POST:
if cmp(request.POST['country'], '') != 0:
country = MySQLdb._mysql.escape_string(request.POST['country'])
if 'min_donation' in request.POST:
if cmp(request.POST['min_donation'], '') != 0:
try:
min_donation = int(
MySQLdb._mysql.escape_string(
request.POST['min_donation'].strip()))
except:
logging.error(
'live_results/daily_totals -- Could not process minimum donation for "%s" '
% request.POST['min_donation'].strip())
min_donation = 0
if 'order_metric' in request.POST:
if cmp(request.POST['order_metric'], 'Date') == 0:
order_str = 'order by 1 desc,3 desc'
elif cmp(request.POST['order_metric'], 'Country') == 0:
order_str = 'order by 2 asc,1 desc'
"""
=== END POST ===
"""
query_name = 'report_daily_totals_by_country'
filename = projSet.__sql_home__ + query_name + '.sql'
sql_stmnt = Hlp.file_to_string(filename)
sql_stmnt = QD.format_query(query_name,
sql_stmnt, [start_day_ts, end_day_ts],
country=country,
min_donation=min_donation,
order_str=order_str)
dl = DL.DataLoader()
results = dl.execute_SQL(sql_stmnt)
html_table = DR.DataReporting()._write_html_table(
results, dl.get_column_names(), use_standard_metric_names=True)
return render_to_response('live_results/daily_totals.html', \
{'html_table' : html_table, 'start_time' : TP.timestamp_convert_format(start_day_ts, 1, 2), 'end_time' : TP.timestamp_convert_format(end_day_ts, 1, 2)}, \
context_instance=RequestContext(request))
def impression_list(request):
err_msg = ''
where_clause = ''
"""
Process times and POST
=============
"""
duration_hrs = 2
end_time, start_time = TP.timestamps_for_interval(
datetime.datetime.utcnow(), 1, hours=-duration_hrs)
if 'earliest_utc_ts' in request.POST:
if cmp(request.POST['earliest_utc_ts'], '') != 0:
earliest_utc_ts = MySQLdb._mysql.escape_string(
request.POST['earliest_utc_ts'].strip())
format = TP.getTimestampFormat(earliest_utc_ts)
if format == 1:
start_time = earliest_utc_ts
if format == 2:
start_time = TP.timestamp_convert_format(earliest_utc_ts, 2, 1)
elif format == -1:
err_msg = err_msg + 'Start timestamp is formatted incorrectly\n'
if 'latest_utc_ts' in request.POST:
if cmp(request.POST['latest_utc_ts'], '') != 0:
latest_utc_ts = MySQLdb._mysql.escape_string(
request.POST['latest_utc_ts'].strip())
format = TP.getTimestampFormat(latest_utc_ts)
if format == 1:
end_time = latest_utc_ts
if format == 2:
end_time = TP.timestamp_convert_format(latest_utc_ts, 2, 1)
elif format == -1:
err_msg = err_msg + 'End timestamp is formatted incorrectly\n'
if 'iso_code' in request.POST:
if cmp(request.POST['iso_code'], '') != 0:
iso_code = MySQLdb._mysql.escape_string(
request.POST['iso_code'].strip())
where_clause = "where bi.country regexp '%s' " % iso_code
"""
Format and execute query
========================
"""
query_name = 'report_country_impressions.sql'
sql_stmnt = Hlp.file_to_string(projSet.__sql_home__ + query_name)
sql_stmnt = sql_stmnt % (start_time, end_time, start_time, end_time,
start_time, end_time, where_clause)
dl = DL.DataLoader()
results = dl.execute_SQL(sql_stmnt)
column_names = dl.get_column_names()
imp_table = DR.DataReporting()._write_html_table(results, column_names)
return render_to_response(
'live_results/impression_list.html', {
'imp_table': imp_table.decode("utf-8"),
'err_msg': err_msg,
'start': TP.timestamp_convert_format(start_time, 1, 2),
'end': TP.timestamp_convert_format(end_time, 1, 2)
},
context_instance=RequestContext(request))
def generate_summary(request):
try:
err_msg = ''
"""
PROCESS POST DATA
=================
Escape all user input that can be entered in text fields
"""
if 'utm_campaign' in request.POST:
utm_campaign = MySQLdb._mysql.escape_string(
request.POST['utm_campaign'])
if 'start_time' in request.POST:
start_time = MySQLdb._mysql.escape_string(
request.POST['start_time'].strip())
if not (TP.is_timestamp(start_time, 1)) and not (TP.is_timestamp(
start_time, 2)):
err_msg = 'Incorrectly formatted start timestamp.'
raise Exception()
if 'end_time' in request.POST:
end_time = MySQLdb._mysql.escape_string(
request.POST['end_time'].strip())
if not (TP.is_timestamp(end_time, 1)) and not (TP.is_timestamp(
end_time, 2)):
err_msg = 'Incorrectly formatted end timestamp.'
raise Exception()
if 'iso_filter' in request.POST:
country = MySQLdb._mysql.escape_string(request.POST['iso_filter'])
else:
country = '.{2}'
if 'measure_confidence' in request.POST:
if cmp(request.POST['measure_confidence'], 'yes') == 0:
measure_confidence = True
else:
measure_confidence = False
else:
measure_confidence = False
if 'one_step' in request.POST:
if cmp(request.POST['one_step'], 'yes') == 0:
use_one_step = True
else:
use_one_step = False
else:
use_one_step = False
if 'donations_only' in request.POST:
if cmp(request.POST['donations_only'], 'yes') == 0:
donations_only = True
else:
donations_only = False
else:
donations_only = False
""" Convert timestamp format if necessary """
if TP.is_timestamp(start_time, 2):
start_time = TP.timestamp_convert_format(start_time, 2, 1)
if TP.is_timestamp(end_time, 2):
end_time = TP.timestamp_convert_format(end_time, 2, 1)
""" =============================================== """
"""
GENERATE A REPORT SUMMARY TABLE
===============================
"""
if donations_only:
srl = DL.SummaryReportingLoader(
query_type=FDH._TESTTYPE_DONATIONS_)
else:
srl = DL.SummaryReportingLoader(
query_type=FDH._TESTTYPE_BANNER_LP_)
srl.run_query(start_time,
end_time,
utm_campaign,
min_views=-1,
country=country)
column_names = srl.get_column_names()
summary_results = srl.get_results()
if not (summary_results):
html_table = '<h3>No artifact summary data available for %s.</h3>' % utm_campaign
else:
summary_results_list = list()
for row in summary_results:
summary_results_list.append(list(row))
summary_results = summary_results_list
"""
Format results to encode html table cell markup in results
"""
if measure_confidence:
ret = DR.ConfidenceReporting(
query_type='', hyp_test='').get_confidence_on_time_range(
start_time,
end_time,
utm_campaign,
one_step=use_one_step,
country=country) # first get color codes on confidence
conf_colour_code = ret[0]
for row_index in range(len(summary_results)):
artifact_index = summary_results[row_index][
0] + '-' + summary_results[row_index][
1] + '-' + summary_results[row_index][2]
for col_index in range(len(column_names)):
is_coloured_cell = False
if column_names[col_index] in conf_colour_code.keys():
if artifact_index in conf_colour_code[
column_names[col_index]].keys():
summary_results[row_index][
col_index] = '<td style="background-color:' + conf_colour_code[
column_names[col_index]][
artifact_index] + ';">' + str(
summary_results[row_index]
[col_index]) + '</td>'
is_coloured_cell = True
if not (is_coloured_cell):
summary_results[row_index][
col_index] = '<td>' + str(
summary_results[row_index]
[col_index]) + '</td>'
html_table = DR.DataReporting()._write_html_table(
summary_results,
column_names,
use_standard_metric_names=True,
omit_cell_markup=True)
else:
html_table = DR.DataReporting()._write_html_table(
summary_results,
column_names,
use_standard_metric_names=True)
""" Generate totals only if it's a non-donation-only query """
if donations_only:
srl = DL.SummaryReportingLoader(
query_type=FDH._QTYPE_TOTAL_DONATIONS_)
else:
srl = DL.SummaryReportingLoader(query_type=FDH._QTYPE_TOTAL_)
srl.run_query(start_time,
end_time,
utm_campaign,
min_views=-1,
country=country)
total_summary_results = srl.get_results()
if not (total_summary_results):
html_table = html_table + '<div class="spacer"></div><div class="spacer"></div><h3>No data available for %s Totals.</h3>' % utm_campaign
else:
html_table = html_table + '<div class="spacer"></div><div class="spacer"></div>' + DR.DataReporting(
)._write_html_table(total_summary_results,
srl.get_column_names(),
use_standard_metric_names=True)
metric_legend_table = DR.DataReporting().get_standard_metrics_legend()
conf_legend_table = DR.ConfidenceReporting(
query_type='bannerlp',
hyp_test='TTest').get_confidence_legend_table()
html_table = '<h4><u>Metrics Legend:</u></h4><div class="spacer"></div>' + metric_legend_table + \
'<div class="spacer"></div><h4><u>Confidence Legend for Hypothesis Testing:</u></h4><div class="spacer"></div>' + conf_legend_table + '<div class="spacer"></div><div class="spacer"></div>' + html_table
"""
DETERMINE PAYMENT METHODS
=========================
"""
ccl = DL.CiviCRMLoader()
pm_data_counts, pm_data_conversions = ccl.get_payment_methods(
utm_campaign, start_time, end_time, country=country)
html_table_pm_counts = DR.IntervalReporting(
).write_html_table_from_rowlists(
pm_data_counts, ['Payment Method', 'Portion of Donations (%)'],
'Landing Page')
html_table_pm_conversions = DR.IntervalReporting(
).write_html_table_from_rowlists(pm_data_conversions, [
'Payment Method', 'Visits', 'Conversions', 'Conversion Rate (%)',
'Amount', 'Amount 25'
], 'Landing Page')
html_table = html_table + '<div class="spacer"></div><h4><u>Payment Methods Breakdown:</u></h4><div class="spacer"></div>' + html_table_pm_counts + \
'<div class="spacer"></div><div class="spacer"></div>' + html_table_pm_conversions + '<div class="spacer"></div><div class="spacer"></div>'
return render_to_response('tests/table_summary.html', {
'html_table': html_table,
'utm_campaign': utm_campaign
},
context_instance=RequestContext(request))
except Exception as inst:
if cmp(err_msg, '') == 0:
err_msg = 'Could not generate campaign tabular results.'
return index(request, err_msg=err_msg)
def execute_process(self, key, **kwargs):
logging.info('Commencing caching of live results data at: %s' %
self.CACHING_HOME)
shelve_key = key
""" Find the earliest and latest page views for a given campaign """
lptl = DL.LandingPageTableLoader(db='db1025')
query_name = 'report_summary_results_country.sql'
query_name_1S = 'report_summary_results_country_1S.sql'
campaign_regexp_filter = '^C_|^C11_'
dl = DL.DataLoader(db='db1025')
end_time, start_time = TP.timestamps_for_interval(
datetime.datetime.utcnow(), 1, hours=-self.DURATION_HRS)
""" Should a one-step query be used? """
use_one_step = lptl.is_one_step(
start_time, end_time, 'C11'
) # Assume it is a one step test if there are no impressions for this campaign in the landing page table
"""
Retrieve the latest time for which impressions have been loaded
===============================================================
"""
sql_stmnt = 'select max(end_time) as latest_ts from squid_log_record where log_completion_pct = 100.00'
results = dl.execute_SQL(sql_stmnt)
latest_timestamp = results[0][0]
latest_timestamp = TP.timestamp_from_obj(latest_timestamp, 2, 3)
latest_timestamp_flat = TP.timestamp_convert_format(
latest_timestamp, 2, 1)
ret = DR.ConfidenceReporting(query_type='', hyp_test='',
db='db1025').get_confidence_on_time_range(
start_time,
end_time,
campaign_regexp_filter,
one_step=use_one_step)
measured_metrics_counts = ret[1]
""" Prepare Summary results """
sql_stmnt = Hlp.file_to_string(projSet.__sql_home__ + query_name)
sql_stmnt = sql_stmnt % (start_time, latest_timestamp_flat, start_time, latest_timestamp_flat, campaign_regexp_filter, start_time, latest_timestamp_flat, \
start_time, end_time, campaign_regexp_filter, start_time, end_time, campaign_regexp_filter, start_time, end_time, campaign_regexp_filter, \
start_time, latest_timestamp_flat, campaign_regexp_filter, start_time, latest_timestamp_flat, campaign_regexp_filter)
logging.info('Executing report_summary_results ...')
results = dl.execute_SQL(sql_stmnt)
column_names = dl.get_column_names()
if use_one_step:
logging.info('... including one step artifacts ...')
sql_stmnt_1S = Hlp.file_to_string(projSet.__sql_home__ +
query_name_1S)
sql_stmnt_1S = sql_stmnt_1S % (start_time, latest_timestamp_flat, start_time, latest_timestamp_flat, campaign_regexp_filter, start_time, latest_timestamp_flat, \
start_time, end_time, campaign_regexp_filter, start_time, end_time, campaign_regexp_filter, start_time, end_time, campaign_regexp_filter, \
start_time, latest_timestamp_flat, campaign_regexp_filter, start_time, latest_timestamp_flat, campaign_regexp_filter)
results = list(results)
results_1S = dl.execute_SQL(sql_stmnt_1S)
""" Ensure that the results are unique """
one_step_keys = list()
for row in results_1S:
one_step_keys.append(str(row[0]) + str(row[1]) + str(row[2]))
new_results = list()
for row in results:
key = str(row[0]) + str(row[1]) + str(row[2])
if not (key in one_step_keys):
new_results.append(row)
results = new_results
results.extend(list(results_1S))
metric_legend_table = DR.DataReporting().get_standard_metrics_legend()
conf_legend_table = DR.ConfidenceReporting(
query_type='bannerlp',
hyp_test='TTest').get_confidence_legend_table()
""" Create a interval loader objects """
sampling_interval = 5 # 5 minute sampling interval for donation plots
ir_cmpgn = DR.IntervalReporting(query_type=FDH._QTYPE_CAMPAIGN_ +
FDH._QTYPE_TIME_,
generate_plot=False,
db='db1025')
ir_banner = DR.IntervalReporting(query_type=FDH._QTYPE_BANNER_ +
FDH._QTYPE_TIME_,
generate_plot=False,
db='db1025')
ir_lp = DR.IntervalReporting(query_type=FDH._QTYPE_LP_ +
FDH._QTYPE_TIME_,
generate_plot=False,
db='db1025')
""" Execute queries """
ir_cmpgn.run(start_time, end_time, sampling_interval, 'donations', '',
{})
ir_banner.run(start_time, end_time, sampling_interval, 'donations', '',
{})
ir_lp.run(start_time, end_time, sampling_interval, 'donations', '', {})
""" Prepare serialized objects """
dict_param = dict()
dict_param['metric_legend_table'] = metric_legend_table
dict_param['conf_legend_table'] = conf_legend_table
dict_param['measured_metrics_counts'] = measured_metrics_counts
dict_param['results'] = results
dict_param['column_names'] = column_names
dict_param['interval'] = sampling_interval
dict_param['duration'] = self.DURATION_HRS
dict_param['start_time'] = TP.timestamp_convert_format(
start_time, 1, 2)
dict_param['end_time'] = TP.timestamp_convert_format(end_time, 1, 2)
dict_param['ir_cmpgn_counts'] = ir_cmpgn._counts_
dict_param['ir_banner_counts'] = ir_banner._counts_
dict_param['ir_lp_counts'] = ir_lp._counts_
dict_param['ir_cmpgn_times'] = ir_cmpgn._times_
dict_param['ir_banner_times'] = ir_banner._times_
dict_param['ir_lp_times'] = ir_lp._times_
self.clear_cached_data(shelve_key)
self.cache_data(dict_param, shelve_key)
logging.info('Caching complete.')
def index(request, **kwargs):
crl = DL.CampaignReportingLoader(query_type='totals')
filter_data = True
""" Determine the start and end times for the query """
start_time_obj = datetime.datetime.utcnow() + datetime.timedelta(days=-1)
end_time = TP.timestamp_from_obj(datetime.datetime.utcnow(), 1, 3)
start_time = TP.timestamp_from_obj(start_time_obj, 1, 3)
"""
PROCESS POST KWARGS
===================
"""
err_msg = ''
try:
err_msg = str(kwargs['kwargs']['err_msg'])
except:
pass
"""
PROCESS POST VARS
=================
"""
""" Process error message """
try:
err_msg = MySQLdb._mysql.escape_string(request.POST['err_msg'])
except KeyError:
pass
""" If the filter form was submitted extract the POST vars """
try:
min_donations_var = MySQLdb._mysql.escape_string(
request.POST['min_donations'].strip())
earliest_utc_ts_var = MySQLdb._mysql.escape_string(
request.POST['utc_ts'].strip())
""" If the user timestamp is earlier than the default start time run the query for the earlier start time """
ts_format = TP.getTimestampFormat(earliest_utc_ts_var)
""" Ensure the validity of the timestamp input """
if ts_format == TP.TS_FORMAT_FORMAT1:
start_time = TP.timestamp_convert_format(earliest_utc_ts_var,
TP.TS_FORMAT_FORMAT1,
TP.TS_FORMAT_FLAT)
elif ts_format == TP.TS_FORMAT_FLAT:
start_time = earliest_utc_ts_var
elif cmp(earliest_utc_ts_var, '') == 0:
start_time = TP.timestamp_from_obj(start_time_obj, 1, 3)
else:
raise Exception()
if cmp(min_donations_var, '') == 0:
min_donations_var = -1
else:
min_donations_var = int(min_donations_var)
except KeyError: # In the case the form was not submitted set minimum donations and retain the default start time
min_donations_var = -1
pass
except Exception: # In the case the form was incorrectly formatted notify the user
min_donations_var = -1
start_time = TP.timestamp_from_obj(start_time_obj, 1, 3)
err_msg = 'Filter fields are incorrect.'
"""
GENERATE CAMPAIGN DATA
======================
"""
campaigns, all_data = crl.run_query({
'metric_name': 'earliest_timestamp',
'start_time': start_time,
'end_time': end_time
})
""" Sort campaigns by earliest access """
sorted_campaigns = sorted(campaigns.iteritems(),
key=operator.itemgetter(1))
sorted_campaigns.reverse()
"""
FILTER CAMPAIGN DATA
====================
"""
new_sorted_campaigns = list()
for campaign in sorted_campaigns:
key = campaign[0]
if campaign[1] > 0:
name = all_data[key][0]
if name == None:
name = 'none'
timestamp = TP.timestamp_convert_format(all_data[key][3], 1, 2)
if filter_data:
if all_data[key][2] > min_donations_var:
new_sorted_campaigns.append([
campaign[0], campaign[1], name, timestamp,
all_data[key][2], all_data[key][4]
])
else:
new_sorted_campaigns.append([
campaign[0], campaign[1], name, timestamp,
all_data[key][2], all_data[key][4]
])
sorted_campaigns = new_sorted_campaigns
return render_to_response('campaigns/index.html', {
'campaigns': sorted_campaigns,
'err_msg': err_msg
},
context_instance=RequestContext(request))
def __init__(self):
self._data_loader_ = DL.TTestLoaderHelp()
def execute_process(self, key, **kwargs):
logging.info('Commencing caching of long term trends data at: %s' %
self.CACHING_HOME)
end_time, start_time = TP.timestamps_for_interval(datetime.datetime.utcnow(), 1, \
hours=-self.VIEW_DURATION_HRS, resolution=1)
""" DATA CONFIG """
countries = DL.CiviCRMLoader().get_ranked_donor_countries(start_time)
countries = countries[1:6]
""" set the metrics to plot """
lttdl = DL.LongTermTrendsLoader(db='storage3')
""" Dictionary object storing lists of regexes - each expression must pass for a label to persist """
# country_groups = {'US': ['(US)'], 'CA': ['(CA)'], 'JP': ['(JP)'], 'IN': ['(IN)'], 'NL': ['(NL)']}
payment_groups = {'Credit Card': ['^cc$'], 'Paypal': ['^pp$']}
currency_groups = {
'USD': ['(USD)'],
'CAD': ['(CAD)'],
'JPY': ['(JPY)'],
'EUR': ['(EUR)']
}
lang_cntry_groups = {
'US': ['US..', '.{4}'],
'EN': ['[^U^S]en', '.{4}']
}
top_cntry_groups = dict()
for country in countries:
top_cntry_groups[country] = [country, '.{2}']
# To include click rate
# groups = [ lang_cntry_groups] metrics = ['click_rate'] metrics_index = [3]
# group_metrics = [DL.LongTermTrendsLoader._MT_RATE_] metric_types = ['country', 'language'] include_totals = [True] include_others = [True]
metrics = [
'impressions', 'views', 'donations', 'donations', 'amount',
'amount', 'diff_don', 'diff_don', 'donations', 'conversion_rate'
]
weights = ['', '', '', '', '', '', 'donations', 'donations', '', '']
metrics_index = [0, 1, 2, 2, 2, 4, 5, 5, 6, 6]
groups = [lang_cntry_groups, lang_cntry_groups, lang_cntry_groups, top_cntry_groups, lang_cntry_groups, currency_groups, \
lang_cntry_groups, lang_cntry_groups, payment_groups, payment_groups]
""" The metrics that are used to build a group string to be qualified via regex - the values of the list metrics are concatenated """
group_metrics = [['country', 'language'], ['country', 'language'], ['country', 'language'], \
['country', 'language'], ['country', 'language'], ['currency'], ['country', 'language'], \
['country', 'language'], ['payment_method'], ['payment_method']]
metric_types = [DL.LongTermTrendsLoader._MT_AMOUNT_, DL.LongTermTrendsLoader._MT_AMOUNT_, DL.LongTermTrendsLoader._MT_AMOUNT_, \
DL.LongTermTrendsLoader._MT_AMOUNT_, DL.LongTermTrendsLoader._MT_AMOUNT_, DL.LongTermTrendsLoader._MT_AMOUNT_, \
DL.LongTermTrendsLoader._MT_RATE_WEIGHTED_, DL.LongTermTrendsLoader._MT_RATE_WEIGHTED_, DL.LongTermTrendsLoader._MT_AMOUNT_, \
DL.LongTermTrendsLoader._MT_RATE_]
include_totals = [
True, True, True, False, True, True, False, False, False, True
]
include_others = [
True, True, True, False, True, True, True, True, True, False
]
hours_back = [0, 0, 0, 0, 0, 0, 24, 168, 0, 0]
time_unit = [
TP.HOUR, TP.HOUR, TP.HOUR, TP.HOUR, TP.HOUR, TP.HOUR, TP.HOUR,
TP.HOUR, TP.HOUR, TP.HOUR
]
data = list()
""" END CONFIG """
""" For each metric use the LongTermTrendsLoader to generate the data to plot """
for index in range(len(metrics)):
dr = DR.DataReporting()
times, counts = lttdl.run_query(start_time, end_time, metrics_index[index], metric_name=metrics[index], metric_type=metric_types[index], \
groups=groups[index], group_metric=group_metrics[index], include_other=include_others[index], \
include_total=include_totals[index], hours_back=hours_back[index], weight_name=weights[index], \
time_unit=time_unit[index])
times = TP.normalize_timestamps(times, False, time_unit[index])
dr._counts_ = counts
dr._times_ = times
empty_data = [0] * len(times[times.keys()[0]])
data.append(dr.get_data_lists([''], empty_data))
dict_param = Hlp.combine_data_lists(data)
dict_param['interval'] = self.VIEW_DURATION_HRS
dict_param['end_time'] = TP.timestamp_convert_format(end_time, 1, 2)
self.clear_cached_data(key)
self.cache_data(dict_param, key)
logging.info('Caching complete.')
def generate_reporting_objects(test_name, start_time, end_time, campaign,
label_dict, label_dict_full, sample_interval,
test_interval, test_type, metric_types,
one_step_var, country):
""" Labels will always be metric names in this case """
# e.g. labels = {'Static banner':'20101227_JA061_US','Fading banner':'20101228_JAFader_US'}
use_labels_var = True
""" Build reporting objects """
ir_cmpgn = DR.IntervalReporting(use_labels=False,
font_size=20,
plot_type='line',
query_type='campaign',
file_path=projSet.__web_home__ +
'campaigns/static/images/')
"""
DETERMINE DONOR DOLLAR BREAKDOWN
================================
"""
try:
logging.info('')
logging.info('Determining Donations Distribution:')
logging.info('===================================\n')
DR.DonorBracketReporting(query_type=FDH._QTYPE_LP_,
file_path=projSet.__web_home__ +
'tests/static/images/').run(
start_time, end_time, campaign)
except:
pass
"""
DETERMINE CATEGORY DISTRIBUTION
===============================
"""
if (0):
DR.CategoryReporting(file_path=projSet.__web_home__ +
'tests/static/images/').run(
start_time, end_time, campaign)
"""
DETERMINE LANGUAGE BREAKDOWN
============================
"""
html_language = ''
if (1):
logging.info('')
logging.info('Determining Languages Distribution:')
logging.info('===================================\n')
columns, data = DL.CiviCRMLoader().get_donor_by_language(
campaign, start_time, end_time)
html_language = DR.DataReporting()._write_html_table(data, columns)
"""
DETERMINE PAYMENT METHODS
=========================
"""
logging.info('')
logging.info('Determining Payment Methods:')
logging.info('============================\n')
ccl = DL.CiviCRMLoader()
pm_data_counts, pm_data_conversions = ccl.get_payment_methods(
campaign, start_time, end_time, country=country)
html_table_pm_counts = DR.IntervalReporting(
).write_html_table_from_rowlists(
pm_data_counts, ['Payment Method', 'Portion of Donations (%)'],
'Landing Page')
html_table_pm_conversions = DR.IntervalReporting(
).write_html_table_from_rowlists(pm_data_conversions, [
'Payment Method', 'Visits', 'Conversions', 'Conversion Rate (%)',
'Amount', 'Amount 25'
], 'Landing Page')
"""
BUILD REPORTING OBJECTS
=======================
"""
if test_type == FDH._TESTTYPE_BANNER_:
ir = DR.IntervalReporting(use_labels=use_labels_var,
font_size=20,
plot_type='step',
query_type=FDH._QTYPE_BANNER_,
file_path=projSet.__web_home__ +
'tests/static/images/')
link_item = '<a href="http://meta.wikimedia.org/w/index.php?title=Special:NoticeTemplate/view&template=%s">%s</a>'
measured_metric = ['don_per_imp', 'amt_norm_per_imp', 'click_rate']
elif test_type == FDH._TESTTYPE_LP_:
ir = DR.IntervalReporting(use_labels=use_labels_var,
font_size=20,
plot_type='step',
query_type=FDH._QTYPE_LP_,
file_path=projSet.__web_home__ +
'tests/static/images/')
link_item = '<a href="http://meta.wikimedia.org/w/index.php?title=Special:NoticeTemplate/view&template=%s">%s</a>'
measured_metric = ['don_per_view', 'amt_norm_per_view']
elif test_type == FDH._TESTTYPE_BANNER_LP_:
ir = DR.IntervalReporting(use_labels=use_labels_var,
font_size=20,
plot_type='step',
query_type=FDH._QTYPE_BANNER_LP_,
file_path=projSet.__web_home__ +
'tests/static/images/')
link_item = '<a href="http://meta.wikimedia.org/w/index.php?title=Special:NoticeTemplate/view&template=%s">%s</a>'
measured_metric = [
'don_per_imp', 'amt_norm_per_imp', 'don_per_view',
'amt_norm_per_view', 'click_rate'
]
"""
GENERATE PLOTS FOR EACH METRIC OF INTEREST
==========================================
"""
logging.info('')
logging.info('Determining Metric Minutely Counts:')
logging.info('==================================\n')
for metric in metric_types:
ir.run(start_time,
end_time,
sample_interval,
metric,
campaign,
label_dict,
one_step=one_step_var,
country=country)
"""
CHECK THE CAMPAIGN VIEWS AND DONATIONS
======================================
"""
ir_cmpgn.run(start_time,
end_time,
sample_interval,
'views',
campaign, {},
one_step=one_step_var,
country=country)
ir_cmpgn.run(start_time,
end_time,
sample_interval,
'donations',
campaign, {},
one_step=one_step_var,
country=country)
"""
PERFORM HYPOTHESIS TESTING
==========================
"""
logging.info('')
logging.info('Executing Confidence Queries:')
logging.info('============================\n')
column_colours = dict()
confidence = list()
cr = DR.ConfidenceReporting(use_labels=use_labels_var,
font_size=20,
plot_type='line',
hyp_test='t_test',
query_type=test_type,
file_path=projSet.__web_home__ +
'tests/static/images/')
for metric in measured_metric:
ret = cr.run(test_name,
campaign,
metric,
label_dict,
start_time,
end_time,
sample_interval,
one_step=one_step_var,
country=country)
confidence.append(ret[0])
column_colours[metric] = ret[1]
"""
GENERATE A REPORT SUMMARY TABLE
===============================
"""
logging.info('')
logging.info('Generating Summary Report:')
logging.info('=========================\n')
"""
if one_step_var == True:
summary_start_time = DL.CiviCRMLoader().get_earliest_donation(campaign)
else:
summary_start_time = DL.LandingPageTableLoader().get_earliest_campaign_view(campaign)
summary_end_time = DL.CiviCRMLoader().get_latest_donation(campaign)
"""
srl = DL.SummaryReportingLoader(query_type=test_type)
srl.run_query(start_time,
end_time,
campaign,
one_step=one_step_var,
country=country)
columns = srl.get_column_names()
summary_results = srl.get_results()
"""
REMOVED - links to pipeline artifacts, this was broken and should be implemented properly later
"""
""" Get Winners, Losers, and percent increase """
winner = list()
loser = list()
percent_increase = list()
labels = list()
for item_long_name in label_dict:
labels.append(label_dict[item_long_name])
for metric in measured_metric:
ret = srl.compare_artifacts(label_dict.keys(), metric, labels=labels)
winner.append(ret[0])
loser.append(ret[1])
percent_increase.append(ret[2])
""" Compose table for showing artifact """
html_table = DR.DataReporting()._write_html_table(
summary_results,
columns,
coloured_columns=column_colours,
use_standard_metric_names=True)
metric_legend_table = DR.DataReporting().get_standard_metrics_legend()
conf_legend_table = DR.ConfidenceReporting(
query_type='bannerlp', hyp_test='TTest').get_confidence_legend_table()
html_table = '<h4><u>Metrics Legend:</u></h4><div class="spacer"></div>' + metric_legend_table + \
'<div class="spacer"></div><h4><u>Confidence Legend for Hypothesis Testing:</u></h4><div class="spacer"></div>' + conf_legend_table + '<div class="spacer"></div><div class="spacer"></div>' + html_table
""" Generate totals for the test summary """
srl = DL.SummaryReportingLoader(query_type=FDH._QTYPE_TOTAL_)
srl.run_query(start_time,
end_time,
campaign,
one_step=one_step_var,
country=country)
html_table = html_table + '<br><br>' + DR.DataReporting(
)._write_html_table(srl.get_results(),
srl.get_column_names(),
use_standard_metric_names=True)
return [
measured_metric, winner, loser, percent_increase, confidence,
html_table_pm_counts, html_table_pm_conversions, html_language,
html_table
]
def execute_process(self, key, **kwargs):
logging.info('Commencing caching of fundraiser totals data at: %s' %
self.CACHING_HOME)
end_time = TP.timestamp_from_obj(datetime.datetime.utcnow(), 1, 3)
""" DATA CONFIG """
""" set the metrics to plot """
lttdl = DL.LongTermTrendsLoader(db='db1025')
start_of_2011_fundraiser = '20111116000000'
countries = DL.CiviCRMLoader().get_ranked_donor_countries(
start_of_2011_fundraiser)
countries.append('Total')
""" Dictionary object storing lists of regexes - each expression must pass for a label to persist """
year_groups = dict()
for country in countries:
if cmp(country, 'Total') == 0:
year_groups['2011 Total'] = ['2011.*']
year_groups['2010 Total'] = ['2010.*']
else:
year_groups['2011 ' + country] = ['2011' + country]
year_groups['2010 ' + country] = ['2010' + country]
metrics = 'amount'
weights = ''
groups = year_groups
group_metrics = ['year', 'country']
metric_types = DL.LongTermTrendsLoader._MT_AMOUNT_
include_totals = False
include_others = False
hours_back = 0
time_unit = TP.DAY
""" END CONFIG """
""" For each metric use the LongTermTrendsLoader to generate the data to plot """
dr = DR.DataReporting()
times, counts = lttdl.run_fundrasing_totals(end_time, metric_name=metrics, metric_type=metric_types, groups=groups, group_metric=group_metrics, include_other=include_others, \
include_total=include_totals, hours_back=hours_back, weight_name=weights, time_unit=time_unit)
dict_param = dict()
for country in countries:
key_2011 = '2011 ' + country
key_2010 = '2010 ' + country
new_counts = dict()
new_counts[key_2010] = counts[key_2010]
new_counts[key_2011] = counts[key_2011]
new_times = dict()
new_times[key_2010] = times[key_2010]
new_times[key_2011] = times[key_2011]
dr._counts_ = new_counts
dr._times_ = new_times
empty_data = [0] * len(new_times[new_times.keys()[0]])
data = list()
data.append(dr.get_data_lists([''], empty_data))
dict_param[country] = Hlp.combine_data_lists(data)
self.clear_cached_data(key)
self.cache_data(dict_param, key)
logging.info('Caching complete.')
def index(request, **kwargs):
"""
PROCESS POST DATA
=================
"""
if 'err_msg' in kwargs:
err_msg = kwargs['err_msg']
else:
err_msg = ''
try:
latest_utc_ts_var = MySQLdb._mysql.escape_string(
request.POST['latest_utc_ts'].strip())
earliest_utc_ts_var = MySQLdb._mysql.escape_string(
request.POST['earliest_utc_ts'].strip())
if not TP.is_timestamp(earliest_utc_ts_var, 1) or not TP.is_timestamp(
earliest_utc_ts_var, 1):
raise TypeError
if latest_utc_ts_var == '':
latest_utc_ts_var = _end_time_
except KeyError:
earliest_utc_ts_var = _beginning_time_
latest_utc_ts_var = _end_time_
except TypeError:
err_msg = 'Please enter a valid timestamp.'
earliest_utc_ts_var = _beginning_time_
latest_utc_ts_var = _end_time_
ttl = DL.TestTableLoader()
columns = ttl.get_column_names()
test_rows = ttl.get_all_test_rows()
""" Build a list of tests -- apply filters """
l = []
utm_campaign_index = ttl.get_test_index('utm_campaign')
html_report_index = ttl.get_test_index('html_report')
for i in test_rows:
test_start_time = ttl.get_test_field(i, 'start_time')
new_row = list(i)
""" Ensure the timestamp is properly formatted """
if TP.is_timestamp(test_start_time, 2):
test_start_time = TP.timestamp_convert_format(
test_start_time, 2, 1)
new_row[
html_report_index] = '<a href="/tests/report/%s">view</a>' % new_row[
utm_campaign_index]
if int(test_start_time) > int(earliest_utc_ts_var) and int(
test_start_time) < int(latest_utc_ts_var):
l.append(new_row)
l.reverse()
test_table = DR.DataReporting()._write_html_table(
l, columns, use_standard_metric_names=True)
return render_to_response('tests/index.html', {
'err_msg': err_msg,
'test_table': test_table
},
context_instance=RequestContext(request))
