def generate_highcharts_config(self, docs, args):
report = self.generate_report(docs, args)
params = args.get('params') or {}
chart_params = params.get('chart') or {}
items = report.get('items') or []
title = chart_params.get('title') or 'Changes to Featuremedia'
subtitle = chart_params.get(
'subtitle') or ChartConfig.gen_subtitle_for_dates(params)
translations = ChartConfig.get_translations(['task.user', 'operation'])
user_translations = (translations.get('task_user')
or {}).get('names') or {}
operation_translations = (translations.get('operation')
or {}).get('names') or {}
rows = []
def gen_date_str(date):
return utc_to_local(app.config['DEFAULT_TIMEZONE'],
date).strftime('%d/%m/%Y %H:%M')
for item in items:
original_image = item.get('original_image') or {}
updates = []
for update in item.get('updates') or []:
updates.append('{} - {} by {}'.format(
gen_date_str(parse_date(update.get('operation_created'))),
operation_translations.get(update.get('operation'))
or update.get('operation'),
user_translations.get(update.get('user')) or ''))
rows.append([
gen_date_str(item.get('versioncreated')),
user_translations.get(item.get('original_creator')) or '',
item.get('slugline') or '',
original_image.get('headline')
or original_image.get('alt_text') or '',
'<ul><li>{}</li></ul>'.format('</li><li>'.join(updates))
])
return {
'highcharts': [{
'id':
'featuremedia_updates',
'type':
'table',
'title':
title,
'subtitle':
subtitle,
'headers': [
'Date', 'Creator', 'Slugline', 'Original Image',
'Featuremedia History'
],
'rows':
rows
}]
}
def gen_chart_config():
chart_params = params.get('chart') or {}
if chart_params.get('title'):
title = chart_params['title']
else:
interval = histogram.get('interval') or 'daily'
if interval == 'hourly':
title = 'Hourly Desk Activity'
else:
title = 'Daily Desk Activity'
if chart_params.get('subtitle'):
subtitle = chart_params['subtitle']
else:
subtitle = ChartConfig.gen_subtitle_for_dates(params)
# Calculate the UTC Offset in minutes for the start date of the results
# This will cause an issue if a report crosses over the daylight savings change
# Any data after the daylight savings change will be 1 hour out
timezone_offset = get_utc_offset_in_minutes(
datetime.utcfromtimestamp(int(report['start_epoch'] / 1000)))
chart = SDChart.Chart('desk_activity_report',
title=title,
subtitle=subtitle,
chart_type='highcharts',
start_of_week=app.config.get('START_OF_WEEK')
or 0,
timezone_offset=timezone_offset,
use_utc=False,
legend_title='Desk Transitions',
default_config=ChartConfig.defaultConfig)
chart.set_translation('desk_transition', 'Desk Transitions', {
'incoming': 'Incoming',
'outgoing': 'Outgoing'
})
axis = chart.add_axis().set_options(
y_title='Desk Activity',
type='datetime',
default_chart_type=chart_params.get('type') or 'column',
point_start=report.get('start_epoch'),
point_interval=report.get('interval'),
stack_labels=False)
axis.add_series().set_options(field='desk_transition',
name='incoming',
data=report.get('incoming'))
axis.add_series().set_options(field='desk_transition',
name='outgoing',
data=report.get('outgoing'))
return chart.gen_config()
def generate_highcharts_config(self, docs, args):
items = list(self.generate_report(docs, args))
params = args.get('params') or {}
chart_params = params.get('chart') or {}
title = chart_params.get('title') or 'Update Time'
subtitle = chart_params.get(
'subtitle') or ChartConfig.gen_subtitle_for_dates(params)
rows = []
def gen_date_str(date):
return utc_to_local(app.config['DEFAULT_TIMEZONE'],
date).strftime('%d/%m/%Y %H:%M')
def gen_update_string(seconds):
times = (utcnow().replace(minute=0, hour=0,
second=0)) + timedelta(seconds=seconds)
times = times.strftime('%H:%M').split(':')
if int(times[0]) > 0:
return '{} hours, {} minutes'.format(times[0], times[1])
return '{} minutes'.format(times[1])
for item in items:
publish_time = item.get('firstpublished')
update_time = publish_time + timedelta(
seconds=item.get('time_to_next_update_publish'))
updated = '{} ({})'.format(
gen_update_string(item.get('time_to_next_update_publish')),
gen_date_str(update_time))
rows.append([
gen_date_str(publish_time),
item.get('slugline') or '',
item.get('headline') or '', updated
])
return {
'highcharts': [{
'id':
'update_time_report',
'type':
'table',
'title':
title,
'subtitle':
subtitle,
'headers': ['Published', 'Slugline', 'Headline', 'Updated In'],
'rows':
rows
}]
}
def gen_summary_chart():
chart = SDChart.Chart(
'mission_report_summary',
title='Mission Report Summary',
subtitle=ChartConfig.gen_subtitle_for_dates(params),
chart_type='highcharts',
height=300,
data_labels=False,
tooltip_header='{point.x}: {point.y}',
tooltip_point='',
default_config=ChartConfig.defaultConfig
)
chart.set_translation('summary', 'Summary', {
'total_stories': 'Total Stories',
'results': 'Results/Fields/Comment/Betting',
'new_stories': 'New Stories',
'rewrites': 'Updates',
'corrections': 'Corrections',
'kills': 'Kills',
'takedowns': 'Takedowns'
})
axis = chart.add_axis().set_options(
type='category',
default_chart_type='line',
y_title='Published Stories',
category_field='summary',
categories=[
'total_stories',
'new_stories',
'results',
'rewrites',
'corrections',
'kills',
'takedowns'
]
)
axis.add_series().set_options(
field='summary',
data=[
report['total_stories'],
report['new_stories']['count'],
report['new_stories']['categories']['results'],
len(report['rewrites']),
len(report['corrections']),
len(report['kills']),
len(report['takedowns'])
]
)
return chart.gen_config()
def gen_summary_chart():
chart = SDChart.Chart(
'mission_report_summary',
title='Mission Report Summary',
subtitle=ChartConfig.gen_subtitle_for_dates(params),
chart_type='highcharts',
height=300,
data_labels=False,
tooltip_header='{point.x}: {point.y}',
tooltip_point='',
default_config=ChartConfig.defaultConfig
)
chart.set_translation('summary', 'Summary', {
'total_stories': 'Total Stories',
'results': 'Results/Fields/Comment/Betting',
'new_stories': 'New Stories',
'rewrites': 'Updates',
'corrections': 'Corrections',
'kills': 'Kills',
'takedowns': 'Takedowns'
})
axis = chart.add_axis().set_options(
type='category',
default_chart_type='line',
y_title='Published Stories',
category_field='summary',
categories=[
'total_stories',
'new_stories',
'results',
'rewrites',
'corrections',
'kills',
'takedowns'
]
)
axis.add_series().set_options(
field='summary',
data=[
report['total_stories'],
report['new_stories']['count'],
report['new_stories']['categories']['results'],
report['rewrites'],
len(report['corrections']),
len(report['kills']),
len(report['takedowns'])
]
)
return chart.gen_config()
def _gen_stacked_chart(chart_id, chart_type):
chart = ChartConfig(chart_id, chart_type)
chart.add_source('anpa_category.qcode', {
'a': {1: 1, 3: 1},
'b': {1: 1, 3: 2},
'c': {1: 2, 3: 1, 5: 1},
})
chart.add_source('urgency', {1: 4, 3: 4, 5: 1})
return chart
def test_translate_source(self):
chart = ChartConfig('source', 'bar')
chart.add_source('source', {'aap': 3, 'ftp': 1, 'ap': 5})
self.assertEqual(chart.translations, {})
chart.load_translations()
self.assertEqual(chart.translations, {
'source': {
'title': 'Source',
'names': {}
}
})
def test_translate_authors(self):
chart = ChartConfig('author', 'bar')
chart.add_source('authors.parent', {'user1': 3, 'user2': 4, 'user3': 5})
self.assertEqual(chart.translations, {})
chart.load_translations()
self.assertEqual(chart.translations, {
'authors_parent': {
'title': 'Author',
'names': {
'user1': 'first user',
'user2': 'second user',
'user3': 'last user'
}
}
})
def test_translate_user(self):
chart = ChartConfig('user', 'bar')
chart.add_source('task.user', {'user1': 3, 'user2': 4, 'user3': 5})
self.assertEqual(chart.translations, {})
chart.load_translations()
self.assertEqual(chart.translations, {
'task_user': {
'title': 'User',
'names': {
'user1': 'first user',
'user2': 'second user',
'user3': 'last user'
}
}
})
def test_translate_desk(self):
chart = ChartConfig('desk', 'bar')
chart.add_source('task.desk', {'desk1': 4, 'desk2': 5, 'desk3': 1})
self.assertEqual(chart.translations, {})
chart.load_translations()
self.assertEqual(chart.translations, {
'task_desk': {
'title': 'Desk',
'names': {
'desk1': 'Politic Desk',
'desk2': 'Sports Desk',
'desk3': 'System Desk'
}
}
})
def test_translate_genre(self):
chart = ChartConfig('genre', 'bar')
chart.add_source('genre.qcode', {'Article': 4, 'Sidebar': 5, 'Factbox': 1})
self.assertEqual(chart.translations, {})
chart.load_translations()
self.assertEqual(chart.translations, {
'genre_qcode': {
'title': 'Genre',
'names': {
'Article': 'Article (news)',
'Sidebar': 'Sidebar',
'Factbox': 'Factbox'
}
}
})
def test_translate_anpa_category(self):
chart = ChartConfig('catrgory', 'bar')
chart.add_source('anpa_category.qcode', {'a': 3, 'b': 4, 'c': 1})
self.assertEqual(chart.translations, {})
chart.load_translations()
self.assertEqual(chart.translations, {
'anpa_category_qcode': {
'title': 'Category',
'names': {
'a': 'Advisories',
'b': 'Basketball',
'c': 'Cricket'
}
}
})
def test_translate_state(self):
chart = ChartConfig('state', 'bar')
chart.add_source('state', {'published': 3, 'killed': 1, 'updated': 5})
self.assertEqual(chart.translations, {})
chart.load_translations()
self.assertEqual(chart.translations, {
'state': {
'title': 'State',
'names': {
'published': 'Published',
'killed': 'Killed',
'corrected': 'Corrected',
'updated': 'Updated',
}
}
})
def test_translate_urgency(self):
chart = ChartConfig('urgency', 'bar')
chart.add_source('urgency', {1: 4, 3: 4, 5: 1})
self.assertEqual(chart.translations, {})
chart.load_translations()
self.assertEqual(chart.translations, {
'urgency': {
'title': 'Urgency',
'names': {
1: 1,
2: 2,
3: 3,
4: 4,
5: 5
}
}
})
def _gen_single_chart(chart_id, chart_type):
chart = ChartConfig(chart_id, chart_type)
chart.add_source('anpa_category.qcode', {'a': 3, 'b': 4, 'c': 1})
return chart
def generate_highcharts_config(self, docs, args):
params = args.get('params') or {}
aggs = args.get('aggs') or {}
group = aggs.get('group') or {}
subgroup = aggs.get('subgroup') or {}
translations = args.get('translations') or {}
chart = params.get('chart') or {}
chart_type = chart.get('type') or 'bar'
report = self.generate_report(docs, args)
chart_config = ChartConfig('content_publishing', chart_type)
chart_config.add_source(group.get('field'), report.get('groups'))
if report.get('subgroups'):
chart_config.add_source(subgroup.get('field'), report['subgroups'])
def gen_title():
if chart.get('title'):
return chart['title']
group_type = group.get('field')
group_title = chart_config.get_source_name(group_type)
if subgroup.get('field'):
subgroup_type = subgroup.get('field')
subgroup_title = chart_config.get_source_name(subgroup_type)
return 'Published Stories per {} with {} breakdown'.format(
group_title,
subgroup_title
)
return 'Published Stories per {}'.format(group_title)
def gen_subtitle():
return ChartConfig.gen_subtitle_for_dates(params)
chart_config.get_title = gen_title
chart_config.get_subtitle = gen_subtitle
chart_config.sort_order = chart.get('sort_order') or 'desc'
chart_config.translations = translations
report['highcharts'] = [chart_config.gen_config()]
return report
def generate_highcharts_config(self, docs, args):
params = args.get('params') or {}
report = self.generate_report(docs, args)
histogram = params.get('histogram') or {}
interval = histogram.get('interval') or 'daily'
point_interval = report.get('interval')
start_epoch = report.get('start_epoch')
with_sms = report.get('with_sms')
without_sms = report.get('without_sms')
chart_type = (params.get('chart') or {}).get('type') or 'column'
if (params.get('chart') or {}).get('title'):
title = params['chart']['title']
else:
if interval == 'hourly':
title = 'Hourly SMS Report'
elif interval == 'weekly':
title = 'Weekly SMS Report'
else:
title = 'Daily SMS Report'
if (params.get('chart') or {}).get('subtitle'):
subtitle = params['chart']['subtitle']
else:
subtitle = ChartConfig.gen_subtitle_for_dates(params)
# Calculate the UTC Offset in minutes for the start date of the results
# This will cause an issue if a report crosses over the daylight savings change
# Any data after the daylight savings change will be 1 hour out
timezone_offset = get_utc_offset_in_minutes(
datetime.utcfromtimestamp(int(report['start_epoch'] / 1000))
)
def gen_chart_config():
chart = SDChart.Chart(
'sms_report',
title=title,
subtitle=subtitle,
chart_type='highcharts',
start_of_week=app.config.get('START_OF_WEEK') or 0,
timezone_offset=timezone_offset,
use_utc=False
)
chart.set_translation('sms', 'SMS', {
'with_sms': 'With SMS',
'without_sms': 'Without SMS'
})
axis = chart.add_axis().set_options(
y_title='Published Stories',
x_title=chart.get_translation_title('sms'),
type='datetime',
default_chart_type=chart_type,
point_start=start_epoch,
point_interval=point_interval,
stack_labels=False
)
axis.add_series().set_options(
field='sms',
name='with_sms',
data=with_sms
)
axis.add_series().set_options(
field='sms',
name='without_sms',
data=without_sms
)
return chart.gen_config()
def gen_table_config():
date_header = 'Date'
if interval == 'hourly':
date_header = 'Date/Time'
elif interval == 'weekly':
date_header = 'Week Starting'
headers = [date_header, 'With SMS', 'Without SMS']
rows = []
totals = [0, 0]
date_format = '%b %-d %H:%M' if interval == 'hourly' else '%b %-d'
for index in range(0, len(with_sms)):
current_interval = datetime.strftime(
datetime.utcfromtimestamp(int(report['start_epoch'] / 1000)) + timedelta(
seconds=(index * (point_interval / 1000)),
minutes=timezone_offset
),
date_format
)
totals[0] += with_sms[index]
totals[1] += without_sms[index]
rows.append([
current_interval,
with_sms[index],
without_sms[index]
])
rows.append([
'Total',
totals[0],
totals[1]
])
return {
'id': 'sms_report',
'type': 'table',
'chart': {'type': 'column'},
'headers': headers,
'title': title,
'rows': rows
}
report['highcharts'] = [
gen_table_config() if chart_type == 'table' else gen_chart_config()
]
return report
def generate_highcharts_config(self, docs, args):
params = args.get('params') or {}
chart_params = params.get('chart') or {}
report = self.generate_report(docs, args)
stats = params.get('stats') or {}
desk_stats = report.get('desk_stats') or {}
desk_ids = list(desk_stats.keys())
stat_types = [
stat
for stat in ['sum', 'max', 'avg', 'min']
if stats.get(stat)
]
sort_order = chart_params.get('sort_order') or 'desc'
def get_sum_stats(desk_id):
return sum([
value
for stat, value in desk_stats[desk_id].items()
if stat in stat_types
])
sorted_desk_ids = [
desk_id
for desk_id in sorted(
desk_ids,
key=lambda kv: get_sum_stats(kv) if sort_order == 'asc'
else -get_sum_stats(kv)
)
]
if chart_params.get('subtitle'):
subtitle = chart_params['subtitle']
else:
subtitle = ChartConfig.gen_subtitle_for_dates(params)
chart = SDChart.Chart(
'production_time_report',
chart_type='highcharts',
title=chart_params.get('title') or 'Production Times',
subtitle=subtitle,
default_config=ChartConfig.defaultConfig,
colour_by_point=False,
data_labels=True,
legend_title='Production Time',
tooltip_header='{series.name}/{point.x}: {point.y}',
translations=ChartConfig.get_translations(['task.desk'])
)
chart.set_translation('production_stats', 'Production Stats', {
'min': 'Minimum',
'sum': 'Sum',
'avg': 'Average',
'max': 'Maximum'
})
axis = chart.add_axis().set_options(
type='category',
default_chart_type=chart_params.get('type') or 'column',
y_title='Seconds spent producing content',
category_field='task.desk',
categories=sorted_desk_ids,
stack_labels=False
)
def gen_data_entry(stat):
return {
'dataLabels': {
'enabled': True,
'format': seconds_to_human_readable(stat)
},
'y': stat
}
for stat_type in stat_types:
axis.add_series().set_options(
field='production_stats',
name=stat_type,
stack=0,
stack_type='normal',
data=[
gen_data_entry((desk_stats.get(desk_id) or {}).get(stat_type) or 0)
for desk_id in sorted_desk_ids
]
)
report['highcharts'] = [chart.gen_config()]
return report
def generate_highcharts_config(self, docs, args):
params = args.get('params') or {}
chart = params.get('chart') or {}
chart_type = chart.get('type') or 'bar'
report = self.generate_report(docs, args)
chart_config = ChartConfig('planning_usage', chart_type)
chart_config.add_source('task.user', report.get('group'))
chart_config.add_source('planning_usage', report.get('subgroup'))
def gen_title():
if chart.get('title'):
return chart['title']
return 'Planning Module Usage'
def gen_subtitle():
return ChartConfig.gen_subtitle_for_dates(params)
def get_y_axis_title():
return 'Items Created'
chart_config.get_title = gen_title
chart_config.get_subtitle = gen_subtitle
chart_config.get_y_axis_title = get_y_axis_title
chart_config.sort_order = chart.get('sort_order') or 'desc'
translations = args.get('translations') or {}
chart_config.translations = translations
report['highcharts'] = [chart_config.gen_config()]
return report
def generate_highcharts_config(self, docs, args):
params = args.get('params') or {}
report = self.generate_report(docs, args)
if (params.get('chart') or {}).get('title'):
title = params['chart']['title']
else:
interval = (params.get('histogram') or {}).get('interval') or 'daily'
if interval == 'hourly':
title = 'Hourly SMS Report'
elif interval == 'weekly':
title = 'Weekly SMS Report'
else:
title = 'Daily SMS Report'
if (params.get('chart') or {}).get('subtitle'):
subtitle = params['chart']['subtitle']
else:
subtitle = ChartConfig.gen_subtitle_for_dates(params)
# Calculate the UTC Offset in minutes for the start date of the results
# This will cause an issue if a report crosses over the daylight savings change
# Any data after the daylight savings change will be 1 hour out
timezone_offset = get_utc_offset_in_minutes(
datetime.utcfromtimestamp(int(report['start_epoch'] / 1000))
)
chart = SDChart.Chart(
'sms_report',
title=title,
subtitle=subtitle,
chart_type='highcharts',
start_of_week=app.config.get('START_OF_WEEK') or 0,
timezone_offset=timezone_offset,
use_utc=False
)
chart.set_translation('sms', 'SMS', {
'with_sms': 'With SMS',
'without_sms': 'Without SMS'
})
axis = chart.add_axis().set_options(
y_title='Published Stories',
x_title=chart.get_translation_title('sms'),
type='datetime',
default_chart_type=(params.get('chart') or {}).get('type') or 'column',
point_start=report.get('start_epoch'),
point_interval=report.get('interval'),
stack_labels=False
)
axis.add_series().set_options(
field='sms',
name='with_sms',
data=report.get('with_sms')
)
axis.add_series().set_options(
field='sms',
name='without_sms',
data=report.get('without_sms')
)
report['highcharts'] = [chart.gen_config()]
return report
def generate_highcharts_config(self, docs, args):
params = args.get('params') or {}
aggs = args.get('aggs') or {}
group = aggs.get('group') or {}
subgroup = aggs.get('subgroup') or {}
translations = args.get('translations') or {}
chart = params.get('chart') or {}
chart_type = chart.get('type') or 'bar'
report = self.generate_report(docs, args)
chart_config = ChartConfig('content_publishing', chart_type)
group_keys = list(report['groups'].keys())
if len(group_keys) == 1 and report.get('subgroups'):
chart_config.add_source(
subgroup.get('field'),
report['subgroups']
)
chart_config.load_translations(group.get('field'))
else:
chart_config.add_source(group.get('field'), report.get('groups'))
if report.get('subgroups'):
chart_config.add_source(subgroup.get('field'), report['subgroups'])
def gen_title():
if chart.get('title'):
return chart['title']
group_type = group.get('field')
group_title = chart_config.get_source_name(group_type)
if len(group_keys) == 1 and report.get('subgroups'):
data_name = chart_config.get_source_title(
group_type,
group_keys[0]
)
return 'Published Stories for {}: {}'.format(
group_title,
data_name
)
return 'Published Stories per {}'.format(group_title)
def gen_subtitle():
return ChartConfig.gen_subtitle_for_dates(params)
chart_config.get_title = gen_title
chart_config.get_subtitle = gen_subtitle
chart_config.sort_order = chart.get('sort_order') or 'desc'
chart_config.translations = translations
report['highcharts'] = [chart_config.gen_config()]
return report
def gen_subtitle():
return ChartConfig.gen_subtitle_for_dates(params)
def generate_highcharts_config(self, docs, args):
params = args.get('params') or {}
report = self.generate_report(docs, args)
if (params.get('chart') or {}).get('title'):
title = params['chart']['title']
else:
interval = (params.get('histogram')
or {}).get('interval') or 'daily'
if interval == 'hourly':
title = 'Hourly SMS Report'
elif interval == 'weekly':
title = 'Weekly SMS Report'
else:
title = 'Daily SMS Report'
if (params.get('chart') or {}).get('subtitle'):
subtitle = params['chart']['subtitle']
else:
subtitle = ChartConfig.gen_subtitle_for_dates(params)
# Calculate the UTC Offset in minutes for the start date of the results
# This will cause an issue if a report crosses over the daylight savings change
# Any data after the daylight savings change will be 1 hour out
timezone_offset = get_utc_offset_in_minutes(
datetime.utcfromtimestamp(int(report['start_epoch'] / 1000)))
chart = SDChart.Chart('sms_report',
title=title,
subtitle=subtitle,
chart_type='highcharts',
start_of_week=app.config.get('START_OF_WEEK')
or 0,
timezone_offset=timezone_offset,
use_utc=False)
chart.set_translation('sms', 'SMS', {
'with_sms': 'With SMS',
'without_sms': 'Without SMS'
})
axis = chart.add_axis().set_options(
y_title='Published Stories',
x_title=chart.get_translation_title('sms'),
type='datetime',
default_chart_type=(params.get('chart') or {}).get('type')
or 'column',
point_start=report.get('start_epoch'),
point_interval=report.get('interval'),
stack_labels=False)
axis.add_series().set_options(field='sms',
name='with_sms',
data=report.get('with_sms'))
axis.add_series().set_options(field='sms',
name='without_sms',
data=report.get('without_sms'))
report['highcharts'] = [chart.gen_config()]
return report