def test_delete_master_deletes_linked(self):
linked_report_info = create_linked_ucr(self.domain_link, self.report.get_id)
soft_delete(self.report)
update_linked_ucr(self.domain_link, linked_report_info.report.get_id)
report = ReportConfiguration.get(linked_report_info.report.get_id)
self.assertTrue(is_deleted(report))
self.report.config.deactivate()
update_linked_ucr(self.domain_link, linked_report_info.report.get_id)
report = ReportConfiguration.get(linked_report_info.report.get_id)
self.assertTrue(report.config.is_deactivated)
def update_linked_ucr(domain_link, report_id):
linked_report = ReportConfiguration.get(report_id)
linked_datasource = linked_report.config
if domain_link.is_remote:
remote_configs = remote_get_ucr_config(domain_link, report_id)
master_report = remote_configs["report"]
master_datasource = remote_configs["datasource"]
else:
master_report = ReportConfiguration.get(linked_report.report_meta.master_id)
master_datasource = master_report.config
_update_linked_datasource(master_datasource, linked_datasource)
_update_linked_report(master_report, linked_report)
def test_remote_link_ucr(self, fake_ucr_getter):
create_domain(self.domain)
self.addCleanup(delete_all_domains)
couch_user = WebUser.create(self.domain, "test", "foobar", None, None)
django_user = couch_user.get_django_user()
self.addCleanup(delete_all_users)
api_key, _ = HQApiKey.objects.get_or_create(user=django_user)
auth_headers = {'HTTP_AUTHORIZATION': 'apikey test:%s' % api_key.key}
self.domain_link.save()
url = reverse('linked_domain:ucr_config', args=[self.domain, self.report.get_id])
headers = auth_headers.copy()
headers[REMOTE_REQUESTER_HEADER] = self.domain_link.linked_domain
resp = self.client.get(url, **headers)
fake_ucr_getter.return_value = json.loads(resp.content)
# Create
linked_report_info = create_linked_ucr(self.domain_link, self.report.get_id)
self.assertEqual(1, len(ReportConfiguration.by_domain(self.domain_link.linked_domain)))
self.assertEqual(self.report.get_id, linked_report_info.report.report_meta.master_id)
# Update
self.report.title = "Another new title"
self.report.save()
update_linked_ucr(self.domain_link, linked_report_info.report.get_id)
report = ReportConfiguration.get(linked_report_info.report.get_id)
self.assertEqual("Another new title", report.title)
def pop_report(report_id, reports):
report = reports.pop(report_id, None)
if report is None:
report = ReportConfiguration.get(report_id)
if report.doc_type == "ReportConfiguration-Deleted":
return None
return report
def _make_model_cell(self, record):
name = LINKED_MODELS_MAP[record.model]
if record.model == MODEL_APP:
detail = record.wrapped_detail
app_name = ugettext_lazy('Unknown App')
if detail:
app_names = self.linked_app_names(self.selected_link.linked_domain)
app_name = app_names.get(detail.app_id, detail.app_id)
return '{} ({})'.format(name, app_name)
if record.model == MODEL_FIXTURE:
detail = record.wrapped_detail
tag = ugettext_lazy('Unknown')
if detail:
data_type = get_fixture_data_type_by_tag(self.selected_link.linked_domain, detail.tag)
if data_type:
tag = data_type.tag
return '{} ({})'.format(name, tag)
if record.model == MODEL_REPORT:
detail = record.wrapped_detail
report_name = ugettext_lazy('Unknown Report')
if detail:
try:
report_name = ReportConfiguration.get(detail.report_id).title
except ResourceNotFound:
pass
return '{} ({})'.format(name, report_name)
return name
def test_link_creates_datasource_and_report(self):
link_info = create_linked_ucr(self.domain_link, self.report.get_id)
new_datasource = DataSourceConfiguration.get(link_info.datasource.get_id)
self.assertEqual(new_datasource.domain, self.domain_link.linked_domain)
new_report = ReportConfiguration.get(link_info.report.get_id)
self.assertEqual(new_report.domain, self.domain_link.linked_domain)
self.assertEqual(self.report.get_id, new_report.report_meta.master_id)
def ucr_config(request, domain, config_id):
report_config = ReportConfiguration.get(config_id)
datasource_id = report_config.config_id
datasource_config = DataSourceConfiguration.get(datasource_id)
return JsonResponse({
"report": report_config.to_json(),
"datasource": datasource_config.to_json(),
})
def pop_report_for_action(action, reports):
report_id = action.wrapped_detail.report_id
try:
report = reports.get(report_id)
del reports[report_id]
except KeyError:
report = ReportConfiguration.get(report_id)
return report
def test_update_ucr(self):
linked_report_info = create_linked_ucr(self.domain_link, self.report.get_id)
self.report.title = "New title"
self.report.save()
update_linked_ucr(self.domain_link, linked_report_info.report.get_id)
report = ReportConfiguration.get(linked_report_info.report.get_id)
self.assertEqual("New title", report.title)
self.assertEqual(self.report.get_id, report.report_meta.master_id)
self.assertNotEqual(self.report.config_id, report.config_id)
def create_linked_ucr(domain_link, report_config_id):
if domain_link.is_remote:
remote_configs = remote_get_ucr_config(domain_link, report_config_id)
datasource = remote_configs["datasource"]
report_config = remote_configs["report"]
else:
report_config = ReportConfiguration.get(report_config_id)
datasource = DataSourceConfiguration.get(report_config.config_id)
new_datasource = _get_or_create_datasource_link(domain_link, datasource)
new_report = _get_or_create_report_link(domain_link, report_config, new_datasource)
return LinkedUCRInfo(datasource=new_datasource, report=new_report)
def test_linked_report_with_app_id_is_not_updated(self):
self.linked_report.config.meta.build.app_id = self.linked_app._id
self.linked_report.save()
self.assertEqual(self.linked_app._id,
self.linked_report.config.meta.build.app_id)
num_of_failed_attempts = migrate_linked_reports()
self.assertEqual(0, num_of_failed_attempts)
# refetch
actual_report = ReportConfiguration.get(self.linked_report._id)
self.assertEqual(self.linked_app._id,
actual_report.config.meta.build.app_id)
def _get_master_model_status(self,
apps,
fixtures,
reports,
ignore_models=None):
model_status = []
ignore_models = ignore_models or []
for model, name in LINKED_MODELS:
if (model not in ignore_models
and model not in (MODEL_APP, MODEL_FIXTURE, MODEL_REPORT)
and
(model != MODEL_CASE_SEARCH
or toggles.SYNC_SEARCH_CASE_CLAIM.enabled(self.domain))):
model_status.append({
'type': model,
'name': name,
'last_update': ugettext('Never'),
'detail': None,
'can_update': True
})
linked_models = dict(LINKED_MODELS)
for app in apps.values():
update = {
'type': MODEL_APP,
'name': '{} ({})'.format(linked_models['app'], app.name),
'last_update': None,
'detail': AppLinkDetail(app_id=app._id).to_json(),
'can_update': True
}
model_status.append(update)
for fixture in fixtures.values():
update = {
'type': MODEL_FIXTURE,
'name': '{} ({})'.format(linked_models['fixture'],
fixture.tag),
'last_update': None,
'detail': FixtureLinkDetail(tag=fixture.tag).to_json(),
'can_update': fixture.is_global,
}
model_status.append(update)
for report in reports.values():
report = ReportConfiguration.get(report.get_id)
update = {
'type': MODEL_REPORT,
'name': f"{linked_models['report']} ({report.title})",
'last_update': None,
'detail': ReportLinkDetail(report_id=report.get_id).to_json(),
'can_update': True,
}
model_status.append(update)
return model_status
def _get_model_status(self, master_link, apps, reports):
model_status = []
if not master_link:
return model_status
models_seen = set()
history = DomainLinkHistory.objects.filter(link=master_link).annotate(row_number=RawSQL(
'row_number() OVER (PARTITION BY model, model_detail ORDER BY date DESC)',
[]
))
linked_models = dict(LINKED_MODELS)
timezone = get_timezone_for_request()
for action in history:
models_seen.add(action.model)
if action.row_number != 1:
# first row is the most recent
continue
name = linked_models[action.model]
update = {
'type': action.model,
'name': name,
'last_update': server_to_user_time(action.date, timezone),
'detail': action.model_detail,
'can_update': True
}
if action.model == 'app':
app_name = ugettext('Unknown App')
if action.model_detail:
detail = action.wrapped_detail
app = apps.pop(detail.app_id, None)
app_name = app.name if app else detail.app_id
if app:
update['detail'] = action.model_detail
else:
update['can_update'] = False
else:
update['can_update'] = False
update['name'] = '{} ({})'.format(name, app_name)
model_status.append(update)
if action.model == 'report':
report_id = action.wrapped_detail.report_id
try:
report = reports.get(report_id)
del reports[report_id]
except KeyError:
report = ReportConfiguration.get(report_id)
update['name'] = f'{name} ({report.title})'
# Add in models and apps that have never been synced
model_status.extend(self._get_master_model_status(apps, reports, ignore_models=models_seen))
return model_status
def test_app_id_for_linked_reports_updates_given_family_id(self):
self.linked_report.config.meta.build.app_id = None
self.linked_report.save()
self.assertIsNone(self.linked_report.config.meta.build.app_id)
self.linked_app.family_id = self.original_app._id
self.linked_app.upstream_app_id = None
self.linked_app.save()
num_of_failed_attempts = migrate_linked_reports()
self.assertEqual(0, num_of_failed_attempts)
# refetch
actual_report = ReportConfiguration.get(self.linked_report._id)
self.assertEqual(self.linked_app._id,
actual_report.config.meta.build.app_id)
def migrate_linked_reports(upstream_domain=None):
logger.setLevel(logging.INFO)
if upstream_domain:
domain_links = DomainLink.all_objects.filter(
master_domain=upstream_domain)
else:
domain_links = DomainLink.all_objects.all()
num_of_failed_attempts = 0
for domain_link in domain_links:
reports = get_report_configs_for_domain(domain_link.linked_domain)
for report in reports:
if report.report_meta.master_id and not report.config.meta.build.app_id:
upstream_report = ReportConfiguration.get(
report.report_meta.master_id)
upstream_datasource = DataSourceConfiguration.get(
upstream_report.config_id)
downstream_app_id = get_downstream_app_id(
domain_link.linked_domain,
upstream_datasource.meta.build.app_id,
)
if not downstream_app_id:
# just as a backup in case upstream_app_id is not set but family_id is
downstream_app_id = get_downstream_app_id(
domain_link.linked_domain,
upstream_datasource.meta.build.app_id,
use_upstream_app_id=False)
if downstream_app_id:
logger.info(
f"Needed to use family_id to find downstream app {downstream_app_id}"
)
if not downstream_app_id:
logger.warning(
f"Could not find downstream_app_id for upstream app"
f" {upstream_datasource.meta.build.app_id} "
f"in downstream domain {domain_link.linked_domain}")
num_of_failed_attempts += 1
report.config.meta.build.app_id = downstream_app_id
report.config.save()
logger.info(
f"Completed linked report migration with {num_of_failed_attempts} failed attempts"
)
return num_of_failed_attempts
def create_linked_ucr(domain_link, report_config_id):
if domain_link.is_remote:
remote_configs = remote_get_ucr_config(domain_link, report_config_id)
datasource = remote_configs["datasource"]
report_config = remote_configs["report"]
else:
report_config = ReportConfiguration.get(report_config_id)
datasource = DataSourceConfiguration.get(report_config.config_id)
# grab the linked app this linked report references
try:
downstream_app_id = get_downstream_app_id(domain_link.linked_domain, datasource.meta.build.app_id)
except MultipleDownstreamAppsError:
raise DomainLinkError(_("This report cannot be linked because it references an app that has multiple "
"downstream apps."))
new_datasource = _get_or_create_datasource_link(domain_link, datasource, downstream_app_id)
new_report = _get_or_create_report_link(domain_link, report_config, new_datasource)
return LinkedUCRInfo(datasource=new_datasource, report=new_report)
def export_table(self):
try:
data = self.data_source
data.set_filter_values(self.filter_values)
except UserReportsError as e:
return self.render_json_response({
'error': e.message,
})
report_config = ReportConfiguration.get(self.report_config_id)
raw_rows = list(data.get_data())
headers = [column['display'] for column in report_config.columns]
columns = [column['field'] for column in report_config.columns]
rows = [[raw_row[column] for column in columns] for raw_row in raw_rows]
return [
[
self.title,
[headers] + rows
]
]
def _safely_get_report_configs(project_name):
try:
configs = ReportConfiguration.by_domain(project_name)
except BadSpecError as e:
logging.exception(e)
# Pick out the UCRs that don't have spec errors
configs = []
for config_id in get_doc_ids_in_domain_by_class(project_name, ReportConfiguration):
try:
configs.append(ReportConfiguration.get(config_id))
except BadSpecError as e:
logging.error("%s with report config %s" % (e.message, config_id))
try:
configs.extend(StaticReportConfiguration.by_domain(project_name))
except BadSpecError as e:
logging.exception(e)
return configs
def _release_report(self, domain_link, model):
report_id = model['detail']['report_id']
found = False
for linked_report in get_report_configs_for_domain(domain_link.linked_domain):
if linked_report.report_meta.master_id == report_id:
found = True
update_linked_ucr(domain_link, linked_report.get_id)
if not found:
report = ReportConfiguration.get(report_id)
if report.report_meta.created_by_builder:
view = 'edit_report_in_builder'
else:
view = 'edit_configurable_report'
url = get_url_base() + reverse(view, args=[domain_link.master_domain, report_id])
return self._error_tuple(
_('Could not find report. <a href="{}">Click here</a> and click "Link Report" to link this '
+ 'report.').format(url),
text=_('Could not find report. Please check that the report has been linked.'),
)
def _safely_get_report_configs(project_name):
try:
configs = ReportConfiguration.by_domain(project_name)
except BadSpecError as e:
logging.exception(e)
# Pick out the UCRs that don't have spec errors
configs = []
for config_id in get_doc_ids_in_domain_by_class(project_name, ReportConfiguration):
try:
configs.append(ReportConfiguration.get(config_id))
except BadSpecError as e:
logging.error("%s with report config %s" % (e.message, config_id))
try:
configs.extend(StaticReportConfiguration.by_domain(project_name))
except BadSpecError as e:
logging.exception(e)
return configs
def _make_model_cell(self, record):
name = LINKED_MODELS_MAP[record.model]
if record.model == 'app':
detail = record.wrapped_detail
app_name = ugettext_lazy('Unknown App')
if detail:
app_names = self.linked_app_names(self.selected_link.linked_domain)
app_name = app_names.get(detail.app_id, detail.app_id)
return '{} ({})'.format(name, app_name)
if record.model == 'report':
detail = record.wrapped_detail
report_name = ugettext_lazy('Unknown Report')
if detail:
try:
report_name = ReportConfiguration.get(detail.report_id).title
except ResourceNotFound:
pass
return '{} ({})'.format(name, report_name)
return name
def export_table(self):
try:
data = self.data_source
data.set_filter_values(self.filter_values)
data.set_order_by([(o['field'], o['order']) for o in self.spec.sort_expression])
except UserReportsError as e:
return self.render_json_response({
'error': e.message,
})
report_config = ReportConfiguration.get(self.report_config_id)
raw_rows = list(data.get_data())
headers = [column.header for column in self.data_source.columns]
columns = [column.column_id for column in report_config.report_columns]
rows = [[raw_row[column] for column in columns] for raw_row in raw_rows]
return [
[
self.title,
[headers] + rows
]
]
def test_remote_link_ucr(self, fake_ucr_getter):
fake_ucr_getter.return_value = {
"report": self.report,
"datasource": self.data_source,
}
# Create
linked_report_info = create_linked_ucr(self.domain_link,
self.report.get_id)
self.assertEqual(
1,
len(ReportConfiguration.by_domain(self.domain_link.linked_domain)))
self.assertEqual(self.report.get_id,
linked_report_info.report.report_meta.master_id)
# Update
self.report.title = "Another new title"
self.report.save()
update_linked_ucr(self.domain_link, linked_report_info.report.get_id)
report = ReportConfiguration.get(linked_report_info.report.get_id)
self.assertEqual("Another new title", report.title)
def _get_model_status(self, master_link, apps, fixtures, reports,
keywords):
model_status = []
if not master_link:
return model_status
models_seen = set()
history = DomainLinkHistory.objects.filter(
link=master_link
).annotate(row_number=RawSQL(
'row_number() OVER (PARTITION BY model, model_detail ORDER BY date DESC)',
[]))
linked_models = dict(LINKED_MODELS)
timezone = get_timezone_for_request()
for action in history:
models_seen.add(action.model)
if action.row_number != 1:
# first row is the most recent
continue
name = linked_models[action.model]
update = {
'type': action.model,
'name': name,
'last_update': server_to_user_time(action.date, timezone),
'detail': action.model_detail,
'can_update': True
}
if action.model == 'app':
app_name = ugettext('Unknown App')
if action.model_detail:
detail = action.wrapped_detail
app = apps.pop(detail.app_id, None)
app_name = app.name if app else detail.app_id
if app:
update['detail'] = action.model_detail
else:
update['can_update'] = False
else:
update['can_update'] = False
update['name'] = '{} ({})'.format(name, app_name)
if action.model == 'fixture':
tag_name = ugettext('Unknown Table')
can_update = False
if action.model_detail:
detail = action.wrapped_detail
tag = action.wrapped_detail.tag
try:
fixture = fixtures.get(tag)
del fixtures[tag]
except KeyError:
fixture = get_fixture_data_type_by_tag(
self.domain, tag)
tag_name = fixture.tag
can_update = fixture.is_global
update['name'] = f'{name} ({tag_name})'
update['can_update'] = can_update
if action.model == 'report':
report_id = action.wrapped_detail.report_id
try:
report = reports.get(report_id)
del reports[report_id]
except KeyError:
report = ReportConfiguration.get(report_id)
update['name'] = f'{name} ({report.title})'
if action.model == 'keyword':
keyword_id = action.wrapped_detail.linked_keyword_id
try:
keyword = keywords[keyword_id].keyword
del keywords[keyword_id]
except KeyError:
try:
keyword = Keyword.objects.get(id=keyword_id).keyword
except Keyword.DoesNotExist:
keyword = ugettext_lazy("Deleted Keyword")
update['can_update'] = False
update['name'] = f'{name} ({keyword})'
model_status.append(update)
# Add in models and apps that have never been synced
model_status.extend(
self._get_master_model_status(apps,
fixtures,
reports,
keywords,
ignore_models=models_seen))
return model_status
def _get_master_model_status(self,
apps,
fixtures,
reports,
keywords,
ignore_models=None):
model_status = []
ignore_models = ignore_models or []
for model, name in LINKED_MODELS:
if (model not in ignore_models
and model not in (MODEL_APP, MODEL_FIXTURE, MODEL_REPORT,
MODEL_KEYWORD) and
(model != MODEL_CASE_SEARCH
or toggles.SYNC_SEARCH_CASE_CLAIM.enabled(self.domain))
and (model != MODEL_DATA_DICTIONARY
or toggles.DATA_DICTIONARY.enabled(self.domain))
and (model != MODEL_DIALER_SETTINGS
or toggles.WIDGET_DIALER.enabled(self.domain))
and (model != MODEL_OTP_SETTINGS
or toggles.GAEN_OTP_SERVER.enabled(self.domain))
and (model != MODEL_HMAC_CALLOUT_SETTINGS
or toggles.HMAC_CALLOUT.enabled(self.domain))):
model_status.append({
'type': model,
'name': name,
'last_update': ugettext('Never'),
'detail': None,
'can_update': True
})
linked_models = dict(LINKED_MODELS)
for app in apps.values():
update = {
'type': MODEL_APP,
'name': '{} ({})'.format(linked_models['app'], app.name),
'last_update': None,
'detail': AppLinkDetail(app_id=app._id).to_json(),
'can_update': True
}
model_status.append(update)
for fixture in fixtures.values():
update = {
'type': MODEL_FIXTURE,
'name': '{} ({})'.format(linked_models['fixture'],
fixture.tag),
'last_update': None,
'detail': FixtureLinkDetail(tag=fixture.tag).to_json(),
'can_update': fixture.is_global,
}
model_status.append(update)
for report in reports.values():
report = ReportConfiguration.get(report.get_id)
update = {
'type': MODEL_REPORT,
'name': f"{linked_models['report']} ({report.title})",
'last_update': None,
'detail': ReportLinkDetail(report_id=report.get_id).to_json(),
'can_update': True,
}
model_status.append(update)
for keyword in keywords.values():
update = {
'type': MODEL_KEYWORD,
'name': f"{linked_models['keyword']} ({keyword.keyword})",
'last_update': None,
'detail':
KeywordLinkDetail(keyword_id=str(keyword.id)).to_json(),
'can_update': True,
}
model_status.append(update)
return model_status
def _report_config_to_fixture(report_config, user):
report_elem = E.report(id=report_config.uuid)
try:
report = ReportConfiguration.get(report_config.report_id)
except ResourceNotFound as err:
# ReportConfiguration not found
raise BadReportConfigurationError('Error getting ReportConfiguration with ID "{}": {}'.format(
report_config.report_id, err))
data_source = ReportFactory.from_spec(report)
all_filter_values = {
filter_slug: filter.get_filter_value(user, report.get_ui_filter(filter_slug))
for filter_slug, filter in report_config.filters.items()
}
filter_values = {
filter_slug: filter_value for filter_slug, filter_value in all_filter_values.items()
if filter_value is not None
}
defer_filters = {
filter_slug: report.get_ui_filter(filter_slug)
for filter_slug, filter_value in all_filter_values.items()
if filter_value is None
}
data_source.set_filter_values(filter_values)
data_source.defer_filters(defer_filters)
rows_elem = E.rows()
deferred_fields = {ui_filter.field for ui_filter in defer_filters.values()}
filter_options_by_field = defaultdict(set)
def _row_to_row_elem(row, index, is_total_row=False):
row_elem = E.row(index=str(index), is_total_row=str(is_total_row))
for k in sorted(row.keys()):
value = serialize(row[k])
row_elem.append(E.column(value, id=k))
if not is_total_row and k in deferred_fields:
filter_options_by_field[k].add(value)
return row_elem
for i, row in enumerate(data_source.get_data()):
rows_elem.append(_row_to_row_elem(row, i))
if data_source.has_total_row:
total_row = data_source.get_total_row()
rows_elem.append(_row_to_row_elem(
dict(
zip(
map(lambda column_config: column_config.column_id, data_source.column_configs),
map(str, total_row)
)
),
data_source.get_total_records(),
is_total_row=True,
))
filters_elem = E.filters()
for filter_slug, ui_filter in defer_filters.items():
# @field is maybe a bad name for this attribute,
# since it's actually the filter slug
filter_elem = E.filter(field=filter_slug)
option_values = filter_options_by_field[ui_filter.field]
choices = ui_filter.choice_provider.get_sorted_choices_for_values(option_values)
for choice in choices:
# add the correct text from ui_filter.choice_provider
option_elem = E.option(choice.display, value=choice.value)
filter_elem.append(option_elem)
filters_elem.append(filter_elem)
report_elem.append(filters_elem)
report_elem.append(rows_elem)
return report_elem
