def release(self, models, linked_domains, build_apps=False):
self._reset(models, linked_domains)
domain_links_by_linked_domain = {
link.linked_domain: link for link in get_linked_domains(self.master_domain)
}
for linked_domain in self.linked_domains:
if linked_domain not in domain_links_by_linked_domain:
self._add_error(linked_domain, _("Project space {} is no longer linked to {}. No content "
"was released to it.").format(self.master_domain, linked_domain))
continue
domain_link = domain_links_by_linked_domain[linked_domain]
for model in self.models:
errors = None
try:
if model['type'] == MODEL_APP:
errors = self._release_app(domain_link, model, self.user, build_apps)
elif model['type'] == MODEL_REPORT:
errors = self._release_report(domain_link, model)
elif model['type'] == MODEL_CASE_SEARCH:
errors = self._release_case_search(domain_link, model, self.user)
else:
errors = self._release_model(domain_link, model, self.user)
except Exception as e: # intentionally broad
errors = [str(e), str(e)]
notify_exception(None, "Exception pushing linked domains: {}".format(e))
if errors:
self._add_error(
linked_domain,
_("Could not update {}: {}").format(model['name'], errors[0]),
text=_("Could not update {}: {}").format(model['name'], errors[1]))
else:
self._add_success(linked_domain, _("Updated {} successfully").format(model['name']))
def page_context(self):
"""
This view services both domains that are master domains and domains that are linked domains
(and legacy domains that are both).
"""
timezone = get_timezone_for_request()
master_link = get_domain_master_link(self.domain)
linked_domains = [self._link_context(link, timezone) for link in get_linked_domains(self.domain)]
(master_apps, linked_apps) = self._get_apps()
(master_reports, linked_reports) = self._get_reports()
# Models belonging to this domain's master domain, for the purpose of pulling
model_status = self._get_model_status(master_link, linked_apps, linked_reports)
# Models belonging to this domain, for the purpose of pushing to linked domains
master_model_status = self._get_master_model_status(master_apps, master_reports)
return {
'domain': self.domain,
'timezone': timezone.localize(datetime.utcnow()).tzname(),
'is_linked_domain': bool(master_link),
'is_master_domain': bool(len(linked_domains)),
'view_data': {
'master_link': self._link_context(master_link, timezone) if master_link else None,
'model_status': sorted(model_status, key=lambda m: m['name']),
'master_model_status': sorted(master_model_status, key=lambda m: m['name']),
'linked_domains': linked_domains,
'models': [
{'slug': model[0], 'name': model[1]}
for model in LINKED_MODELS
]
},
}
def handle(self, domain, case_type, **options):
domains = {domain}
if options["and_linked"]:
domains = domains | {
link.linked_domain
for link in get_linked_domains(domain)
}
if options["username"]:
user_id = username_to_user_id(options["username"])
if not user_id:
raise Exception("The username you entered is invalid")
else:
user_id = SYSTEM_USER_ID
self.output_file = options["output_file"]
options.pop("and_linked")
options.pop("username")
options.pop("output_file", None)
self.extra_options = options
for domain in sorted(domains):
print(f"Processing {domain}")
self.update_cases(domain, case_type, user_id)
def page_context(self):
context = self.pagination_context
context['linked_domains'] = [
domain_link.linked_domain
for domain_link in get_linked_domains(self.domain)
]
context['linkable_keywords'] = self._linkable_keywords()
return context
def linked_downstream_reports_by_domain(master_domain, report_id):
"""A dict of all downstream domains with and if this is already linked to `report_id`
"""
from corehq.apps.linked_domain.dbaccessors import get_linked_domains
linked_domains = {}
for domain_link in get_linked_domains(master_domain):
linked_domains[domain_link.linked_domain] = any(
r for r in get_linked_report_configs(domain_link.linked_domain, report_id)
)
return linked_domains
def handle(self, domain, case_type, username, **options):
domains = {domain}
if options["and_linked"]:
domains = domains | {
link.linked_domain
for link in get_linked_domains(domain)
}
for domain in domains:
print(f"Processing {domain}")
update_cases(domain, case_type, username)
def handle(self, domain, case_type, **options):
domains = {domain}
if options["and_linked"]:
domains = domains | {
link.linked_domain
for link in get_linked_domains(domain)
}
if options["username"]:
user_id = username_to_user_id(options["username"])
if not user_id:
raise Exception("The username you entered is invalid")
else:
user_id = SYSTEM_USER_ID
self.location = options.get('location')
for domain in domains:
print(f"Processing {domain}")
self.update_cases(domain, case_type, user_id)
def handle(self, domain, case_type, **options):
# logger.debug will record something to a file but not print it
self.logger = logging.getLogger(self.logger_name)
if not settings.UNIT_TESTING:
self.logger.addHandler(
logging.FileHandler(self.logger_name.split(".")[-1] + ".txt"))
self.logger.setLevel(logging.DEBUG)
self.logger.debug(
f"{datetime.datetime.utcnow()} Starting run: {options}")
domains = {domain}
if options.pop("and_linked"):
domains = domains | {
link.linked_domain
for link in get_linked_domains(domain)
}
username = options.pop("username")
self.case_type = case_type
self.extra_options = options
for i, domain in enumerate(sorted(domains), start=1):
case_ids = self.find_case_ids(domain)
self.logger.debug(
f"Found {len(case_ids)} cases in {domain} ({i}/{len(domains)})"
)
update_count = update_cases(
domain=domain,
update_fn=self.case_blocks,
case_ids=with_progress_bar(case_ids, oneline=False),
form_meta=SystemFormMeta.for_script(self.logger_name,
username),
)
self.logger.debug(
f"Made {update_count} updates in {domain} ({i}/{len(domains)})"
)
self.logger.debug(f"{datetime.datetime.utcnow()} Script complete")
def options(self):
links = get_linked_domains(self.domain)
return [
(str(link.id), link.linked_domain)
for link in links
]
def page_context(self):
timezone = get_timezone_for_request()
def _link_context(link, timezone=timezone):
return {
'linked_domain':
link.linked_domain,
'master_domain':
link.master_domain,
'remote_base_url':
link.remote_base_url,
'remote_username':
link.remote_username,
'remote_api_key':
link.remote_api_key,
'is_remote':
link.is_remote,
'last_update':
server_to_user_time(link.last_pull, timezone)
if link.last_pull else 'Never',
}
model_status = []
linked_models = dict(LINKED_MODELS)
master_link = get_domain_master_link(self.domain)
if master_link:
linked_apps = {
app._id: app
for app in get_brief_apps_in_domain(self.domain)
if app.doc_type == 'LinkedApplication'
}
models_seen = set()
history = DomainLinkHistory.objects.filter(
link=master_link
).annotate(row_number=RawSQL(
'row_number() OVER (PARTITION BY model, model_detail ORDER BY date DESC)',
[]))
for action in history:
models_seen.add(action.model)
if action.row_number != 1:
# first row is the most recent
continue
name = linked_models[action.model]
update = {
'type': action.model,
'name': name,
'last_update': server_to_user_time(action.date, timezone),
'detail': action.model_detail,
'can_update': True
}
if action.model == 'app':
app_name = 'Unknown App'
if action.model_detail:
detail = action.wrapped_detail
app = linked_apps.pop(detail.app_id, None)
app_name = app.name if app else detail.app_id
if app:
update['detail'] = action.model_detail
else:
update['can_update'] = False
else:
update['can_update'] = False
update['name'] = '{} ({})'.format(name, app_name)
model_status.append(update)
# Add in models that have never been synced
for model, name in LINKED_MODELS:
if model not in models_seen and model != 'app':
model_status.append({
'type': model,
'name': name,
'last_update': ugettext('Never'),
'detail': None,
'can_update': True
})
# Add in apps that have never been synced
if linked_apps:
for app in linked_apps.values():
update = {
'type': 'app',
'name': '{} ({})'.format(linked_models['app'],
app.name),
'last_update': None,
'detail': AppLinkDetail(app_id=app._id).to_json(),
'can_update': True
}
model_status.append(update)
return {
'domain': self.domain,
'timezone': timezone.localize(datetime.utcnow()).tzname(),
'view_data': {
'master_link':
_link_context(master_link) if master_link else None,
'model_status':
sorted(model_status, key=lambda m: m['name']),
'linked_domains': [
_link_context(link)
for link in get_linked_domains(self.domain)
],
'models': [{
'slug': model[0],
'name': model[1]
} for model in LINKED_MODELS]
},
}
def push_models(master_domain, models, linked_domains, build_apps, username):
domain_links_by_linked_domain = {
link.linked_domain: link
for link in get_linked_domains(master_domain)
}
user = CouchUser.get_by_username(username)
errors_by_domain = defaultdict(list)
successes_by_domain = defaultdict(list)
for linked_domain in linked_domains:
if linked_domain not in domain_links_by_linked_domain:
errors_by_domain[linked_domain].append(
_("Project space {} is no longer linked to {}. No content "
"was released to it.").format(master_domain, linked_domain))
continue
domain_link = domain_links_by_linked_domain[linked_domain]
for model in models:
try:
found = False
updated_app = False
built_app = False
if model['type'] == MODEL_APP:
app_id = model['detail']['app_id']
for linked_app in get_apps_in_domain(linked_domain,
include_remote=False):
if is_linked_app(
linked_app) and linked_app.family_id == app_id:
found = True
if toggles.MULTI_MASTER_LINKED_DOMAINS.enabled(
linked_domain):
errors_by_domain[linked_domain].append(
textwrap.dedent(
_("""
Could not update {} because multi master flag is in use
""".strip()).format(model['name'])))
continue
app = update_linked_app(linked_app, app_id,
user.user_id)
updated_app = True
if build_apps:
build = app.make_build()
build.is_released = True
build.save(increment_version=False)
built_app = True
elif model['type'] == MODEL_REPORT:
report_id = model['detail']['report_id']
for linked_report in get_report_configs_for_domain(
linked_domain):
if linked_report.report_meta.master_id == report_id:
found = True
update_linked_ucr(domain_link,
linked_report.get_id)
elif (model['type'] == MODEL_CASE_SEARCH and
not toggles.SYNC_SEARCH_CASE_CLAIM.enabled(linked_domain)
):
errors_by_domain[linked_domain].append(
textwrap.dedent(
_("""
Could not update {} because case claim flag is not on
""".strip()).format(model['name'])))
continue
else:
found = True
update_model_type(domain_link,
model['type'],
model_detail=model['detail'])
domain_link.update_last_pull(model['type'],
user._id,
model_details=model['detail'])
if found:
successes_by_domain[linked_domain].append(
_("{} was updated").format(model['name']))
else:
errors_by_domain[linked_domain].append(
_("Could not find {}").format(model['name']))
except Exception as e: # intentionally broad
if model[
'type'] == MODEL_APP and updated_app and build_apps and not built_app:
# Updating an app can be a 2-step process, make it clear which one failed
errors_by_domain[linked_domain].append(
textwrap.dedent(
_("""
Updated {} but could not make and release build: {}
""".strip()).format(model['name'], str(e))))
else:
errors_by_domain[linked_domain].append(
textwrap.dedent(
_("""
Could not update {}: {}
""".strip()).format(model['name'], str(e))))
notify_exception(
None, "Exception pushing linked domains: {}".format(e))
subject = _("Linked project release complete.")
if errors_by_domain:
subject += _(" Errors occurred.")
error_domain_count = len(errors_by_domain)
success_domain_count = len(linked_domains) - error_domain_count
message = _("""
Release complete. {} project(s) succeeded. {}
The following content was released:
{}
The following linked project spaces received content:
""").format(
success_domain_count,
_("{} project(s) encountered errors.").format(error_domain_count)
if error_domain_count else "",
"\n".join(["- " + m['name'] for m in models]))
for linked_domain in linked_domains:
if linked_domain not in errors_by_domain:
message += _("\n- {} updated successfully").format(linked_domain)
else:
message += _("\n- {} encountered errors:").format(linked_domain)
for msg in errors_by_domain[linked_domain] + successes_by_domain[
linked_domain]:
message += "\n - " + msg
send_mail_async.delay(subject, message, settings.DEFAULT_FROM_EMAIL,
[user.email or user.username])
def options(self):
links = get_linked_domains(self.domain)
return [
(str(link.id), link.linked_domain)
for link in links
]
def options(self):
links = get_linked_domains(self.domain, include_deleted=True)
return [(str(link.id), link.linked_domain) for link in links]
def page_context(self):
timezone = get_timezone_for_request()
def _link_context(link, timezone=timezone):
return {
'linked_domain': link.linked_domain,
'master_domain': link.qualified_master,
'remote_base_url': link.remote_base_url,
'is_remote': link.is_remote,
'last_update': server_to_user_time(link.last_pull, timezone) if link.last_pull else 'Never',
}
model_status = []
linked_models = dict(LINKED_MODELS)
master_link = get_domain_master_link(self.domain)
if master_link:
linked_apps = {
app._id: app for app in get_brief_apps_in_domain(self.domain)
if app.doc_type == 'LinkedApplication'
}
models_seen = set()
history = DomainLinkHistory.objects.filter(link=master_link).annotate(row_number=RawSQL(
'row_number() OVER (PARTITION BY model, model_detail ORDER BY date DESC)',
[]
))
for action in history:
models_seen.add(action.model)
if action.row_number != 1:
# first row is the most recent
continue
name = linked_models[action.model]
update = {
'type': action.model,
'name': name,
'last_update': server_to_user_time(action.date, timezone),
'detail': action.model_detail,
'can_update': True
}
if action.model == 'app':
app_name = 'Unknown App'
if action.model_detail:
detail = action.wrapped_detail
app = linked_apps.pop(detail.app_id, None)
app_name = app.name if app else detail.app_id
if app:
update['detail'] = action.model_detail
else:
update['can_update'] = False
else:
update['can_update'] = False
update['name'] = '{} ({})'.format(name, app_name)
model_status.append(update)
# Add in models that have never been synced
for model, name in LINKED_MODELS:
if model not in models_seen and model != 'app':
model_status.append({
'type': model,
'name': name,
'last_update': ugettext('Never'),
'detail': None,
'can_update': True
})
# Add in apps that have never been synced
if linked_apps:
for app in linked_apps.values():
update = {
'type': 'app',
'name': '{} ({})'.format(linked_models['app'], app.name),
'last_update': None,
'detail': AppLinkDetail(app_id=app._id).to_json(),
'can_update': True
}
model_status.append(update)
return {
'domain': self.domain,
'timezone': timezone.localize(datetime.utcnow()).tzname(),
'view_data': {
'master_link': _link_context(master_link) if master_link else None,
'model_status': sorted(model_status, key=lambda m: m['name']),
'linked_domains': [
_link_context(link) for link in get_linked_domains(self.domain)
],
'models': [
{'slug': model[0], 'name': model[1]}
for model in LINKED_MODELS
]
},
}
def page_context(self):
"""
This view services both domains that are upstream, downstream, and legacy domains that are both
"""
timezone = get_timezone_for_request()
upstream_link = get_upstream_domain_link(self.domain)
linked_domains = [build_domain_link_view_model(link, timezone) for link in get_linked_domains(self.domain)]
upstream_apps, downstream_apps = get_upstream_and_downstream_apps(self.domain)
upstream_fixtures, downstream_fixtures = get_upstream_and_downstream_fixtures(self.domain, upstream_link)
upstream_reports, downstream_reports = get_upstream_and_downstream_reports(self.domain)
upstream_keywords, downstream_keywords = get_upstream_and_downstream_keywords(self.domain)
upstream_ucr_expressions, downstream_ucr_expressions = get_upstream_and_downstream_ucr_expressions(
self.domain
)
is_superuser = self.request.couch_user.is_superuser
timezone = get_timezone_for_request()
view_models_to_pull = build_pullable_view_models_from_data_models(
self.domain,
upstream_link,
downstream_apps,
downstream_fixtures,
downstream_reports,
downstream_keywords,
downstream_ucr_expressions,
timezone,
is_superuser=is_superuser
)
view_models_to_push = build_view_models_from_data_models(
self.domain,
upstream_apps,
upstream_fixtures,
upstream_reports,
upstream_keywords,
upstream_ucr_expressions,
is_superuser=is_superuser
)
available_domains_to_link = get_available_domains_to_link(self.request.domain, self.request.couch_user)
upstream_domain_urls = []
for domain in get_available_upstream_domains(self.request.domain, self.request.couch_user):
upstream_domain_urls.append({'name': domain, 'url': reverse('domain_links', args=[domain])})
if upstream_link and upstream_link.is_remote:
remote_linkable_ucr = get_remote_linkable_ucr(upstream_link)
else:
remote_linkable_ucr = None
return {
'domain': self.domain,
'timezone': timezone.localize(datetime.utcnow()).tzname(),
'view_data': {
'is_superuser': is_superuser,
'is_downstream_domain': bool(upstream_link),
'upstream_domains': upstream_domain_urls,
'available_domains': available_domains_to_link,
'upstream_link': build_domain_link_view_model(upstream_link, timezone) if upstream_link else None,
'view_models_to_pull': sorted(view_models_to_pull, key=lambda m: m['name']),
'view_models_to_push': sorted(view_models_to_push, key=lambda m: m['name']),
'linked_domains': sorted(linked_domains, key=lambda d: d['downstream_domain']),
'linkable_ucr': remote_linkable_ucr,
'has_full_access': can_domain_access_linked_domains(self.domain, include_lite_version=False),
},
}
