def release(self, models, linked_domains, build_apps=False): self._reset(models, linked_domains) domain_links_by_linked_domain = { link.linked_domain: link for link in get_linked_domains(self.master_domain) } for linked_domain in self.linked_domains: if linked_domain not in domain_links_by_linked_domain: self._add_error(linked_domain, _("Project space {} is no longer linked to {}. No content " "was released to it.").format(self.master_domain, linked_domain)) continue domain_link = domain_links_by_linked_domain[linked_domain] for model in self.models: errors = None try: if model['type'] == MODEL_APP: errors = self._release_app(domain_link, model, self.user, build_apps) elif model['type'] == MODEL_REPORT: errors = self._release_report(domain_link, model) elif model['type'] == MODEL_CASE_SEARCH: errors = self._release_case_search(domain_link, model, self.user) else: errors = self._release_model(domain_link, model, self.user) except Exception as e: # intentionally broad errors = [str(e), str(e)] notify_exception(None, "Exception pushing linked domains: {}".format(e)) if errors: self._add_error( linked_domain, _("Could not update {}: {}").format(model['name'], errors[0]), text=_("Could not update {}: {}").format(model['name'], errors[1])) else: self._add_success(linked_domain, _("Updated {} successfully").format(model['name']))
def page_context(self): """ This view services both domains that are master domains and domains that are linked domains (and legacy domains that are both). """ timezone = get_timezone_for_request() master_link = get_domain_master_link(self.domain) linked_domains = [self._link_context(link, timezone) for link in get_linked_domains(self.domain)] (master_apps, linked_apps) = self._get_apps() (master_reports, linked_reports) = self._get_reports() # Models belonging to this domain's master domain, for the purpose of pulling model_status = self._get_model_status(master_link, linked_apps, linked_reports) # Models belonging to this domain, for the purpose of pushing to linked domains master_model_status = self._get_master_model_status(master_apps, master_reports) return { 'domain': self.domain, 'timezone': timezone.localize(datetime.utcnow()).tzname(), 'is_linked_domain': bool(master_link), 'is_master_domain': bool(len(linked_domains)), 'view_data': { 'master_link': self._link_context(master_link, timezone) if master_link else None, 'model_status': sorted(model_status, key=lambda m: m['name']), 'master_model_status': sorted(master_model_status, key=lambda m: m['name']), 'linked_domains': linked_domains, 'models': [ {'slug': model[0], 'name': model[1]} for model in LINKED_MODELS ] }, }
def handle(self, domain, case_type, **options): domains = {domain} if options["and_linked"]: domains = domains | { link.linked_domain for link in get_linked_domains(domain) } if options["username"]: user_id = username_to_user_id(options["username"]) if not user_id: raise Exception("The username you entered is invalid") else: user_id = SYSTEM_USER_ID self.output_file = options["output_file"] options.pop("and_linked") options.pop("username") options.pop("output_file", None) self.extra_options = options for domain in sorted(domains): print(f"Processing {domain}") self.update_cases(domain, case_type, user_id)
def page_context(self): context = self.pagination_context context['linked_domains'] = [ domain_link.linked_domain for domain_link in get_linked_domains(self.domain) ] context['linkable_keywords'] = self._linkable_keywords() return context
def linked_downstream_reports_by_domain(master_domain, report_id): """A dict of all downstream domains with and if this is already linked to `report_id` """ from corehq.apps.linked_domain.dbaccessors import get_linked_domains linked_domains = {} for domain_link in get_linked_domains(master_domain): linked_domains[domain_link.linked_domain] = any( r for r in get_linked_report_configs(domain_link.linked_domain, report_id) ) return linked_domains
def handle(self, domain, case_type, username, **options): domains = {domain} if options["and_linked"]: domains = domains | { link.linked_domain for link in get_linked_domains(domain) } for domain in domains: print(f"Processing {domain}") update_cases(domain, case_type, username)
def handle(self, domain, case_type, **options): domains = {domain} if options["and_linked"]: domains = domains | { link.linked_domain for link in get_linked_domains(domain) } if options["username"]: user_id = username_to_user_id(options["username"]) if not user_id: raise Exception("The username you entered is invalid") else: user_id = SYSTEM_USER_ID self.location = options.get('location') for domain in domains: print(f"Processing {domain}") self.update_cases(domain, case_type, user_id)
def handle(self, domain, case_type, **options): # logger.debug will record something to a file but not print it self.logger = logging.getLogger(self.logger_name) if not settings.UNIT_TESTING: self.logger.addHandler( logging.FileHandler(self.logger_name.split(".")[-1] + ".txt")) self.logger.setLevel(logging.DEBUG) self.logger.debug( f"{datetime.datetime.utcnow()} Starting run: {options}") domains = {domain} if options.pop("and_linked"): domains = domains | { link.linked_domain for link in get_linked_domains(domain) } username = options.pop("username") self.case_type = case_type self.extra_options = options for i, domain in enumerate(sorted(domains), start=1): case_ids = self.find_case_ids(domain) self.logger.debug( f"Found {len(case_ids)} cases in {domain} ({i}/{len(domains)})" ) update_count = update_cases( domain=domain, update_fn=self.case_blocks, case_ids=with_progress_bar(case_ids, oneline=False), form_meta=SystemFormMeta.for_script(self.logger_name, username), ) self.logger.debug( f"Made {update_count} updates in {domain} ({i}/{len(domains)})" ) self.logger.debug(f"{datetime.datetime.utcnow()} Script complete")
def options(self): links = get_linked_domains(self.domain) return [ (str(link.id), link.linked_domain) for link in links ]
def page_context(self): timezone = get_timezone_for_request() def _link_context(link, timezone=timezone): return { 'linked_domain': link.linked_domain, 'master_domain': link.master_domain, 'remote_base_url': link.remote_base_url, 'remote_username': link.remote_username, 'remote_api_key': link.remote_api_key, 'is_remote': link.is_remote, 'last_update': server_to_user_time(link.last_pull, timezone) if link.last_pull else 'Never', } model_status = [] linked_models = dict(LINKED_MODELS) master_link = get_domain_master_link(self.domain) if master_link: linked_apps = { app._id: app for app in get_brief_apps_in_domain(self.domain) if app.doc_type == 'LinkedApplication' } models_seen = set() history = DomainLinkHistory.objects.filter( link=master_link ).annotate(row_number=RawSQL( 'row_number() OVER (PARTITION BY model, model_detail ORDER BY date DESC)', [])) for action in history: models_seen.add(action.model) if action.row_number != 1: # first row is the most recent continue name = linked_models[action.model] update = { 'type': action.model, 'name': name, 'last_update': server_to_user_time(action.date, timezone), 'detail': action.model_detail, 'can_update': True } if action.model == 'app': app_name = 'Unknown App' if action.model_detail: detail = action.wrapped_detail app = linked_apps.pop(detail.app_id, None) app_name = app.name if app else detail.app_id if app: update['detail'] = action.model_detail else: update['can_update'] = False else: update['can_update'] = False update['name'] = '{} ({})'.format(name, app_name) model_status.append(update) # Add in models that have never been synced for model, name in LINKED_MODELS: if model not in models_seen and model != 'app': model_status.append({ 'type': model, 'name': name, 'last_update': ugettext('Never'), 'detail': None, 'can_update': True }) # Add in apps that have never been synced if linked_apps: for app in linked_apps.values(): update = { 'type': 'app', 'name': '{} ({})'.format(linked_models['app'], app.name), 'last_update': None, 'detail': AppLinkDetail(app_id=app._id).to_json(), 'can_update': True } model_status.append(update) return { 'domain': self.domain, 'timezone': timezone.localize(datetime.utcnow()).tzname(), 'view_data': { 'master_link': _link_context(master_link) if master_link else None, 'model_status': sorted(model_status, key=lambda m: m['name']), 'linked_domains': [ _link_context(link) for link in get_linked_domains(self.domain) ], 'models': [{ 'slug': model[0], 'name': model[1] } for model in LINKED_MODELS] }, }
def push_models(master_domain, models, linked_domains, build_apps, username): domain_links_by_linked_domain = { link.linked_domain: link for link in get_linked_domains(master_domain) } user = CouchUser.get_by_username(username) errors_by_domain = defaultdict(list) successes_by_domain = defaultdict(list) for linked_domain in linked_domains: if linked_domain not in domain_links_by_linked_domain: errors_by_domain[linked_domain].append( _("Project space {} is no longer linked to {}. No content " "was released to it.").format(master_domain, linked_domain)) continue domain_link = domain_links_by_linked_domain[linked_domain] for model in models: try: found = False updated_app = False built_app = False if model['type'] == MODEL_APP: app_id = model['detail']['app_id'] for linked_app in get_apps_in_domain(linked_domain, include_remote=False): if is_linked_app( linked_app) and linked_app.family_id == app_id: found = True if toggles.MULTI_MASTER_LINKED_DOMAINS.enabled( linked_domain): errors_by_domain[linked_domain].append( textwrap.dedent( _(""" Could not update {} because multi master flag is in use """.strip()).format(model['name']))) continue app = update_linked_app(linked_app, app_id, user.user_id) updated_app = True if build_apps: build = app.make_build() build.is_released = True build.save(increment_version=False) built_app = True elif model['type'] == MODEL_REPORT: report_id = model['detail']['report_id'] for linked_report in get_report_configs_for_domain( linked_domain): if linked_report.report_meta.master_id == report_id: found = True update_linked_ucr(domain_link, linked_report.get_id) elif (model['type'] == MODEL_CASE_SEARCH and not toggles.SYNC_SEARCH_CASE_CLAIM.enabled(linked_domain) ): errors_by_domain[linked_domain].append( textwrap.dedent( _(""" Could not update {} because case claim flag is not on """.strip()).format(model['name']))) continue else: found = True update_model_type(domain_link, model['type'], model_detail=model['detail']) domain_link.update_last_pull(model['type'], user._id, model_details=model['detail']) if found: successes_by_domain[linked_domain].append( _("{} was updated").format(model['name'])) else: errors_by_domain[linked_domain].append( _("Could not find {}").format(model['name'])) except Exception as e: # intentionally broad if model[ 'type'] == MODEL_APP and updated_app and build_apps and not built_app: # Updating an app can be a 2-step process, make it clear which one failed errors_by_domain[linked_domain].append( textwrap.dedent( _(""" Updated {} but could not make and release build: {} """.strip()).format(model['name'], str(e)))) else: errors_by_domain[linked_domain].append( textwrap.dedent( _(""" Could not update {}: {} """.strip()).format(model['name'], str(e)))) notify_exception( None, "Exception pushing linked domains: {}".format(e)) subject = _("Linked project release complete.") if errors_by_domain: subject += _(" Errors occurred.") error_domain_count = len(errors_by_domain) success_domain_count = len(linked_domains) - error_domain_count message = _(""" Release complete. {} project(s) succeeded. {} The following content was released: {} The following linked project spaces received content: """).format( success_domain_count, _("{} project(s) encountered errors.").format(error_domain_count) if error_domain_count else "", "\n".join(["- " + m['name'] for m in models])) for linked_domain in linked_domains: if linked_domain not in errors_by_domain: message += _("\n- {} updated successfully").format(linked_domain) else: message += _("\n- {} encountered errors:").format(linked_domain) for msg in errors_by_domain[linked_domain] + successes_by_domain[ linked_domain]: message += "\n - " + msg send_mail_async.delay(subject, message, settings.DEFAULT_FROM_EMAIL, [user.email or user.username])
def options(self): links = get_linked_domains(self.domain) return [ (str(link.id), link.linked_domain) for link in links ]
def options(self): links = get_linked_domains(self.domain, include_deleted=True) return [(str(link.id), link.linked_domain) for link in links]
def page_context(self): timezone = get_timezone_for_request() def _link_context(link, timezone=timezone): return { 'linked_domain': link.linked_domain, 'master_domain': link.qualified_master, 'remote_base_url': link.remote_base_url, 'is_remote': link.is_remote, 'last_update': server_to_user_time(link.last_pull, timezone) if link.last_pull else 'Never', } model_status = [] linked_models = dict(LINKED_MODELS) master_link = get_domain_master_link(self.domain) if master_link: linked_apps = { app._id: app for app in get_brief_apps_in_domain(self.domain) if app.doc_type == 'LinkedApplication' } models_seen = set() history = DomainLinkHistory.objects.filter(link=master_link).annotate(row_number=RawSQL( 'row_number() OVER (PARTITION BY model, model_detail ORDER BY date DESC)', [] )) for action in history: models_seen.add(action.model) if action.row_number != 1: # first row is the most recent continue name = linked_models[action.model] update = { 'type': action.model, 'name': name, 'last_update': server_to_user_time(action.date, timezone), 'detail': action.model_detail, 'can_update': True } if action.model == 'app': app_name = 'Unknown App' if action.model_detail: detail = action.wrapped_detail app = linked_apps.pop(detail.app_id, None) app_name = app.name if app else detail.app_id if app: update['detail'] = action.model_detail else: update['can_update'] = False else: update['can_update'] = False update['name'] = '{} ({})'.format(name, app_name) model_status.append(update) # Add in models that have never been synced for model, name in LINKED_MODELS: if model not in models_seen and model != 'app': model_status.append({ 'type': model, 'name': name, 'last_update': ugettext('Never'), 'detail': None, 'can_update': True }) # Add in apps that have never been synced if linked_apps: for app in linked_apps.values(): update = { 'type': 'app', 'name': '{} ({})'.format(linked_models['app'], app.name), 'last_update': None, 'detail': AppLinkDetail(app_id=app._id).to_json(), 'can_update': True } model_status.append(update) return { 'domain': self.domain, 'timezone': timezone.localize(datetime.utcnow()).tzname(), 'view_data': { 'master_link': _link_context(master_link) if master_link else None, 'model_status': sorted(model_status, key=lambda m: m['name']), 'linked_domains': [ _link_context(link) for link in get_linked_domains(self.domain) ], 'models': [ {'slug': model[0], 'name': model[1]} for model in LINKED_MODELS ] }, }
def page_context(self): """ This view services both domains that are upstream, downstream, and legacy domains that are both """ timezone = get_timezone_for_request() upstream_link = get_upstream_domain_link(self.domain) linked_domains = [build_domain_link_view_model(link, timezone) for link in get_linked_domains(self.domain)] upstream_apps, downstream_apps = get_upstream_and_downstream_apps(self.domain) upstream_fixtures, downstream_fixtures = get_upstream_and_downstream_fixtures(self.domain, upstream_link) upstream_reports, downstream_reports = get_upstream_and_downstream_reports(self.domain) upstream_keywords, downstream_keywords = get_upstream_and_downstream_keywords(self.domain) upstream_ucr_expressions, downstream_ucr_expressions = get_upstream_and_downstream_ucr_expressions( self.domain ) is_superuser = self.request.couch_user.is_superuser timezone = get_timezone_for_request() view_models_to_pull = build_pullable_view_models_from_data_models( self.domain, upstream_link, downstream_apps, downstream_fixtures, downstream_reports, downstream_keywords, downstream_ucr_expressions, timezone, is_superuser=is_superuser ) view_models_to_push = build_view_models_from_data_models( self.domain, upstream_apps, upstream_fixtures, upstream_reports, upstream_keywords, upstream_ucr_expressions, is_superuser=is_superuser ) available_domains_to_link = get_available_domains_to_link(self.request.domain, self.request.couch_user) upstream_domain_urls = [] for domain in get_available_upstream_domains(self.request.domain, self.request.couch_user): upstream_domain_urls.append({'name': domain, 'url': reverse('domain_links', args=[domain])}) if upstream_link and upstream_link.is_remote: remote_linkable_ucr = get_remote_linkable_ucr(upstream_link) else: remote_linkable_ucr = None return { 'domain': self.domain, 'timezone': timezone.localize(datetime.utcnow()).tzname(), 'view_data': { 'is_superuser': is_superuser, 'is_downstream_domain': bool(upstream_link), 'upstream_domains': upstream_domain_urls, 'available_domains': available_domains_to_link, 'upstream_link': build_domain_link_view_model(upstream_link, timezone) if upstream_link else None, 'view_models_to_pull': sorted(view_models_to_pull, key=lambda m: m['name']), 'view_models_to_push': sorted(view_models_to_push, key=lambda m: m['name']), 'linked_domains': sorted(linked_domains, key=lambda d: d['downstream_domain']), 'linkable_ucr': remote_linkable_ucr, 'has_full_access': can_domain_access_linked_domains(self.domain, include_lite_version=False), }, }