def overwrite_app(app, master_build, report_map=None, maintain_ids=False):
excluded_fields = set(Application._meta_fields).union([
'date_created', 'build_profiles', 'copy_history', 'copy_of', 'name',
'comment', 'doc_type'
])
master_json = master_build.to_json()
for key, value in master_json.iteritems():
if key not in excluded_fields:
app[key] = value
app['version'] = master_json['version']
wrapped_app = wrap_app(app)
for module in wrapped_app.modules:
if isinstance(module, ReportModule):
if report_map is not None:
for config in module.report_configs:
try:
config.report_id = report_map[config.report_id]
except KeyError:
raise AppEditingError('Dynamic UCR used in linked app')
else:
raise AppEditingError('Report map not passed to overwrite_app')
if maintain_ids:
id_map = _get_form_id_map(app)
wrapped_app = _update_form_ids(wrapped_app, master_build, id_map)
wrapped_app.copy_attachments(master_build)
enable_usercase_if_necessary(wrapped_app)
wrapped_app.save(increment_version=False)
def get_app_by_version(domain, app_id, version):
app = get_build_doc_by_version(domain, app_id, version)
if not app:
raise Exception(
"No app found with id '{}' and version '{}', on '{}'".format(
app_id, version, domain))
return wrap_app(app)
def handle(self, commit, **options):
logger.setLevel('DEBUG')
start_date = datetime(2020, 3, 19) # Initial release of FFX app
end_date = datetime(2020, 4, 4) # Release of analytics
app_query = AppES().term('doc_type', 'Application') \
.missing('created_from_template') \
.date_range('date_created', gt=start_date, lt=end_date)
hits = app_query.run().hits
logger.info(f"Pulled {len(hits)} apps from ES")
hits = [
h for h in hits
if 'FFX' in h['name'] and len(h['modules']) == 9 and (
not h['family_id'] or h['family_id'] in KNOWN_FAMILY_IDS)
]
logger.info(
f"Filtered to {len(hits)} apps likely imported from app library")
for hit in hits:
app = wrap_app(Application.get_db().get(hit['_id']))
app.created_from_template = _get_app_id(hit['date_created'])
if commit:
app.save(increment_version=False)
logger.info(
f"Done with backfill_created_from_template, commit={commit}")
def setup_ccz_file_for_hosting(hosted_ccz_id):
try:
hosted_ccz = HostedCCZ.objects.get(pk=hosted_ccz_id)
except HostedCCZ.DoesNotExist:
return
version = hosted_ccz.version
ccz_utility = hosted_ccz.utility
# set up the file if not already present
if not ccz_utility.file_exists():
# profile_id should be None and not any other false value
profile_id = hosted_ccz.profile_id or None
build = wrap_app(
get_build_doc_by_version(hosted_ccz.domain, hosted_ccz.app_id,
version))
ccz_file = create_files_for_ccz(
build,
profile_id,
download_targeted_version=build.has_commcare_flavor)
try:
with open(ccz_file, 'rb') as ccz:
ccz_utility.store_file_in_blobdb(ccz,
name=hosted_ccz.file_name)
except:
exc = sys.exc_info()
# delete the file from blob db if it was added but later failed
hosted_ccz.delete_ccz()
six.reraise(*exc)
def setup_ccz_file_for_hosting(hosted_ccz_id, user_email=None):
try:
hosted_ccz = HostedCCZ.objects.get(pk=hosted_ccz_id)
except HostedCCZ.DoesNotExist:
return
hosted_ccz.update_status('building')
version = hosted_ccz.version
ccz_utility = hosted_ccz.utility
# set up the file if not already present
if not ccz_utility.file_exists():
# profile_id should be None and not any other false value
profile_id = hosted_ccz.profile_id or None
build = wrap_app(
get_build_doc_by_version(hosted_ccz.domain, hosted_ccz.app_id,
version))
try:
ccz_file = create_files_for_ccz(build,
profile_id,
download_targeted_version=bool(
build.commcare_flavor))
with open(ccz_file, 'rb') as ccz:
ccz_utility.store_file_in_blobdb(ccz,
name=hosted_ccz.file_name)
hosted_ccz.update_status('completed')
except:
exc = sys.exc_info()
hosted_ccz.update_status('failed')
if user_email:
_notify_failure_to_user(hosted_ccz, build, user_email)
# delete the file from blob db if it was added but later failed
hosted_ccz.delete_ccz()
six.reraise(*exc)
else:
hosted_ccz.update_status('completed')
def overwrite_app(app, master_build, report_map=None):
excluded_fields = set(Application._meta_fields).union([
'date_created', 'build_profiles', 'copy_history', 'copy_of',
'name', 'comment', 'doc_type', '_LAZY_ATTACHMENTS', 'practice_mobile_worker_id',
'custom_base_url'
])
master_json = master_build.to_json()
app_json = app.to_json()
id_map = _get_form_id_map(app_json) # do this before we change the source
for key, value in six.iteritems(master_json):
if key not in excluded_fields:
app_json[key] = value
app_json['version'] = master_json['version']
wrapped_app = wrap_app(app_json)
for module in wrapped_app.get_report_modules():
if report_map is None:
raise AppEditingError('Report map not passed to overwrite_app')
for config in module.report_configs:
try:
config.report_id = report_map[config.report_id]
except KeyError:
raise AppEditingError(config.report_id)
wrapped_app = _update_form_ids(wrapped_app, master_build, id_map)
enable_usercase_if_necessary(wrapped_app)
return wrapped_app
def _add_overrides_for_build(self, doc):
linked_build = wrap_app(doc)
log_prefix = "{}{} app {}, build {}".format(
"[DRY RUN] " if self.dry_run else "", linked_build.domain,
linked_build.origin_id, linked_build.get_id)
if not linked_build.upstream_app_id or not linked_build.upstream_version:
return
if not linked_build.domain_link:
logger.error(
"{}: Skipping due to missing domain link".format(log_prefix))
return
try:
master_build = get_master_app_by_version(
linked_build.domain_link, linked_build.upstream_app_id,
linked_build.upstream_version)
except ActionNotPermitted:
logger.error("{}: Skipping due to 403".format(log_prefix))
return
if not master_build:
logger.info(
"{}: Skipping, no master build found".format(log_prefix))
return
linked_map = self._get_xmlns_map(linked_build)
master_map = self._get_xmlns_map(master_build)
override_map = {
master_form_unique_id: linked_map[xmlns]
for xmlns, master_form_unique_id in master_map.items()
if xmlns in linked_map
}
if not override_map:
logger.info(
"{}: Skipping, no forms found to map".format(log_prefix))
return
current_overrides = {
pre_id: override.post_id
for pre_id, override in get_xform_resource_overrides(
linked_build.domain, linked_build.origin_id).items()
}
if set(override_map.items()) - set(current_overrides.items()):
logger.info("{}: Found {} overrides, updating with {}".format(
log_prefix, len(current_overrides), len(override_map)))
if not self.dry_run:
try:
add_xform_resource_overrides(linked_build.domain,
linked_build.origin_id,
override_map)
except ResourceOverrideError as e:
logger.error("{}".format(str(
e))) # skip log_prefix, error message has same info
else:
logger.info(
"{}: Skipping, all {} overrides already present".format(
log_prefix, len(override_map)))
def get_all_latest_builds_for_user(user):
app_ids = [(domain, app_id) for domain in user.domains
for app_id in get_app_ids_in_domain(domain)]
app_docs = [
get_latest_released_app_doc(app_id[0], app_id[1]) for app_id in app_ids
]
return [wrap_app(app_doc) for app_doc in app_docs if app_doc is not None]
def handle(self, path, build_slug, **options):
app_slugs = []
perfpath = os.path.join(path, '{}-performance.txt'.format(build_slug))
if os.path.exists(perfpath):
os.remove(perfpath)
for name in os.listdir(os.path.join(path, 'src')):
_JSON = '.json'
if name.endswith(_JSON):
app_slugs.append(name[:-len(_JSON)])
for slug in app_slugs:
print('Fetching %s...' % slug)
source_path = os.path.join(path, 'src', '%s.json' % slug)
with open(source_path, encoding='utf-8') as f:
j = json.load(f)
app = wrap_app(j)
app.version = 1
if not app.domain:
app.domain = "test"
build_path = os.path.join(path, build_slug, slug)
print(' Creating files...')
if track_perf:
with record_performance_stats(perfpath, slug):
files = app.create_all_files()
else:
files = app.create_all_files()
self.write_files(files, build_path)
def overwrite_app(app, master_build, report_map=None):
excluded_fields = set(Application._meta_fields).union([
'date_created', 'build_profiles', 'copy_history', 'copy_of',
'name', 'comment', 'doc_type', '_LAZY_ATTACHMENTS', 'practice_mobile_worker_id',
'custom_base_url'
])
master_json = master_build.to_json()
app_json = app.to_json()
id_map = _get_form_id_map(app_json) # do this before we change the source
for key, value in six.iteritems(master_json):
if key not in excluded_fields:
app_json[key] = value
app_json['version'] = master_json['version']
wrapped_app = wrap_app(app_json)
for module in wrapped_app.get_report_modules():
if report_map is None:
raise AppEditingError('Report map not passed to overwrite_app')
for config in module.report_configs:
try:
config.report_id = report_map[config.report_id]
except KeyError:
raise AppEditingError(config.report_id)
wrapped_app = _update_form_ids(wrapped_app, master_build, id_map)
enable_usercase_if_necessary(wrapped_app)
return wrapped_app
def get_master_app_by_version(domain_link, upstream_app_id, upstream_version):
if domain_link.is_remote:
app = get_app_by_version(domain_link, upstream_app_id, upstream_version)
else:
app = get_build_doc_by_version(domain_link.master_domain, upstream_app_id, upstream_version)
if app:
return wrap_app(app)
def get_all_latest_builds_for_user(user):
app_ids = [
(domain, app_id)
for domain in user.domains
for app_id in get_app_ids_in_domain(domain)
]
app_docs = [get_latest_released_app_doc(app_id[0], app_id[1]) for app_id in app_ids]
return [wrap_app(app_doc) for app_doc in app_docs if app_doc is not None]
def overwrite_app(app, master_build, report_map=None):
excluded_fields = set(Application._meta_fields).union([
'date_created',
'build_profiles',
'copy_history',
'copy_of',
'name',
'comment',
'doc_type',
'_LAZY_ATTACHMENTS',
'practice_mobile_worker_id',
'custom_base_url',
'family_id',
])
master_json = master_build.to_json()
app_json = app.to_json()
for key, value in master_json.items():
if key not in excluded_fields:
app_json[key] = value
app_json['version'] = app_json.get('version', 1)
app_json['upstream_version'] = master_json['version']
app_json['upstream_app_id'] = master_json['copy_of']
wrapped_app = wrap_app(app_json)
for module in wrapped_app.get_report_modules():
if report_map is None:
raise AppEditingError('Report map not passed to overwrite_app')
for config in module.report_configs:
try:
config.report_id = report_map[config.report_id]
except KeyError:
raise AppEditingError(config.report_id)
# Legacy linked apps have different form unique ids than their master app(s). These mappings
# are stored as ResourceOverride objects. Look up to see if this app has any.
from corehq.apps.app_manager.suite_xml.post_process.resources import get_xform_resource_overrides
overrides = get_xform_resource_overrides(domain=wrapped_app.domain,
app_id=wrapped_app.get_id)
ids_map = {
pre_id: override.post_id
for pre_id, override in overrides.items()
}
wrapped_app = _update_forms(wrapped_app, master_build, ids_map)
# Multimedia versions should be set based on the linked app's versions, not those of the master app.
for path in wrapped_app.multimedia_map.keys():
wrapped_app.multimedia_map[path].version = None
wrapped_app.set_media_versions()
enable_usercase_if_necessary(wrapped_app)
return wrapped_app
def _update_form_ids(app, master_app, form_ids_by_xmlns):
_attachments = master_app.get_attachments()
app_source = app.to_json()
app_source.pop('external_blobs')
app_source['_attachments'] = _attachments
updated_source = update_form_unique_ids(app_source, form_ids_by_xmlns)
attachments = app_source.pop('_attachments')
new_wrapped_app = wrap_app(updated_source)
save = partial(new_wrapped_app.save, increment_version=False)
return new_wrapped_app.save_attachments(attachments, save)
def _update_form_ids(app, master_app, id_map):
_attachments = master_app.get_attachments()
app_source = app.to_json()
app_source.pop('external_blobs')
app_source['_attachments'] = _attachments
updated_source = update_form_unique_ids(app_source, id_map)
attachments = app_source.pop('_attachments')
new_wrapped_app = wrap_app(updated_source)
save = partial(new_wrapped_app.save, increment_version=False)
return new_wrapped_app.save_attachments(attachments, save)
def get_report_configs(domain, ls_app_version):
if ls_app_version:
app = wrap_app(
get_build_by_version(domain,
SUPERVISOR_APP_ID,
ls_app_version,
return_doc=True))
else:
app = get_app(domain, SUPERVISOR_APP_ID, latest=True)
return {
report_config.report_id: report_config
for module in app.get_report_modules()
for report_config in module.report_configs
if report_config.report_id in REPORT_IDS
}
def pull_master_app(request, domain, app_id):
app = get_current_app_doc(domain, app_id)
master_app = get_app(None, app['master'])
latest_master_build = get_app(None, app['master'], latest=True)
params = {}
if app['domain'] in master_app.linked_whitelist:
excluded_fields = set(Application._meta_fields).union([
'date_created', 'build_profiles', 'copy_history', 'copy_of',
'name', 'comment', 'doc_type'
])
master_json = latest_master_build.to_json()
for key, value in master_json.iteritems():
if key not in excluded_fields:
app[key] = value
app['version'] = master_json['version']
wrapped_app = wrap_app(app)
mobile_ucrs = False
for module in wrapped_app.modules:
if isinstance(module, ReportModule):
mobile_ucrs = True
break
if mobile_ucrs:
messages.error(
request,
_('This linked application uses mobile UCRs '
'which are currently not supported. For this application '
'to function correctly, you will need to remove those modules '
'or revert to a previous version that did not include them.')
)
else:
messages.success(
request,
_('Your linked application was successfully updated to the latest version.'
))
wrapped_app.copy_attachments(latest_master_build)
wrapped_app.save(increment_version=False)
else:
messages.error(
request,
_('This project is not authorized to update from the master application. '
'Please contact the maintainer of the master app if you believe this is a mistake. '
))
return HttpResponseRedirect(
reverse_util('view_app', params=params, args=[domain, app_id]))
def overwrite_app(app, master_build, include_ucrs=False, report_map=None):
excluded_fields = set(Application._meta_fields).union([
'date_created', 'build_profiles', 'copy_history', 'copy_of', 'name',
'comment', 'doc_type'
])
master_json = master_build.to_json()
for key, value in master_json.iteritems():
if key not in excluded_fields:
app[key] = value
app['version'] = master_json['version']
wrapped_app = wrap_app(app)
for module in wrapped_app.modules:
if isinstance(module, ReportModule):
if include_ucrs and report_map is not None:
for config in module.report_configs:
config.report_id = report_map[config.report_id]
else:
raise AppEditingError()
wrapped_app.copy_attachments(master_build)
wrapped_app.save(increment_version=False)
def overwrite_app(app, master_build, report_map=None):
excluded_fields = set(Application._meta_fields).union([
'date_created', 'build_profiles', 'copy_history', 'copy_of',
'name', 'comment', 'doc_type', '_LAZY_ATTACHMENTS', 'practice_mobile_worker_id',
'custom_base_url', 'family_id',
])
master_json = master_build.to_json()
app_json = app.to_json()
old_form_ids_by_xmlns = _get_historical_form_ids_by_xmlns(app) # before updating anything
for key, value in master_json.items():
if key not in excluded_fields:
app_json[key] = value
app_json['version'] = app_json.get('version', 1)
app_json['upstream_version'] = master_json['version']
app_json['upstream_app_id'] = master_json['copy_of']
wrapped_app = wrap_app(app_json)
for module in wrapped_app.get_report_modules():
if report_map is None:
raise AppEditingError('Report map not passed to overwrite_app')
for config in module.report_configs:
try:
config.report_id = report_map[config.report_id]
except KeyError:
raise AppEditingError(config.report_id)
ids_map = _map_old_form_ids_to_new(wrapped_app, old_form_ids_by_xmlns)
wrapped_app = _update_form_ids(wrapped_app, master_build, ids_map)
# Multimedia versions should be set based on the linked app's versions, not those of the master app.
for path in wrapped_app.multimedia_map.keys():
wrapped_app.multimedia_map[path].version = None
wrapped_app.set_media_versions()
enable_usercase_if_necessary(wrapped_app)
return wrapped_app
def handle(self, from_domain, from_app_id, to_domain, *args, **options):
self.from_domain = from_domain
self.to_domain = to_domain
to_app_id = options.get('to-app-id')
version = options.get('version')
if to_app_id:
app = get_current_app(self.to_domain, to_app_id)
print('Overwriting application: {}'.format(app.name))
else:
print('Creating new application')
app = Application()
if version:
from_app_doc = get_build_doc_by_version(self.from_domain, from_app_id, version)
else:
from_app_doc = get_latest_released_app_doc(self.from_domain, from_app_id)
if not from_app_doc:
raise CommandError("From app not found")
from_app = wrap_app(from_app_doc)
print('Overwring app with "{}" (version {})'.format(from_app.name, from_app.version))
overwrite_app(app, from_app, self.report_map)
def update_linked_whitelist(request, domain, app_id):
app = wrap_app(get_current_app_doc(domain, app_id))
new_whitelist = json.loads(request.POST.get('whitelist'))
app.linked_whitelist = new_whitelist
app.save()
return HttpResponse()
def _get_build_functional_version(domain, app_id, version):
app_build = wrap_app(
get_build_by_version(domain, app_id, version, return_doc=True))
return app_build.profile.get('custom_properties',
{}).get('cc-app-version-tag')
def _convert_app_from_remote_linking_source(app_json):
attachments = app_json.pop('_LAZY_ATTACHMENTS', {})
app = wrap_app(app_json)
app._LAZY_ATTACHMENTS = attachments
return app
def _convert_app_from_remote_linking_source(app_json):
attachments = app_json.pop('_LAZY_ATTACHMENTS', {})
app = wrap_app(app_json)
app._LAZY_ATTACHMENTS = attachments
return app
def get_brief_apps(domain_link):
apps = _do_simple_request('linked_domain:brief_apps',
domain_link)['brief_apps']
return [wrap_app(app) for app in apps]
