def test_prune_autogenerated_builds(self):
kafka_seq = get_topic_offset(topics.APP)
couch_seq = get_current_seq(Application.get_db())
# Build #1, manually generated
app = self._create_app('test-prune-app')
build1 = app.make_build()
build1.save()
self.assertFalse(build1.is_auto_generated)
# Build #2, auto-generated
app.save()
autogenerate_build(app, 'username')
# Build #3, manually generated
app.save()
build3 = app.make_build()
build3.save()
# All 3 builds should show up in ES
self.refresh_elasticsearch(kafka_seq, couch_seq)
build_ids_in_es = AppES().domain(self.domain).is_build().values_list(
'_id', flat=True)
self.assertEqual(len(build_ids_in_es), 3)
# prune, which should delete the autogenerated build
prune_auto_generated_builds(self.domain, app.id)
# Build2 should no longer be in ES
self.refresh_elasticsearch(kafka_seq, couch_seq)
build_ids_in_es = AppES().domain(self.domain).is_build().values_list(
'_id', flat=True)
self.assertItemsEqual(build_ids_in_es, [build1._id, build3._id])
def get_domain_languages(domain):
query = AppES().domain(domain).terms_aggregation("langs", "languages").size(0)
app_languages = query.run().aggregations.languages.keys
translation_doc = StandaloneTranslationDoc.get_obj(domain, "sms")
sms_languages = translation_doc.langs if translation_doc else []
domain_languages = []
for lang_code in set(app_languages + sms_languages):
name = langcodes.get_name(lang_code)
label = u"{} ({})".format(lang_code, name) if name else lang_code
domain_languages.append((lang_code, label))
return sorted(domain_languages) or langcodes.get_all_langs_for_select()
def test_app_pillow_kafka(self):
consumer = get_test_kafka_consumer(topics.APP)
# have to get the seq id before the change is processed
kafka_seq = get_topic_offset(topics.APP)
couch_seq = get_current_seq(Application.get_db())
app_name = 'app-{}'.format(uuid.uuid4().hex)
app = self._create_app(app_name)
app_db_pillow = get_application_db_kafka_pillow('test_app_db_pillow')
app_db_pillow.process_changes(couch_seq, forever=False)
# confirm change made it to kafka
message = next(consumer)
change_meta = change_meta_from_kafka_message(message.value)
self.assertEqual(app._id, change_meta.document_id)
self.assertEqual(self.domain, change_meta.domain)
# send to elasticsearch
app_pillow = get_app_to_elasticsearch_pillow()
app_pillow.process_changes(since=kafka_seq, forever=False)
self.es.indices.refresh(APP_INDEX_INFO.index)
# confirm change made it to elasticserach
results = AppES().run()
self.assertEqual(1, results.total)
app_doc = results.hits[0]
self.assertEqual(self.domain, app_doc['domain'])
self.assertEqual(app['_id'], app_doc['_id'])
self.assertEqual(app_name, app_doc['name'])
def get_case_types_from_apps(domain):
"""
Get the case types of modules in applications in the domain.
:returns: A set of case_types
"""
q = (AppES().domain(domain).is_build(False).size(0).terms_aggregation(
'modules.case_type.exact', 'case_types'))
return set(q.run().aggregations.case_types.keys) - {''}
def apps_update_calculated_properties():
query = AppES().is_build(False).values_list('_id', 'domain', scroll=True)
for doc_id, domain in query:
doc = {
"_id": doc_id,
"cp_is_active": is_app_active(doc_id, domain),
}
send_to_elasticsearch('apps', doc, es_merge_update=True)
def paginate_releases(request, domain, app_id):
limit = request.GET.get('limit')
only_show_released = json.loads(
request.GET.get('only_show_released', 'false'))
build_comment = request.GET.get('build_comment')
page = int(request.GET.get('page', 1))
page = max(page, 1)
try:
limit = int(limit)
except (TypeError, ValueError):
limit = 10
timezone = get_timezone_for_user(request.couch_user, domain)
app_es = (AppES().start((page - 1) * limit).size(limit).sort(
'version', desc=True).domain(domain).is_build().app_id(app_id))
if only_show_released:
app_es = app_es.is_released()
if build_comment:
app_es = app_es.build_comment(build_comment)
results = app_es.exclude_source().run()
app_ids = results.doc_ids
apps = get_docs(Application.get_db(), app_ids)
for app in apps:
app.pop('translations')
saved_apps = [
SavedAppBuild.wrap(
app, scrap_old_conventions=False).to_saved_build_json(timezone)
for app in apps
]
j2me_enabled_configs = CommCareBuildConfig.j2me_enabled_config_labels()
for app in saved_apps:
app['include_media'] = app['doc_type'] != 'RemoteApp'
app['j2me_enabled'] = app['menu_item_label'] in j2me_enabled_configs
app['target_commcare_flavor'] = (
SavedAppBuild.get(app['_id']).target_commcare_flavor
if toggles.TARGET_COMMCARE_FLAVOR.enabled(domain) else 'none')
if toggles.APPLICATION_ERROR_REPORT.enabled(request.couch_user.username):
versions = [app['version'] for app in saved_apps]
num_errors_dict = _get_error_counts(domain, app_id, versions)
for app in saved_apps:
app['num_errors'] = num_errors_dict.get(app['version'], 0)
total_apps = results.total
num_pages = int(ceil(total_apps / limit))
return json_response({
'apps': saved_apps,
'pagination': {
'total': total_apps,
'num_pages': num_pages,
'current_page': page,
}
})
def test_hard_delete_app(self):
consumer = get_test_kafka_consumer(topics.APP)
# have to get the seq id before the change is processed
kafka_seq = get_topic_offset(topics.APP)
couch_seq = get_current_seq(Application.get_db())
app = self._create_app('test_hard_deleted_app', cleanup=False)
app_db_pillow = get_application_db_kafka_pillow('test_app_db_pillow')
app_db_pillow.process_changes(couch_seq, forever=False)
# confirm change made it to kafka
message = next(consumer)
change_meta = change_meta_from_kafka_message(message.value)
self.assertEqual(app._id, change_meta.document_id)
self.assertEqual(self.domain, change_meta.domain)
# send to elasticsearch
app_pillow = get_app_to_elasticsearch_pillow()
app_pillow.process_changes(since=kafka_seq, forever=False)
self.es.indices.refresh(APP_INDEX_INFO.index)
# confirm change made it to elasticserach
results = AppES().run()
self.assertEqual(1, results.total)
couch_seq = get_current_seq(Application.get_db())
kafka_seq = get_topic_offset(topics.APP)
app.delete()
app_db_pillow.process_changes(couch_seq, forever=False)
# confirm change made it to kafka. Would raise StopIteration otherwise
next(consumer)
# send to elasticsearch
app_pillow = get_app_to_elasticsearch_pillow()
app_pillow.process_changes(since=kafka_seq, forever=False)
self.es.indices.refresh(APP_INDEX_INFO.index)
# confirm deletion made it to elasticserach
results = AppES().run()
self.assertEqual(0, results.total)
def get_domain_languages(domain):
query = (AppES().domain(domain).terms_aggregation('langs',
'languages').size(0))
app_languages = query.run().aggregations.languages.keys
translation_doc = StandaloneTranslationDoc.get_obj(domain, 'sms')
sms_languages = translation_doc.langs if translation_doc else []
domain_languages = []
for lang_code in set(app_languages + sms_languages):
name = langcodes.get_name(lang_code)
label = u"{} ({})".format(lang_code, name) if name else lang_code
domain_languages.append((lang_code, label))
return sorted(domain_languages) or langcodes.get_all_langs_for_select()
def get_apps_updated_between_dates(domain, startdate, enddate):
return AppES(for_export=True).domain(domain)\
.date_range('last_modified', gte=startdate, lte=enddate).run().hits
def get_app_languages(domain):
query = (AppES().domain(domain).terms_aggregation('langs',
'languages').size(0))
return set(query.run().aggregations.languages.keys)
def get_app_ids(self):
q = AppES().created_from_template(True).is_build(False).fields('_id')
results = q.run()
return [app['_id'] for app in results.hits]
