def test_prune_autogenerated_builds(self): kafka_seq = get_topic_offset(topics.APP) couch_seq = get_current_seq(Application.get_db()) # Build #1, manually generated app = self._create_app('test-prune-app') build1 = app.make_build() build1.save() self.assertFalse(build1.is_auto_generated) # Build #2, auto-generated app.save() autogenerate_build(app, 'username') # Build #3, manually generated app.save() build3 = app.make_build() build3.save() # All 3 builds should show up in ES self.refresh_elasticsearch(kafka_seq, couch_seq) build_ids_in_es = AppES().domain(self.domain).is_build().values_list( '_id', flat=True) self.assertEqual(len(build_ids_in_es), 3) # prune, which should delete the autogenerated build prune_auto_generated_builds(self.domain, app.id) # Build2 should no longer be in ES self.refresh_elasticsearch(kafka_seq, couch_seq) build_ids_in_es = AppES().domain(self.domain).is_build().values_list( '_id', flat=True) self.assertItemsEqual(build_ids_in_es, [build1._id, build3._id])
def test_app_pillow_kafka(self): consumer = get_test_kafka_consumer(topics.APP) # have to get the seq id before the change is processed kafka_seq = get_topic_offset(topics.APP) couch_seq = get_current_seq(Application.get_db()) app_name = 'app-{}'.format(uuid.uuid4().hex) app = self._create_app(app_name) app_db_pillow = get_application_db_kafka_pillow('test_app_db_pillow') app_db_pillow.process_changes(couch_seq, forever=False) # confirm change made it to kafka message = next(consumer) change_meta = change_meta_from_kafka_message(message.value) self.assertEqual(app._id, change_meta.document_id) self.assertEqual(self.domain, change_meta.domain) # send to elasticsearch app_pillow = get_app_to_elasticsearch_pillow() app_pillow.process_changes(since=kafka_seq, forever=False) self.es.indices.refresh(APP_INDEX_INFO.index) # confirm change made it to elasticserach results = AppES().run() self.assertEqual(1, results.total) app_doc = results.hits[0] self.assertEqual(self.domain, app_doc['domain']) self.assertEqual(app['_id'], app_doc['_id']) self.assertEqual(app_name, app_doc['name'])
def get_case_types_from_apps(domain): """ Get the case types of modules in applications in the domain. :returns: A set of case_types """ q = (AppES().domain(domain).is_build(False).size(0).terms_aggregation( 'modules.case_type.exact', 'case_types')) return set(q.run().aggregations.case_types.keys) - {''}
def apps_update_calculated_properties(): query = AppES().is_build(False).values_list('_id', 'domain', scroll=True) for doc_id, domain in query: doc = { "_id": doc_id, "cp_is_active": is_app_active(doc_id, domain), } send_to_elasticsearch('apps', doc, es_merge_update=True)
def paginate_releases(request, domain, app_id): limit = request.GET.get('limit') only_show_released = json.loads( request.GET.get('only_show_released', 'false')) build_comment = request.GET.get('build_comment') page = int(request.GET.get('page', 1)) page = max(page, 1) try: limit = int(limit) except (TypeError, ValueError): limit = 10 timezone = get_timezone_for_user(request.couch_user, domain) app_es = (AppES().start((page - 1) * limit).size(limit).sort( 'version', desc=True).domain(domain).is_build().app_id(app_id)) if only_show_released: app_es = app_es.is_released() if build_comment: app_es = app_es.build_comment(build_comment) results = app_es.exclude_source().run() app_ids = results.doc_ids apps = get_docs(Application.get_db(), app_ids) for app in apps: app.pop('translations') saved_apps = [ SavedAppBuild.wrap( app, scrap_old_conventions=False).to_saved_build_json(timezone) for app in apps ] j2me_enabled_configs = CommCareBuildConfig.j2me_enabled_config_labels() for app in saved_apps: app['include_media'] = app['doc_type'] != 'RemoteApp' app['j2me_enabled'] = app['menu_item_label'] in j2me_enabled_configs app['target_commcare_flavor'] = ( SavedAppBuild.get(app['_id']).target_commcare_flavor if toggles.TARGET_COMMCARE_FLAVOR.enabled(domain) else 'none') if toggles.APPLICATION_ERROR_REPORT.enabled(request.couch_user.username): versions = [app['version'] for app in saved_apps] num_errors_dict = _get_error_counts(domain, app_id, versions) for app in saved_apps: app['num_errors'] = num_errors_dict.get(app['version'], 0) total_apps = results.total num_pages = int(ceil(total_apps / limit)) return json_response({ 'apps': saved_apps, 'pagination': { 'total': total_apps, 'num_pages': num_pages, 'current_page': page, } })
def test_hard_delete_app(self): consumer = get_test_kafka_consumer(topics.APP) # have to get the seq id before the change is processed kafka_seq = get_topic_offset(topics.APP) couch_seq = get_current_seq(Application.get_db()) app = self._create_app('test_hard_deleted_app', cleanup=False) app_db_pillow = get_application_db_kafka_pillow('test_app_db_pillow') app_db_pillow.process_changes(couch_seq, forever=False) # confirm change made it to kafka message = next(consumer) change_meta = change_meta_from_kafka_message(message.value) self.assertEqual(app._id, change_meta.document_id) self.assertEqual(self.domain, change_meta.domain) # send to elasticsearch app_pillow = get_app_to_elasticsearch_pillow() app_pillow.process_changes(since=kafka_seq, forever=False) self.es.indices.refresh(APP_INDEX_INFO.index) # confirm change made it to elasticserach results = AppES().run() self.assertEqual(1, results.total) couch_seq = get_current_seq(Application.get_db()) kafka_seq = get_topic_offset(topics.APP) app.delete() app_db_pillow.process_changes(couch_seq, forever=False) # confirm change made it to kafka. Would raise StopIteration otherwise next(consumer) # send to elasticsearch app_pillow = get_app_to_elasticsearch_pillow() app_pillow.process_changes(since=kafka_seq, forever=False) self.es.indices.refresh(APP_INDEX_INFO.index) # confirm deletion made it to elasticserach results = AppES().run() self.assertEqual(0, results.total)
def get_domain_languages(domain): query = (AppES().domain(domain).terms_aggregation('langs', 'languages').size(0)) app_languages = query.run().aggregations.languages.keys translation_doc = StandaloneTranslationDoc.get_obj(domain, 'sms') sms_languages = translation_doc.langs if translation_doc else [] domain_languages = [] for lang_code in set(app_languages + sms_languages): name = langcodes.get_name(lang_code) label = u"{} ({})".format(lang_code, name) if name else lang_code domain_languages.append((lang_code, label)) return sorted(domain_languages) or langcodes.get_all_langs_for_select()
def get_apps_updated_between_dates(domain, startdate, enddate): return AppES(for_export=True).domain(domain)\ .date_range('last_modified', gte=startdate, lte=enddate).run().hits
def get_app_languages(domain): query = (AppES().domain(domain).terms_aggregation('langs', 'languages').size(0)) return set(query.run().aggregations.languages.keys)
def get_app_ids(self): q = AppES().created_from_template(True).is_build(False).fields('_id') results = q.run() return [app['_id'] for app in results.hits]