def tearDownClass(cls): for user in cls.mobile_users + [cls.web_user]: user.delete('state', deleted_by=None) for domain_obj in cls.domains: Domain.get_db().delete_doc(domain_obj) ensure_index_deleted(USER_INDEX) super().tearDownClass()
def load_objects(self, object_strings, force=False): from corehq.apps.domain.models import Domain objects = list(object_strings) assert len(objects) == 1, "Only 1 domain allowed per dump" domain_dict = json.loads(objects[0]) domain_name = domain_dict['name'] try: existing_domain = Domain.get_by_name(domain_name, strict=True) except ResourceNotFound: pass else: if existing_domain: if force: self.stderr.write( 'Loading data for existing domain: {}'.format( domain_name)) else: raise DataExistsException("Domain: {}".format(domain_name)) Domain.get_db().bulk_save([domain_dict], new_edits=False) self.stdout.write('Loaded Domain') return 1, Counter({'Domain': 1})
def tearDownClass(cls): cls.app.delete() cls.registry.delete() cls.restore_user._couch_user.delete(None, None) cls.restore_user_domain_1._couch_user.delete(None, None) Domain.get_db().delete_doc(cls.domain_obj.get_id) super().tearDownClass()
def handle(self, *args, **kwargs): domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)] with IterativeSaver(Domain.get_db()) as iter_db: for domain_doc in iter_docs(Domain.get_db(), domain_ids): if (domain_doc.pop('location_types', None) or domain_doc.pop('obsolete_location_types', None)): print("Removing location_types from domain {} - {}".format( domain_doc['name'], domain_doc['_id'])) iter_db.save(domain_doc)
def handle(self, *args, **kwargs): domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)] with IterativeSaver(Domain.get_db()) as iter_db: for domain_doc in iter_docs(Domain.get_db(), domain_ids): if ( domain_doc.pop('location_types', None) or domain_doc.pop('obsolete_location_types', None) ): print ("Removing location_types from domain {} - {}" .format(domain_doc['name'], domain_doc['_id'])) iter_db.save(domain_doc)
def generate_invoices(based_on_date=None): """ Generates all invoices for the past month. """ today = based_on_date or datetime.date.today() invoice_start, invoice_end = get_previous_month_date_range(today) log_accounting_info( "Starting up invoices for %(start)s - %(end)s" % { 'start': invoice_start.strftime(USER_DATE_FORMAT), 'end': invoice_end.strftime(USER_DATE_FORMAT), }) all_domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)] for domain_doc in iter_docs(Domain.get_db(), all_domain_ids): domain = Domain.wrap(domain_doc) try: invoice_factory = DomainInvoiceFactory(invoice_start, invoice_end, domain) invoice_factory.create_invoices() log_accounting_info("Sent invoices for domain %s" % domain.name) except CreditLineError as e: log_accounting_error("There was an error utilizing credits for " "domain %s: %s" % (domain.name, e)) except InvoiceError as e: log_accounting_error("Could not create invoice for domain %s: %s" % (domain.name, e)) except InvoiceAlreadyCreatedError as e: log_accounting_error("Invoice already existed for domain %s: %s" % (domain.name, e)) except Exception as e: log_accounting_error("Error occurred while creating invoice for " "domain %s: %s" % (domain.name, e)) if not settings.UNIT_TESTING: _invoicing_complete_soft_assert(False, "Invoicing is complete!")
def _domain_to_change_meta(domain): domain_doc = domain.to_json() return change_meta_from_doc( document=domain_doc, data_source_type=data_sources.SOURCE_COUCH, data_source_name=Domain.get_db().dbname, )
def iter_domains(): for row in paginate_view(Domain.get_db(), 'domain/domains', 100, reduce=False, include_docs=False): yield row['key']
def _copy(config): # unfortunately the only couch view we have for this needs to go by domain # will be a bit slow database = Domain.get_db() assert database.uri == config.source_db.uri, 'can only use "copy" with the main HQ DB as the source' domain_names = Domain.get_all_names() for domain in domain_names: for doc_type in config.doc_types: ids_of_this_type = [row['id'] for row in database.view( 'domain/docs', startkey=[domain, doc_type], endkey=[domain, doc_type, {}], reduce=False, include_docs=False, )] if ids_of_this_type: new_revs = dict([ (row['id'], row['value']['rev']) for row in config.dest_db.view('_all_docs', keys=ids_of_this_type, include_docs=False) if 'error' not in row ]) for id_group in chunked(ids_of_this_type, 500): docs = get_docs(database, id_group) for doc in docs: if doc['_id'] in new_revs: doc['_rev'] = new_revs[doc['_id']] config.dest_db.bulk_save(docs) print 'copied {} {}s from {}'.format(len(ids_of_this_type), doc_type, domain) print 'copy docs complete'
def _domain_to_change_meta(domain): domain_doc = domain.to_json() return change_meta_from_doc( document=domain_doc, data_source_type=data_sources.COUCH, data_source_name=Domain.get_db().dbname, )
def purge_locations(domain): """ Delete all location data associated with <domain>. This means Locations, SQLLocations, LocationTypes, and anything which has a ForeignKey relationship to SQLLocation (as of 2015-03-02, this includes only StockStates and some custom stuff). """ location_ids = set([r['id'] for r in Location.get_db().view( 'locations/by_type', reduce=False, startkey=[domain], endkey=[domain, {}], ).all()]) iter_bulk_delete(Location.get_db(), location_ids) for loc in SQLLocation.objects.filter(domain=domain).iterator(): if loc.supply_point_id: case = CommCareCase.get(loc.supply_point_id) case.delete() loc.delete() db = Domain.get_db() domain_obj = Domain.get_by_name(domain) # cached lookup is fast but stale domain_json = db.get(domain_obj._id) # get latest raw, unwrapped doc domain_json.pop('obsolete_location_types', None) domain_json.pop('location_types', None) db.save_doc(domain_json)
def _get_domains_without_last_modified_date(self): docs = iter_docs(Domain.get_db(), [ domain['id'] for domain in Domain.view( "domain/domains", reduce=False, include_docs=False) ]) return filter( lambda x: 'last_modified' not in x or not x['last_modified'], docs)
def deleted_domain_exists(domain): if isinstance(domain, Domain): domain = domain.name # double check just in case row = Domain.get_db().view('domain/deleted_domains', key=domain, reduce=True).one() return bool(row)
def setUpModule(): if isinstance(Domain.get_db(), Mock): # needed to skip setUp for javascript tests thread on Travis return _call_center_domain_mock = mock.patch( 'corehq.apps.callcenter.data_source.call_center_data_source_configuration_provider' ) _call_center_domain_mock.start() domain = create_domain('champ-cameroon') with override_settings(SERVER_ENVIRONMENT='production'): configs = StaticDataSourceConfiguration.by_domain(domain.name) adapters = [get_indicator_adapter(config) for config in configs] for adapter in adapters: adapter.build_table() engine = connection_manager.get_engine(UCR_ENGINE_ID) metadata = sqlalchemy.MetaData(bind=engine) metadata.reflect(bind=engine, extend_existing=True) path = os.path.join(os.path.dirname(__file__), 'fixtures') for file_name in os.listdir(path): with open(os.path.join(path, file_name), encoding='utf-8') as f: table_name = get_table_name(domain.name, file_name[:-4]) table = metadata.tables[table_name] postgres_copy.copy_from( f, table, engine, format='csv' if six.PY3 else b'csv', null='' if six.PY3 else b'', header=True ) _call_center_domain_mock.stop()
def check_exchange_index(): latest_snapshot = Domain.get_db().view('domain/published_snapshots', limit=1, descending=True, include_docs=True).one() if latest_snapshot is not None: doc_id = latest_snapshot['id'] couch_rev = latest_snapshot['doc']['_rev'] return _check_es_rev(ExchangePillow.es_index, doc_id, couch_rev) else: return {"%s_status" % ExchangePillow.es_index: False, "%s_message" % ExchangePillow.es_index: "Exchange stale" }
def generate_invoices(based_on_date=None, check_existing=False, is_test=False): """ Generates all invoices for the past month. """ today = based_on_date or datetime.date.today() invoice_start, invoice_end = utils.get_previous_month_date_range(today) logger.info("[Billing] Starting up invoices for %(start)s - %(end)s" % { 'start': invoice_start.strftime("%d %B %Y"), 'end': invoice_end.strftime("%d %B %Y"), }) all_domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)] for domain_doc in iter_docs(Domain.get_db(), all_domain_ids): domain = Domain.wrap(domain_doc) if (check_existing and Invoice.objects.filter( subscription__subscriber__domain=domain, date_created__gte=today).count() != 0): pass elif is_test: logger.info("[Billing] Ready to create invoice for domain %s" % domain.name) else: try: invoice_factory = DomainInvoiceFactory( invoice_start, invoice_end, domain) invoice_factory.create_invoices() logger.info("[BILLING] Sent invoices for domain %s" % domain.name) except CreditLineError as e: logger.error( "[BILLING] There was an error utilizing credits for " "domain %s: %s" % (domain.name, e) ) except BillingContactInfoError as e: subject = "[%s] Invoice Generation Issue" % domain.name email_content = render_to_string( 'accounting/invoice_error_email.html', { 'project': domain.name, 'error_msg': 'BillingContactInfoError: %s' % e, } ) send_HTML_email( subject, settings.BILLING_EMAIL, email_content, email_from="Dimagi Billing Bot <%s>" % settings.DEFAULT_FROM_EMAIL ) except InvoiceError as e: logger.error( "[BILLING] Could not create invoice for domain %s: %s" % ( domain.name, e )) except Exception as e: logger.error( "[BILLING] Error occurred while creating invoice for " "domain %s: %s" % (domain.name, e) ) # And finally... if not is_test: send_bookkeeper_email()
def setUpClass(cls): super(AppStatusIntegrationTest, cls).setUpClass() delete_all_docs_by_doc_type(Domain.get_db(), ['Domain', 'Domain-Deleted']) delete_all_docs_by_doc_type(CommCareUser.get_db(), ['CommCareUser', 'WebUser']) delete_all_docs_by_doc_type(Application.get_db(), ['Application', 'Application-Deleted']) cls.domain_records = [ Domain(name=cls.domain, hr_name='One', creating_user_id='abc', is_active=True), ] for domain in cls.domain_records: domain.save() cls.user_records = [ # TODO: Handle WebUsers who have multiple domains # WebUser.create( # cls.domain, # 'web-user', # '***', # date_joined=datetime.utcnow(), # first_name='A', # last_name='B', # email='*****@*****.**', # is_active=True, # is_staff=False, # is_superuser=True, # ), CommCareUser.create( cls.domain, 'commcare-user', '***', date_joined=datetime.utcnow(), email='*****@*****.**', is_active=True, is_staff=True, is_superuser=False, ), ] cls.form_records = [ create_form_for_test(cls.domain, user_id=cls.user_records[0]._id), create_form_for_test(cls.domain, user_id=cls.user_records[0]._id), create_form_for_test(cls.domain, user_id=cls.user_records[0]._id), ] cls.sync_records = [] for user in cls.user_records: restore_user = OTARestoreCommCareUser(user.domain, user) device = MockDevice(cls.domain_records[0], restore_user) cls.sync_records.append(device.sync()) cls.batch = create_batch(cls.slug)
def _get_domains_without_last_modified_date(self): docs = iter_docs(Domain.get_db(), [ domain['id'] for domain in Domain.view( "domain/domains", reduce=False, include_docs=False ) ]) return [x for x in docs if 'last_modified' not in x or not x['last_modified']]
def db_comparisons(request): comparison_config = [{ 'description': 'Users (base_doc is "CouchUser")', 'couch_db': CommCareUser.get_db(), 'view_name': 'users/by_username', 'es_query': UserES().remove_default_filter('active').remove_default_filter( 'mobile_worker').size(0), 'sql_rows': User.objects.count(), }, { 'description': 'Domains (doc_type is "Domain")', 'couch_db': Domain.get_db(), 'view_name': 'domain/by_status', 'es_query': DomainES().size(0), 'sql_rows': None, }, { 'description': 'Forms (doc_type is "XFormInstance")', 'couch_db': XFormInstance.get_db(), 'view_name': 'couchforms/by_xmlns', 'es_query': FormES().remove_default_filter('has_xmlns').remove_default_filter( 'has_user').size(0), 'sql_rows': FormData.objects.count(), }, { 'description': 'Cases (doc_type is "CommCareCase")', 'couch_db': CommCareCase.get_db(), 'view_name': 'case/by_owner', 'es_query': CaseES().size(0), 'sql_rows': None, }] comparisons = [] for comp in comparison_config: comparisons.append({ 'description': comp['description'], 'couch_docs': comp['couch_db'].view( comp['view_name'], reduce=True, ).one()['value'], 'es_docs': comp['es_query'].run().total, 'sql_rows': comp['sql_rows'] if comp['sql_rows'] else 'n/a', }) return json_response(comparisons)
def generate_invoices(based_on_date=None, check_existing=False, is_test=False): """ Generates all invoices for the past month. """ today = based_on_date or datetime.date.today() invoice_start, invoice_end = utils.get_previous_month_date_range(today) logger.info( "[Billing] Starting up invoices for %(start)s - %(end)s" % { 'start': invoice_start.strftime("%d %B %Y"), 'end': invoice_end.strftime("%d %B %Y"), }) all_domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)] for domain_doc in iter_docs(Domain.get_db(), all_domain_ids): domain = Domain.wrap(domain_doc) if (check_existing and Invoice.objects.filter(subscription__subscriber__domain=domain, date_created__gte=today).count() != 0): pass elif is_test: logger.info("[Billing] Ready to create invoice for domain %s" % domain.name) else: try: invoice_factory = DomainInvoiceFactory(invoice_start, invoice_end, domain) invoice_factory.create_invoices() logger.info("[BILLING] Sent invoices for domain %s" % domain.name) except CreditLineError as e: logger.error( "[BILLING] There was an error utilizing credits for " "domain %s: %s" % (domain.name, e)) except BillingContactInfoError as e: subject = "[%s] Invoice Generation Issue" % domain.name email_content = render_to_string( 'accounting/invoice_error_email.html', { 'project': domain.name, 'error_msg': 'BillingContactInfoError: %s' % e, }) send_HTML_email(subject, settings.BILLING_EMAIL, email_content, email_from="Dimagi Billing Bot <%s>" % settings.DEFAULT_FROM_EMAIL) except InvoiceError as e: logger.error( "[BILLING] Could not create invoice for domain %s: %s" % (domain.name, e)) except Exception as e: logger.error( "[BILLING] Error occurred while creating invoice for " "domain %s: %s" % (domain.name, e)) # And finally... if not is_test: send_bookkeeper_email()
def tearDownModule(): if isinstance(Domain.get_db(), Mock): # needed to skip setUp for javascript tests thread on Travis return _call_center_domain_mock = mock.patch( 'corehq.apps.callcenter.data_source.call_center_data_source_configuration_provider' ) _call_center_domain_mock.start() Domain.get_by_name('champ-cameroon').delete() _call_center_domain_mock.stop()
def query(self): db = Domain.get_db() startkey = self.search_string endkey = "{}Z".format(self.search_string) if startkey else '' query = db.view( 'domain/domains', reduce=False, startkey=startkey, endkey=endkey, limit=20, ) return query
def query(self): db = Domain.get_db() search_params = {'reduce': False, 'limit': 20} if self.search_string: # Search by string range: https://docs.couchdb.org/en/latest/ddocs/views/collation.html#string-ranges search_params['startkey'] = self.search_string search_params['endkey'] = "{}\ufff0".format(self.search_string) query = db.view('domain/domains', **search_params) return query
def handle(self, *args, **options): def update_domain(doc): doc['location_restriction_for_users'] = False return DocUpdate(doc) domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)] res = iter_update(Domain.get_db(), update_domain, domain_ids) print "number of domains updated:", len(res.updated_ids) print "Domains that already had it disabled:", res.ignored_ids print "Not Found:", res.not_found_ids print "Deleted (sure as shit better be empty):", res.deleted_ids print "Errored:", res.error_ids
def handle(self, *args, **options): raise CommandError( 'copy_group_data is currently broken. ' 'Ask Danny or Ethan to fix it along the lines of ' 'https://github.com/dimagi/commcare-hq/pull/9180/files#diff-9d976dc051a36a028c6604581dfbce5dR95' ) if len(args) != 2: raise CommandError('Usage is copy_group_data %s' % self.args) sourcedb = Database(args[0]) group_id = args[1] exclude_user_owned = options["exclude_user_owned"] print 'getting group' group = Group.wrap(sourcedb.get(group_id)) group.save(force_update=True) print 'getting domain' domain = Domain.wrap( sourcedb.view('domain/domains', key=group.domain, include_docs=True, reduce=False, limit=1).one()['doc']) dt = DocumentTransform(domain._obj, sourcedb) save(dt, Domain.get_db()) owners = [group_id] if not exclude_user_owned: owners.extend(group.users) print 'getting case ids' with OverrideDB(CommCareCase, sourcedb): case_ids = get_case_ids_in_domain_by_owner(domain.name, owner_id__in=owners) xform_ids = set() print 'copying %s cases' % len(case_ids) for i, subset in enumerate(chunked(case_ids, CHUNK_SIZE)): print i * CHUNK_SIZE cases = [ CommCareCase.wrap(case['doc']) for case in sourcedb.all_docs( keys=list(subset), include_docs=True, ) ] for case in cases:
def dump(self, output_stream): from corehq.apps.domain.models import Domain domain_obj = Domain.get_by_name(self.domain, strict=True) if not domain_obj: raise DomainDumpError("Domain not found: {}".format(self.domain)) domain_dict = _get_doc_with_attachments(Domain.get_db(), domain_obj.to_json()) domain_obj = Domain.wrap(domain_dict) json.dump(domain_obj.to_json(), output_stream) output_stream.write('\n') self.stdout.write('Dumping {} Domain\n'.format(1)) return Counter({'Domain': 1})
def generate_invoices(based_on_date=None, check_existing=False, is_test=False): """ Generates all invoices for the past month. """ today = based_on_date or datetime.date.today() invoice_start, invoice_end = utils.get_previous_month_date_range(today) logger.info("[Billing] Starting up invoices for %(start)s - %(end)s" % { 'start': invoice_start.strftime(USER_DATE_FORMAT), 'end': invoice_end.strftime(USER_DATE_FORMAT), }) all_domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)] for domain_doc in iter_docs(Domain.get_db(), all_domain_ids): domain = Domain.wrap(domain_doc) if (check_existing and Invoice.objects.filter( subscription__subscriber__domain=domain, date_created__gte=today).count() != 0): pass elif is_test: logger.info("[Billing] Ready to create invoice for domain %s" % domain.name) else: try: invoice_factory = DomainInvoiceFactory( invoice_start, invoice_end, domain) invoice_factory.create_invoices() logger.info("[BILLING] Sent invoices for domain %s" % domain.name) except CreditLineError as e: logger.error( "[BILLING] There was an error utilizing credits for " "domain %s: %s" % (domain.name, e) ) except BillingContactInfoError as e: logger.info("BillingContactInfoError: %s" % e) except InvoiceError as e: logger.error( "[BILLING] Could not create invoice for domain %s: %s" % ( domain.name, e )) except InvoiceAlreadyCreatedError as e: logger.error( "[BILLING] Invoice already existed for domain %s: %s" % ( domain.name, e )) except Exception as e: logger.error( "[BILLING] Error occurred while creating invoice for " "domain %s: %s" % (domain.name, e) )
def build(self): return ElasticPillowReindexer( pillow_or_processor=get_domain_kafka_to_elasticsearch_pillow(), change_provider=CouchViewChangeProvider( couch_db=Domain.get_db(), view_name='all_docs/by_doc_type', view_kwargs={ 'startkey': ['Domain'], 'endkey': ['Domain', {}], 'include_docs': True, }), elasticsearch=get_es_new(), index_info=DOMAIN_INDEX_INFO, **self.options)
def _assign_explicit_community_subscriptions(apps, schema_editor): today = date.today() all_domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)] for domain_doc in iter_docs(Domain.get_db(), all_domain_ids): domain_name = domain_doc['name'] from_date = date(today.year, today.month, 1) try: while from_date <= today: ensure_explicit_community_subscription(domain_name, from_date) from_date += timedelta(days=1) except Exception as e: log_accounting_error( "During community subscription assignment for domain %s: %s" % (domain_name, e.message))
def load_objects(self, object_strings, force=False): from corehq.apps.domain.models import Domain objects = list(object_strings) assert len(objects) == 1, "Only 1 domain allowed per dump" domain_dict = json.loads(objects[0]) domain_name = domain_dict['name'] try: existing_domain = Domain.get_by_name(domain_name, strict=True) except ResourceNotFound: pass else: if existing_domain: if force: self.stderr.write('Loading data for existing domain: {}'.format(domain_name)) else: raise DataExistsException("Domain: {}".format(domain_name)) Domain.get_db().bulk_save([domain_dict], new_edits=False) self.stdout.write('Loaded Domain') return 1, Counter({'Domain': 1})
def check_exchange_index(): latest_snapshot = Domain.get_db().view('domain/published_snapshots', limit=1, descending=True, include_docs=True).one() if latest_snapshot is not None: doc_id = latest_snapshot['id'] couch_rev = latest_snapshot['doc']['_rev'] return _check_es_rev(ExchangePillow.es_index, doc_id, couch_rev) else: return { "%s_status" % ExchangePillow.es_index: False, "%s_message" % ExchangePillow.es_index: "Exchange stale" }
def _assign_explicit_community_subscriptions(apps, schema_editor): today = date.today() all_domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)] for domain_doc in iter_docs(Domain.get_db(), all_domain_ids): domain_name = domain_doc['name'] from_date = date(today.year, today.month, 1) try: while from_date <= today: ensure_explicit_community_subscription(domain_name, from_date) from_date += timedelta(days=1) except Exception as e: log_accounting_error( "During community subscription assignment for domain %s: %s" % (domain_name, e.message) )
def handle(self, *args, **options): domain = args[0] old_db = Domain.get_db() new_db = RepeatRecord.get_db() assert old_db.dbname != new_db.dbname doc_ids = get_doc_ids_in_domain_by_type(domain, 'RepeatRecord', old_db) count = len(doc_ids) chunksize = 250 for i, docs in enumerate(chunked(iter_docs(old_db, doc_ids, chunksize), chunksize)): for doc in docs: if '_rev' in doc: del doc['_rev'] new_db.bulk_save(docs, new_edits=False) print 'checked %s / %s' % (i * chunksize, count)
def get_domain_reindexer(): return ElasticPillowReindexer( pillow=get_domain_kafka_to_elasticsearch_pillow(), change_provider=CouchViewChangeProvider( couch_db=Domain.get_db(), view_name='all_docs/by_doc_type', view_kwargs={ 'startkey': ['Domain'], 'endkey': ['Domain', {}], 'include_docs': True, } ), elasticsearch=get_es_new(), index_info=DOMAIN_INDEX_INFO, )
def handle(self, *args, **options): if len(args) != 1: raise CommandError("Usage is ./manage.py migrate_domain_bu [migration_file]!") updates = {} with open(args[0], 'rb') as csvfile: reader = csv.reader(csvfile, delimiter=',') for row in reader: updates[Domain.get_by_name(row[0])['_id']] = row[1] def update_domain(doc): Domain.wrap(doc).internal.business_unit = updates[doc['_id']] return DocUpdate(doc) iter_update(Domain.get_db(), update_domain, updates.keys(), verbose=True)
def db_comparisons(request): def _simple_view_couch_query(db, view_name): return db.view(view_name, reduce=True).one()['value'] def _count_real_forms(): all_forms = _simple_view_couch_query(XFormInstance.get_db(), 'couchforms/by_xmlns') device_logs = XFormInstance.get_db().view('couchforms/by_xmlns', key=DEVICE_LOG_XMLNS).one()['value'] return all_forms - device_logs comparison_config = [ { 'description': 'Users (base_doc is "CouchUser")', 'couch_docs': _simple_view_couch_query(CommCareUser.get_db(), 'users/by_username'), 'es_query': UserES().remove_default_filter('active').size(0), 'sql_rows': User.objects.count(), }, { 'description': 'Domains (doc_type is "Domain")', 'couch_docs': _simple_view_couch_query(Domain.get_db(), 'domain/by_status'), 'es_query': DomainES().size(0), 'sql_rows': None, }, { 'description': 'Forms (doc_type is "XFormInstance")', 'couch_docs': _count_real_forms(), 'es_query': FormES().remove_default_filter('has_xmlns') .remove_default_filter('has_user') .size(0), 'sql_rows': FormData.objects.count(), }, { 'description': 'Cases (doc_type is "CommCareCase")', 'couch_docs': _simple_view_couch_query(CommCareCase.get_db(), 'case/by_owner'), 'es_query': CaseES().size(0), 'sql_rows': CaseData.objects.count(), } ] comparisons = [] for comp in comparison_config: comparisons.append({ 'description': comp['description'], 'couch_docs': comp['couch_docs'], 'es_docs': comp['es_query'].run().total, 'sql_rows': comp['sql_rows'] if comp['sql_rows'] else 'n/a', }) return json_response(comparisons)
def db_comparisons(request): comparison_config = [ { 'description': 'Users (base_doc is "CouchUser")', 'couch_db': CommCareUser.get_db(), 'view_name': 'users/by_username', 'es_query': UserES().remove_default_filter('active') .remove_default_filter('mobile_worker') .size(0), 'sql_rows': User.objects.count(), }, { 'description': 'Domains (doc_type is "Domain")', 'couch_db': Domain.get_db(), 'view_name': 'domain/by_status', 'es_query': DomainES().size(0), 'sql_rows': None, }, { 'description': 'Forms (doc_type is "XFormInstance")', 'couch_db': XFormInstance.get_db(), 'view_name': 'couchforms/by_xmlns', 'es_query': FormES().remove_default_filter('has_xmlns') .remove_default_filter('has_user') .size(0), 'sql_rows': FormData.objects.count(), }, { 'description': 'Cases (doc_type is "CommCareCase")', 'couch_db': CommCareCase.get_db(), 'view_name': 'case/by_owner', 'es_query': CaseES().size(0), 'sql_rows': None, } ] comparisons = [] for comp in comparison_config: comparisons.append({ 'description': comp['description'], 'couch_docs': comp['couch_db'].view( comp['view_name'], reduce=True, ).one()['value'], 'es_docs': comp['es_query'].run().total, 'sql_rows': comp['sql_rows'] if comp['sql_rows'] else 'n/a', }) return json_response(comparisons)
def count_downloads_for_all_snapshots(domain_id): """ domain_id should represent an actual (non-snapshot) domain for this to make sense but that is not checked. It'll just be 0 otherwise. """ try: return Domain.get_db().view( 'domain/snapshots', startkey=[domain_id], endkey=[domain_id, {}], reduce=True, include_docs=False, ).one()["value"] except TypeError: return 0
def handle(self, *args, **options): db = Domain.get_db() def get_doc_ids(): for result in db.view("domain/domains", reduce=False).all(): yield result["id"] for result in ApplicationBase.get_db().view( "app_manager/applications", startkey=[None], endkey=[None, {}], reduce=False ): yield result["id"] for doc in iter_docs(db, get_doc_ids()): if "secure_submissions" not in doc: print "Updated", doc.get("doc_type"), doc.get("_id") doc["secure_submissions"] = False db.save_doc(doc)
def db_comparisons(request): def _simple_view_couch_query(db, view_name): return db.view(view_name, reduce=True).one()["value"] def _count_real_forms(): all_forms = _simple_view_couch_query(XFormInstance.get_db(), "couchforms/by_xmlns") device_logs = XFormInstance.get_db().view("couchforms/by_xmlns", key=DEVICE_LOG_XMLNS).one()["value"] return all_forms - device_logs comparison_config = [ { "description": 'Users (base_doc is "CouchUser")', "couch_docs": _simple_view_couch_query(CommCareUser.get_db(), "users/by_username"), "es_query": UserES().remove_default_filter("active").size(0), "sql_rows": User.objects.count(), }, { "description": 'Domains (doc_type is "Domain")', "couch_docs": _simple_view_couch_query(Domain.get_db(), "domain/by_status"), "es_query": DomainES().size(0), "sql_rows": None, }, { "description": 'Forms (doc_type is "XFormInstance")', "couch_docs": _count_real_forms(), "es_query": FormES().remove_default_filter("has_xmlns").remove_default_filter("has_user").size(0), "sql_rows": FormData.objects.count(), }, { "description": 'Cases (doc_type is "CommCareCase")', "couch_docs": _simple_view_couch_query(CommCareCase.get_db(), "case/by_owner"), "es_query": CaseES().size(0), "sql_rows": CaseData.objects.count(), }, ] comparisons = [] for comp in comparison_config: comparisons.append( { "description": comp["description"], "couch_docs": comp["couch_docs"], "es_docs": comp["es_query"].run().total, "sql_rows": comp["sql_rows"] if comp["sql_rows"] else "n/a", } ) return json_response(comparisons)
def setUpClass(cls): super(FormFactIntegrationTest, cls).setUpClass() delete_all_docs_by_doc_type(Domain.get_db(), ['Domain', 'Domain-Deleted']) delete_all_docs_by_doc_type(CommCareUser.get_db(), ['CommCareUser', 'WebUser']) cls.domain_records = [ Domain(name=cls.domain, hr_name='One', creating_user_id='abc', is_active=True), ] for domain in cls.domain_records: domain.save() cls.user_records = [ # TODO: Handle WebUsers who have multiple domains # WebUser.create( # cls.domain, # 'web-user', # '***', # date_joined=datetime.utcnow(), # first_name='A', # last_name='B', # email='*****@*****.**', # is_active=True, # is_staff=False, # is_superuser=True, # ), CommCareUser.create( cls.domain, 'commcare-user', '***', date_joined=datetime.utcnow(), email='*****@*****.**', is_active=True, is_staff=True, is_superuser=False, ), ] cls.form_records = [ create_form_for_test(cls.domain, user_id=cls.user_records[0]._id), create_form_for_test(cls.domain, user_id=cls.user_records[0]._id), create_form_for_test(cls.domain, user_id=cls.user_records[0]._id), ]
def repl_domains(self): prod_db = get_prod_db(self.source_uri) domain_docs = [prod_db.view('domain/domains', key=x, include_docs=True, limit=1, reduce=False).one() for x in self.domain_list] domain_docs = filter(lambda x: x is not None, domain_docs) domain_doc_ids = [x['doc']['_id'] for x in domain_docs] if len(domain_doc_ids) == 0: print "No domains matching your request found, exiting to prevent a full db replication" sys.exit() params = { 'continuous': True, 'doc_ids': domain_doc_ids } target_server = Domain.get_db().server self.do_repl(target_server, params)
def tearDownModule(): if isinstance(Domain.get_db(), Mock): # needed to skip setUp for javascript tests thread on Travis return _call_center_domain_mock = mock.patch( 'corehq.apps.callcenter.data_source.call_center_data_source_configuration_provider' ) _call_center_domain_mock.start() configs = StaticDataSourceConfiguration.by_domain('champ-cameroon') adapters = [get_indicator_adapter(config) for config in configs] for adapter in adapters: adapter.drop_table() Domain.get_by_name('champ-cameroon').delete() _call_center_domain_mock.stop()
def handle(self, dry_run=False, log_file=False, *args, **options): if not dry_run: confirm_force_reset = raw_input( "Are you sure you want to apply downgrades and upgrades to " "ALL domains? Type 'yes' to continue. \n") if confirm_force_reset != 'yes': return if log_file: orig_stdout = sys.stdout f = file('subscription_changes.txt', 'w') sys.stdout = f all_domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)] for domain_doc in iter_docs(Domain.get_db(), all_domain_ids): domain = Domain.wrap(domain_doc) logging.info("%s START" % domain.name) print("\n") plan_version, subscription = Subscription.get_subscribed_plan_by_domain( domain) if not subscription: subscriber = Subscriber.objects.get_or_create( domain=domain.name)[0] print("Active subscription not found for domain %s" % domain.name) else: subscriber = subscription.subscriber if not dry_run: print("%s => %s" % (domain.name, plan_version.plan.name)) subscriber.apply_upgrades_and_downgrades( new_plan_version=plan_version, verbose=True, new_subscription=subscription, ) else: print("[DRY RUN] %s => %s" % (domain.name, plan_version.plan.name)) logging.info("%s END" % domain.name) if log_file: sys.stdout = orig_stdout f.close()
def handle(self, dry_run=False, log_file=False, *args, **options): if not dry_run: confirm_force_reset = raw_input( "Are you sure you want to apply downgrades and upgrades to " "ALL domains? Type 'yes' to continue. \n" ) if confirm_force_reset != 'yes': return if log_file: orig_stdout = sys.stdout f = file('subscription_changes.txt', 'w') sys.stdout = f all_domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)] for domain_doc in iter_docs(Domain.get_db(), all_domain_ids): domain = Domain.wrap(domain_doc) logging.info("%s START" % domain.name) print ("\n") plan_version, subscription = Subscription.get_subscribed_plan_by_domain(domain) if not subscription: subscriber = Subscriber.objects.get_or_create(domain=domain.name)[0] print ("Active subscription not found for domain %s" % domain.name) else: subscriber = subscription.subscriber if not dry_run: print ("%s => %s" % (domain.name, plan_version.plan.name)) subscriber.apply_upgrades_and_downgrades( new_plan_version=plan_version, verbose=True, new_subscription=subscription, ) else: print ("[DRY RUN] %s => %s" % (domain.name, plan_version.plan.name)) logging.info("%s END" % domain.name) if log_file: sys.stdout = orig_stdout f.close()
def handle(self, doc_types, *args, **options): input = raw_input('\n'.join([ '\n\nReally delete documents of the following types: {}?', 'This operation is not reversible. Enter a number N to delete the first ' 'N found, or type "delete all" to delete everything.', '', ]).format(doc_types)) if input == 'delete all': remaining = None else: try: remaining = int(input) except ValueError: print 'aborting' sys.exit() doc_types = doc_types.split(',') deleted = 0 # unfortunately the only couch view we have for this needs to go by domain # will be a bit slow domain_names = Domain.get_all_names() database = Domain.get_db() for domain in domain_names: for doc_type in doc_types: docs = [row['doc'] for row in database.view( 'domain/docs', startkey=[domain, doc_type], endkey=[domain, doc_type, {}], reduce=False, include_docs=True, )][:remaining] if docs: count = len(docs) print 'deleting {} {}s from {}'.format(count, doc_type, domain) database.delete_docs(docs) deleted += count if remaining is not None: remaining -= count if remaining <= 0: return print 'successfully deleted {} documents'.format(deleted)
def db_comparisons(request): def _simple_view_couch_query(db, view_name): return db.view(view_name, reduce=True).one()['value'] comparison_config = [ { 'description': 'Users (base_doc is "CouchUser")', 'couch_docs': _simple_view_couch_query(CommCareUser.get_db(), 'users/by_username'), 'es_query': UserES().remove_default_filter('active').size(0), 'sql_rows': User.objects.count(), }, { 'description': 'Domains (doc_type is "Domain")', 'couch_docs': _simple_view_couch_query(Domain.get_db(), 'domain/by_status'), 'es_query': DomainES().size(0), 'sql_rows': None, }, { 'description': 'Forms (doc_type is "XFormInstance")', 'couch_docs': get_number_of_forms_all_domains_in_couch(), 'es_query': FormES().remove_default_filter('has_xmlns') .remove_default_filter('has_user') .size(0), 'sql_rows': FormData.objects.exclude(domain__isnull=True).count(), }, { 'description': 'Cases (doc_type is "CommCareCase")', 'couch_docs': get_total_case_count(), 'es_query': CaseES().size(0), 'sql_rows': CaseData.objects.exclude(domain__isnull=True).count(), } ] comparisons = [] for comp in comparison_config: comparisons.append({ 'description': comp['description'], 'couch_docs': comp['couch_docs'], 'es_docs': comp['es_query'].run().total, 'sql_rows': comp['sql_rows'] if comp['sql_rows'] else 'n/a', }) return json_response(comparisons)
def setUpClass(cls): super(FormFactIntegrationTest, cls).setUpClass() delete_all_docs_by_doc_type(Domain.get_db(), ['Domain', 'Domain-Deleted']) delete_all_docs_by_doc_type(CommCareUser.get_db(), ['CommCareUser', 'WebUser']) cls.domain_records = [ Domain(name=cls.domain, hr_name='One', creating_user_id='abc', is_active=True), ] for domain in cls.domain_records: domain.save() cls.user_records = [ # TODO: Handle WebUsers who have multiple domains # WebUser.create( # cls.domain, # 'web-user', # '***', # date_joined=datetime.utcnow(), # first_name='A', # last_name='B', # email='*****@*****.**', # is_active=True, # is_staff=False, # is_superuser=True, # ), CommCareUser.create( cls.domain, 'commcare-user', '***', date_joined=datetime.utcnow(), email='*****@*****.**', is_active=True, is_staff=True, is_superuser=False, ), ] cls.form_records = [ create_form_for_test(cls.domain, user_id=cls.user_records[0]._id), create_form_for_test(cls.domain, user_id=cls.user_records[0]._id), create_form_for_test(cls.domain, user_id=cls.user_records[0]._id), ] cls.batch = create_batch(cls.slug)
def tearDownModule(): if isinstance(Domain.get_db(), Mock): # needed to skip setUp for javascript tests thread on Travis return _call_center_domain_mock = mock.patch( 'corehq.apps.callcenter.data_source.call_center_data_source_configuration_provider' ) domain = Domain.get_by_name('test-pna') engine = connection_manager.get_engine(UCR_ENGINE_ID) metadata = sqlalchemy.MetaData(bind=engine) metadata.reflect(bind=engine, extend_existing=True) path = os.path.join(os.path.dirname(__file__), 'fixtures') for file_name in os.listdir(path): table_name = get_table_name(domain.name, file_name[:-4]) table = metadata.tables[table_name] table.drop() _call_center_domain_mock.start() domain.delete() _call_center_domain_mock.stop()