def tearDownClass(cls):
for user in cls.mobile_users + [cls.web_user]:
user.delete('state', deleted_by=None)
for domain_obj in cls.domains:
Domain.get_db().delete_doc(domain_obj)
ensure_index_deleted(USER_INDEX)
super().tearDownClass()
def load_objects(self, object_strings, force=False):
from corehq.apps.domain.models import Domain
objects = list(object_strings)
assert len(objects) == 1, "Only 1 domain allowed per dump"
domain_dict = json.loads(objects[0])
domain_name = domain_dict['name']
try:
existing_domain = Domain.get_by_name(domain_name, strict=True)
except ResourceNotFound:
pass
else:
if existing_domain:
if force:
self.stderr.write(
'Loading data for existing domain: {}'.format(
domain_name))
else:
raise DataExistsException("Domain: {}".format(domain_name))
Domain.get_db().bulk_save([domain_dict], new_edits=False)
self.stdout.write('Loaded Domain')
return 1, Counter({'Domain': 1})
def tearDownClass(cls):
cls.app.delete()
cls.registry.delete()
cls.restore_user._couch_user.delete(None, None)
cls.restore_user_domain_1._couch_user.delete(None, None)
Domain.get_db().delete_doc(cls.domain_obj.get_id)
super().tearDownClass()
def handle(self, *args, **kwargs):
domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)]
with IterativeSaver(Domain.get_db()) as iter_db:
for domain_doc in iter_docs(Domain.get_db(), domain_ids):
if (domain_doc.pop('location_types', None)
or domain_doc.pop('obsolete_location_types', None)):
print("Removing location_types from domain {} - {}".format(
domain_doc['name'], domain_doc['_id']))
iter_db.save(domain_doc)
def handle(self, *args, **kwargs):
domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)]
with IterativeSaver(Domain.get_db()) as iter_db:
for domain_doc in iter_docs(Domain.get_db(), domain_ids):
if (
domain_doc.pop('location_types', None) or
domain_doc.pop('obsolete_location_types', None)
):
print ("Removing location_types from domain {} - {}"
.format(domain_doc['name'], domain_doc['_id']))
iter_db.save(domain_doc)
def generate_invoices(based_on_date=None):
"""
Generates all invoices for the past month.
"""
today = based_on_date or datetime.date.today()
invoice_start, invoice_end = get_previous_month_date_range(today)
log_accounting_info(
"Starting up invoices for %(start)s - %(end)s" % {
'start': invoice_start.strftime(USER_DATE_FORMAT),
'end': invoice_end.strftime(USER_DATE_FORMAT),
})
all_domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)]
for domain_doc in iter_docs(Domain.get_db(), all_domain_ids):
domain = Domain.wrap(domain_doc)
try:
invoice_factory = DomainInvoiceFactory(invoice_start, invoice_end,
domain)
invoice_factory.create_invoices()
log_accounting_info("Sent invoices for domain %s" % domain.name)
except CreditLineError as e:
log_accounting_error("There was an error utilizing credits for "
"domain %s: %s" % (domain.name, e))
except InvoiceError as e:
log_accounting_error("Could not create invoice for domain %s: %s" %
(domain.name, e))
except InvoiceAlreadyCreatedError as e:
log_accounting_error("Invoice already existed for domain %s: %s" %
(domain.name, e))
except Exception as e:
log_accounting_error("Error occurred while creating invoice for "
"domain %s: %s" % (domain.name, e))
if not settings.UNIT_TESTING:
_invoicing_complete_soft_assert(False, "Invoicing is complete!")
def _domain_to_change_meta(domain):
domain_doc = domain.to_json()
return change_meta_from_doc(
document=domain_doc,
data_source_type=data_sources.SOURCE_COUCH,
data_source_name=Domain.get_db().dbname,
)
def iter_domains():
for row in paginate_view(Domain.get_db(),
'domain/domains',
100,
reduce=False,
include_docs=False):
yield row['key']
def _copy(config):
# unfortunately the only couch view we have for this needs to go by domain
# will be a bit slow
database = Domain.get_db()
assert database.uri == config.source_db.uri, 'can only use "copy" with the main HQ DB as the source'
domain_names = Domain.get_all_names()
for domain in domain_names:
for doc_type in config.doc_types:
ids_of_this_type = [row['id'] for row in database.view(
'domain/docs',
startkey=[domain, doc_type],
endkey=[domain, doc_type, {}],
reduce=False,
include_docs=False,
)]
if ids_of_this_type:
new_revs = dict([
(row['id'], row['value']['rev'])
for row in config.dest_db.view('_all_docs', keys=ids_of_this_type, include_docs=False)
if 'error' not in row
])
for id_group in chunked(ids_of_this_type, 500):
docs = get_docs(database, id_group)
for doc in docs:
if doc['_id'] in new_revs:
doc['_rev'] = new_revs[doc['_id']]
config.dest_db.bulk_save(docs)
print 'copied {} {}s from {}'.format(len(ids_of_this_type), doc_type, domain)
print 'copy docs complete'
def _domain_to_change_meta(domain):
domain_doc = domain.to_json()
return change_meta_from_doc(
document=domain_doc,
data_source_type=data_sources.COUCH,
data_source_name=Domain.get_db().dbname,
)
def purge_locations(domain):
"""
Delete all location data associated with <domain>.
This means Locations, SQLLocations, LocationTypes, and anything which
has a ForeignKey relationship to SQLLocation (as of 2015-03-02, this
includes only StockStates and some custom stuff).
"""
location_ids = set([r['id'] for r in Location.get_db().view(
'locations/by_type',
reduce=False,
startkey=[domain],
endkey=[domain, {}],
).all()])
iter_bulk_delete(Location.get_db(), location_ids)
for loc in SQLLocation.objects.filter(domain=domain).iterator():
if loc.supply_point_id:
case = CommCareCase.get(loc.supply_point_id)
case.delete()
loc.delete()
db = Domain.get_db()
domain_obj = Domain.get_by_name(domain) # cached lookup is fast but stale
domain_json = db.get(domain_obj._id) # get latest raw, unwrapped doc
domain_json.pop('obsolete_location_types', None)
domain_json.pop('location_types', None)
db.save_doc(domain_json)
def _get_domains_without_last_modified_date(self):
docs = iter_docs(Domain.get_db(), [
domain['id'] for domain in Domain.view(
"domain/domains", reduce=False, include_docs=False)
])
return filter(
lambda x: 'last_modified' not in x or not x['last_modified'], docs)
def deleted_domain_exists(domain):
if isinstance(domain, Domain):
domain = domain.name # double check just in case
row = Domain.get_db().view('domain/deleted_domains',
key=domain,
reduce=True).one()
return bool(row)
def setUpModule():
if isinstance(Domain.get_db(), Mock):
# needed to skip setUp for javascript tests thread on Travis
return
_call_center_domain_mock = mock.patch(
'corehq.apps.callcenter.data_source.call_center_data_source_configuration_provider'
)
_call_center_domain_mock.start()
domain = create_domain('champ-cameroon')
with override_settings(SERVER_ENVIRONMENT='production'):
configs = StaticDataSourceConfiguration.by_domain(domain.name)
adapters = [get_indicator_adapter(config) for config in configs]
for adapter in adapters:
adapter.build_table()
engine = connection_manager.get_engine(UCR_ENGINE_ID)
metadata = sqlalchemy.MetaData(bind=engine)
metadata.reflect(bind=engine, extend_existing=True)
path = os.path.join(os.path.dirname(__file__), 'fixtures')
for file_name in os.listdir(path):
with open(os.path.join(path, file_name), encoding='utf-8') as f:
table_name = get_table_name(domain.name, file_name[:-4])
table = metadata.tables[table_name]
postgres_copy.copy_from(
f, table, engine, format='csv' if six.PY3 else b'csv',
null='' if six.PY3 else b'', header=True
)
_call_center_domain_mock.stop()
def check_exchange_index():
latest_snapshot = Domain.get_db().view('domain/published_snapshots', limit=1, descending=True, include_docs=True).one()
if latest_snapshot is not None:
doc_id = latest_snapshot['id']
couch_rev = latest_snapshot['doc']['_rev']
return _check_es_rev(ExchangePillow.es_index, doc_id, couch_rev)
else:
return {"%s_status" % ExchangePillow.es_index: False, "%s_message" % ExchangePillow.es_index: "Exchange stale" }
def generate_invoices(based_on_date=None, check_existing=False, is_test=False):
"""
Generates all invoices for the past month.
"""
today = based_on_date or datetime.date.today()
invoice_start, invoice_end = utils.get_previous_month_date_range(today)
logger.info("[Billing] Starting up invoices for %(start)s - %(end)s" % {
'start': invoice_start.strftime("%d %B %Y"),
'end': invoice_end.strftime("%d %B %Y"),
})
all_domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)]
for domain_doc in iter_docs(Domain.get_db(), all_domain_ids):
domain = Domain.wrap(domain_doc)
if (check_existing and
Invoice.objects.filter(
subscription__subscriber__domain=domain,
date_created__gte=today).count() != 0):
pass
elif is_test:
logger.info("[Billing] Ready to create invoice for domain %s"
% domain.name)
else:
try:
invoice_factory = DomainInvoiceFactory(
invoice_start, invoice_end, domain)
invoice_factory.create_invoices()
logger.info("[BILLING] Sent invoices for domain %s"
% domain.name)
except CreditLineError as e:
logger.error(
"[BILLING] There was an error utilizing credits for "
"domain %s: %s" % (domain.name, e)
)
except BillingContactInfoError as e:
subject = "[%s] Invoice Generation Issue" % domain.name
email_content = render_to_string(
'accounting/invoice_error_email.html', {
'project': domain.name,
'error_msg': 'BillingContactInfoError: %s' % e,
}
)
send_HTML_email(
subject, settings.BILLING_EMAIL, email_content,
email_from="Dimagi Billing Bot <%s>" % settings.DEFAULT_FROM_EMAIL
)
except InvoiceError as e:
logger.error(
"[BILLING] Could not create invoice for domain %s: %s" % (
domain.name, e
))
except Exception as e:
logger.error(
"[BILLING] Error occurred while creating invoice for "
"domain %s: %s" % (domain.name, e)
)
# And finally...
if not is_test:
send_bookkeeper_email()
def setUpClass(cls):
super(AppStatusIntegrationTest, cls).setUpClass()
delete_all_docs_by_doc_type(Domain.get_db(),
['Domain', 'Domain-Deleted'])
delete_all_docs_by_doc_type(CommCareUser.get_db(),
['CommCareUser', 'WebUser'])
delete_all_docs_by_doc_type(Application.get_db(),
['Application', 'Application-Deleted'])
cls.domain_records = [
Domain(name=cls.domain,
hr_name='One',
creating_user_id='abc',
is_active=True),
]
for domain in cls.domain_records:
domain.save()
cls.user_records = [
# TODO: Handle WebUsers who have multiple domains
# WebUser.create(
# cls.domain,
# 'web-user',
# '***',
# date_joined=datetime.utcnow(),
# first_name='A',
# last_name='B',
# email='*****@*****.**',
# is_active=True,
# is_staff=False,
# is_superuser=True,
# ),
CommCareUser.create(
cls.domain,
'commcare-user',
'***',
date_joined=datetime.utcnow(),
email='*****@*****.**',
is_active=True,
is_staff=True,
is_superuser=False,
),
]
cls.form_records = [
create_form_for_test(cls.domain, user_id=cls.user_records[0]._id),
create_form_for_test(cls.domain, user_id=cls.user_records[0]._id),
create_form_for_test(cls.domain, user_id=cls.user_records[0]._id),
]
cls.sync_records = []
for user in cls.user_records:
restore_user = OTARestoreCommCareUser(user.domain, user)
device = MockDevice(cls.domain_records[0], restore_user)
cls.sync_records.append(device.sync())
cls.batch = create_batch(cls.slug)
def _get_domains_without_last_modified_date(self):
docs = iter_docs(Domain.get_db(), [
domain['id']
for domain in Domain.view(
"domain/domains",
reduce=False,
include_docs=False
)
])
return [x for x in docs if 'last_modified' not in x or not x['last_modified']]
def db_comparisons(request):
comparison_config = [{
'description':
'Users (base_doc is "CouchUser")',
'couch_db':
CommCareUser.get_db(),
'view_name':
'users/by_username',
'es_query':
UserES().remove_default_filter('active').remove_default_filter(
'mobile_worker').size(0),
'sql_rows':
User.objects.count(),
}, {
'description': 'Domains (doc_type is "Domain")',
'couch_db': Domain.get_db(),
'view_name': 'domain/by_status',
'es_query': DomainES().size(0),
'sql_rows': None,
}, {
'description':
'Forms (doc_type is "XFormInstance")',
'couch_db':
XFormInstance.get_db(),
'view_name':
'couchforms/by_xmlns',
'es_query':
FormES().remove_default_filter('has_xmlns').remove_default_filter(
'has_user').size(0),
'sql_rows':
FormData.objects.count(),
}, {
'description': 'Cases (doc_type is "CommCareCase")',
'couch_db': CommCareCase.get_db(),
'view_name': 'case/by_owner',
'es_query': CaseES().size(0),
'sql_rows': None,
}]
comparisons = []
for comp in comparison_config:
comparisons.append({
'description':
comp['description'],
'couch_docs':
comp['couch_db'].view(
comp['view_name'],
reduce=True,
).one()['value'],
'es_docs':
comp['es_query'].run().total,
'sql_rows':
comp['sql_rows'] if comp['sql_rows'] else 'n/a',
})
return json_response(comparisons)
def generate_invoices(based_on_date=None, check_existing=False, is_test=False):
"""
Generates all invoices for the past month.
"""
today = based_on_date or datetime.date.today()
invoice_start, invoice_end = utils.get_previous_month_date_range(today)
logger.info(
"[Billing] Starting up invoices for %(start)s - %(end)s" % {
'start': invoice_start.strftime("%d %B %Y"),
'end': invoice_end.strftime("%d %B %Y"),
})
all_domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)]
for domain_doc in iter_docs(Domain.get_db(), all_domain_ids):
domain = Domain.wrap(domain_doc)
if (check_existing and
Invoice.objects.filter(subscription__subscriber__domain=domain,
date_created__gte=today).count() != 0):
pass
elif is_test:
logger.info("[Billing] Ready to create invoice for domain %s" %
domain.name)
else:
try:
invoice_factory = DomainInvoiceFactory(invoice_start,
invoice_end, domain)
invoice_factory.create_invoices()
logger.info("[BILLING] Sent invoices for domain %s" %
domain.name)
except CreditLineError as e:
logger.error(
"[BILLING] There was an error utilizing credits for "
"domain %s: %s" % (domain.name, e))
except BillingContactInfoError as e:
subject = "[%s] Invoice Generation Issue" % domain.name
email_content = render_to_string(
'accounting/invoice_error_email.html', {
'project': domain.name,
'error_msg': 'BillingContactInfoError: %s' % e,
})
send_HTML_email(subject,
settings.BILLING_EMAIL,
email_content,
email_from="Dimagi Billing Bot <%s>" %
settings.DEFAULT_FROM_EMAIL)
except InvoiceError as e:
logger.error(
"[BILLING] Could not create invoice for domain %s: %s" %
(domain.name, e))
except Exception as e:
logger.error(
"[BILLING] Error occurred while creating invoice for "
"domain %s: %s" % (domain.name, e))
# And finally...
if not is_test:
send_bookkeeper_email()
def tearDownModule():
if isinstance(Domain.get_db(), Mock):
# needed to skip setUp for javascript tests thread on Travis
return
_call_center_domain_mock = mock.patch(
'corehq.apps.callcenter.data_source.call_center_data_source_configuration_provider'
)
_call_center_domain_mock.start()
Domain.get_by_name('champ-cameroon').delete()
_call_center_domain_mock.stop()
def query(self):
db = Domain.get_db()
startkey = self.search_string
endkey = "{}Z".format(self.search_string) if startkey else ''
query = db.view(
'domain/domains',
reduce=False,
startkey=startkey,
endkey=endkey,
limit=20,
)
return query
def query(self):
db = Domain.get_db()
startkey = self.search_string
endkey = "{}Z".format(self.search_string) if startkey else ''
query = db.view(
'domain/domains',
reduce=False,
startkey=startkey,
endkey=endkey,
limit=20,
)
return query
def query(self):
db = Domain.get_db()
search_params = {'reduce': False, 'limit': 20}
if self.search_string:
# Search by string range: https://docs.couchdb.org/en/latest/ddocs/views/collation.html#string-ranges
search_params['startkey'] = self.search_string
search_params['endkey'] = "{}\ufff0".format(self.search_string)
query = db.view('domain/domains', **search_params)
return query
def handle(self, *args, **options):
def update_domain(doc):
doc['location_restriction_for_users'] = False
return DocUpdate(doc)
domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)]
res = iter_update(Domain.get_db(), update_domain, domain_ids)
print "number of domains updated:", len(res.updated_ids)
print "Domains that already had it disabled:", res.ignored_ids
print "Not Found:", res.not_found_ids
print "Deleted (sure as shit better be empty):", res.deleted_ids
print "Errored:", res.error_ids
def handle(self, *args, **options):
raise CommandError(
'copy_group_data is currently broken. '
'Ask Danny or Ethan to fix it along the lines of '
'https://github.com/dimagi/commcare-hq/pull/9180/files#diff-9d976dc051a36a028c6604581dfbce5dR95'
)
if len(args) != 2:
raise CommandError('Usage is copy_group_data %s' % self.args)
sourcedb = Database(args[0])
group_id = args[1]
exclude_user_owned = options["exclude_user_owned"]
print 'getting group'
group = Group.wrap(sourcedb.get(group_id))
group.save(force_update=True)
print 'getting domain'
domain = Domain.wrap(
sourcedb.view('domain/domains',
key=group.domain,
include_docs=True,
reduce=False,
limit=1).one()['doc'])
dt = DocumentTransform(domain._obj, sourcedb)
save(dt, Domain.get_db())
owners = [group_id]
if not exclude_user_owned:
owners.extend(group.users)
print 'getting case ids'
with OverrideDB(CommCareCase, sourcedb):
case_ids = get_case_ids_in_domain_by_owner(domain.name,
owner_id__in=owners)
xform_ids = set()
print 'copying %s cases' % len(case_ids)
for i, subset in enumerate(chunked(case_ids, CHUNK_SIZE)):
print i * CHUNK_SIZE
cases = [
CommCareCase.wrap(case['doc']) for case in sourcedb.all_docs(
keys=list(subset),
include_docs=True,
)
]
for case in cases:
def dump(self, output_stream):
from corehq.apps.domain.models import Domain
domain_obj = Domain.get_by_name(self.domain, strict=True)
if not domain_obj:
raise DomainDumpError("Domain not found: {}".format(self.domain))
domain_dict = _get_doc_with_attachments(Domain.get_db(), domain_obj.to_json())
domain_obj = Domain.wrap(domain_dict)
json.dump(domain_obj.to_json(), output_stream)
output_stream.write('\n')
self.stdout.write('Dumping {} Domain\n'.format(1))
return Counter({'Domain': 1})
def dump(self, output_stream):
from corehq.apps.domain.models import Domain
domain_obj = Domain.get_by_name(self.domain, strict=True)
if not domain_obj:
raise DomainDumpError("Domain not found: {}".format(self.domain))
domain_dict = _get_doc_with_attachments(Domain.get_db(), domain_obj.to_json())
domain_obj = Domain.wrap(domain_dict)
json.dump(domain_obj.to_json(), output_stream)
output_stream.write('\n')
self.stdout.write('Dumping {} Domain\n'.format(1))
return Counter({'Domain': 1})
def generate_invoices(based_on_date=None, check_existing=False, is_test=False):
"""
Generates all invoices for the past month.
"""
today = based_on_date or datetime.date.today()
invoice_start, invoice_end = utils.get_previous_month_date_range(today)
logger.info("[Billing] Starting up invoices for %(start)s - %(end)s" % {
'start': invoice_start.strftime(USER_DATE_FORMAT),
'end': invoice_end.strftime(USER_DATE_FORMAT),
})
all_domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)]
for domain_doc in iter_docs(Domain.get_db(), all_domain_ids):
domain = Domain.wrap(domain_doc)
if (check_existing and
Invoice.objects.filter(
subscription__subscriber__domain=domain,
date_created__gte=today).count() != 0):
pass
elif is_test:
logger.info("[Billing] Ready to create invoice for domain %s"
% domain.name)
else:
try:
invoice_factory = DomainInvoiceFactory(
invoice_start, invoice_end, domain)
invoice_factory.create_invoices()
logger.info("[BILLING] Sent invoices for domain %s"
% domain.name)
except CreditLineError as e:
logger.error(
"[BILLING] There was an error utilizing credits for "
"domain %s: %s" % (domain.name, e)
)
except BillingContactInfoError as e:
logger.info("BillingContactInfoError: %s" % e)
except InvoiceError as e:
logger.error(
"[BILLING] Could not create invoice for domain %s: %s" % (
domain.name, e
))
except InvoiceAlreadyCreatedError as e:
logger.error(
"[BILLING] Invoice already existed for domain %s: %s" % (
domain.name, e
))
except Exception as e:
logger.error(
"[BILLING] Error occurred while creating invoice for "
"domain %s: %s" % (domain.name, e)
)
def build(self):
return ElasticPillowReindexer(
pillow_or_processor=get_domain_kafka_to_elasticsearch_pillow(),
change_provider=CouchViewChangeProvider(
couch_db=Domain.get_db(),
view_name='all_docs/by_doc_type',
view_kwargs={
'startkey': ['Domain'],
'endkey': ['Domain', {}],
'include_docs': True,
}),
elasticsearch=get_es_new(),
index_info=DOMAIN_INDEX_INFO,
**self.options)
def _assign_explicit_community_subscriptions(apps, schema_editor):
today = date.today()
all_domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)]
for domain_doc in iter_docs(Domain.get_db(), all_domain_ids):
domain_name = domain_doc['name']
from_date = date(today.year, today.month, 1)
try:
while from_date <= today:
ensure_explicit_community_subscription(domain_name, from_date)
from_date += timedelta(days=1)
except Exception as e:
log_accounting_error(
"During community subscription assignment for domain %s: %s" %
(domain_name, e.message))
def load_objects(self, object_strings, force=False):
from corehq.apps.domain.models import Domain
objects = list(object_strings)
assert len(objects) == 1, "Only 1 domain allowed per dump"
domain_dict = json.loads(objects[0])
domain_name = domain_dict['name']
try:
existing_domain = Domain.get_by_name(domain_name, strict=True)
except ResourceNotFound:
pass
else:
if existing_domain:
if force:
self.stderr.write('Loading data for existing domain: {}'.format(domain_name))
else:
raise DataExistsException("Domain: {}".format(domain_name))
Domain.get_db().bulk_save([domain_dict], new_edits=False)
self.stdout.write('Loaded Domain')
return 1, Counter({'Domain': 1})
def check_exchange_index():
latest_snapshot = Domain.get_db().view('domain/published_snapshots',
limit=1,
descending=True,
include_docs=True).one()
if latest_snapshot is not None:
doc_id = latest_snapshot['id']
couch_rev = latest_snapshot['doc']['_rev']
return _check_es_rev(ExchangePillow.es_index, doc_id, couch_rev)
else:
return {
"%s_status" % ExchangePillow.es_index: False,
"%s_message" % ExchangePillow.es_index: "Exchange stale"
}
def _assign_explicit_community_subscriptions(apps, schema_editor):
today = date.today()
all_domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)]
for domain_doc in iter_docs(Domain.get_db(), all_domain_ids):
domain_name = domain_doc['name']
from_date = date(today.year, today.month, 1)
try:
while from_date <= today:
ensure_explicit_community_subscription(domain_name, from_date)
from_date += timedelta(days=1)
except Exception as e:
log_accounting_error(
"During community subscription assignment for domain %s: %s"
% (domain_name, e.message)
)
def handle(self, *args, **options):
domain = args[0]
old_db = Domain.get_db()
new_db = RepeatRecord.get_db()
assert old_db.dbname != new_db.dbname
doc_ids = get_doc_ids_in_domain_by_type(domain, 'RepeatRecord', old_db)
count = len(doc_ids)
chunksize = 250
for i, docs in enumerate(chunked(iter_docs(old_db, doc_ids, chunksize), chunksize)):
for doc in docs:
if '_rev' in doc:
del doc['_rev']
new_db.bulk_save(docs, new_edits=False)
print 'checked %s / %s' % (i * chunksize, count)
def get_domain_reindexer():
return ElasticPillowReindexer(
pillow=get_domain_kafka_to_elasticsearch_pillow(),
change_provider=CouchViewChangeProvider(
couch_db=Domain.get_db(),
view_name='all_docs/by_doc_type',
view_kwargs={
'startkey': ['Domain'],
'endkey': ['Domain', {}],
'include_docs': True,
}
),
elasticsearch=get_es_new(),
index_info=DOMAIN_INDEX_INFO,
)
def handle(self, *args, **options):
if len(args) != 1:
raise CommandError("Usage is ./manage.py migrate_domain_bu [migration_file]!")
updates = {}
with open(args[0], 'rb') as csvfile:
reader = csv.reader(csvfile, delimiter=',')
for row in reader:
updates[Domain.get_by_name(row[0])['_id']] = row[1]
def update_domain(doc):
Domain.wrap(doc).internal.business_unit = updates[doc['_id']]
return DocUpdate(doc)
iter_update(Domain.get_db(), update_domain, updates.keys(), verbose=True)
def db_comparisons(request):
def _simple_view_couch_query(db, view_name):
return db.view(view_name, reduce=True).one()['value']
def _count_real_forms():
all_forms = _simple_view_couch_query(XFormInstance.get_db(), 'couchforms/by_xmlns')
device_logs = XFormInstance.get_db().view('couchforms/by_xmlns', key=DEVICE_LOG_XMLNS).one()['value']
return all_forms - device_logs
comparison_config = [
{
'description': 'Users (base_doc is "CouchUser")',
'couch_docs': _simple_view_couch_query(CommCareUser.get_db(), 'users/by_username'),
'es_query': UserES().remove_default_filter('active').size(0),
'sql_rows': User.objects.count(),
},
{
'description': 'Domains (doc_type is "Domain")',
'couch_docs': _simple_view_couch_query(Domain.get_db(), 'domain/by_status'),
'es_query': DomainES().size(0),
'sql_rows': None,
},
{
'description': 'Forms (doc_type is "XFormInstance")',
'couch_docs': _count_real_forms(),
'es_query': FormES().remove_default_filter('has_xmlns')
.remove_default_filter('has_user')
.size(0),
'sql_rows': FormData.objects.count(),
},
{
'description': 'Cases (doc_type is "CommCareCase")',
'couch_docs': _simple_view_couch_query(CommCareCase.get_db(), 'case/by_owner'),
'es_query': CaseES().size(0),
'sql_rows': CaseData.objects.count(),
}
]
comparisons = []
for comp in comparison_config:
comparisons.append({
'description': comp['description'],
'couch_docs': comp['couch_docs'],
'es_docs': comp['es_query'].run().total,
'sql_rows': comp['sql_rows'] if comp['sql_rows'] else 'n/a',
})
return json_response(comparisons)
def db_comparisons(request):
comparison_config = [
{
'description': 'Users (base_doc is "CouchUser")',
'couch_db': CommCareUser.get_db(),
'view_name': 'users/by_username',
'es_query': UserES().remove_default_filter('active')
.remove_default_filter('mobile_worker')
.size(0),
'sql_rows': User.objects.count(),
},
{
'description': 'Domains (doc_type is "Domain")',
'couch_db': Domain.get_db(),
'view_name': 'domain/by_status',
'es_query': DomainES().size(0),
'sql_rows': None,
},
{
'description': 'Forms (doc_type is "XFormInstance")',
'couch_db': XFormInstance.get_db(),
'view_name': 'couchforms/by_xmlns',
'es_query': FormES().remove_default_filter('has_xmlns')
.remove_default_filter('has_user')
.size(0),
'sql_rows': FormData.objects.count(),
},
{
'description': 'Cases (doc_type is "CommCareCase")',
'couch_db': CommCareCase.get_db(),
'view_name': 'case/by_owner',
'es_query': CaseES().size(0),
'sql_rows': None,
}
]
comparisons = []
for comp in comparison_config:
comparisons.append({
'description': comp['description'],
'couch_docs': comp['couch_db'].view(
comp['view_name'],
reduce=True,
).one()['value'],
'es_docs': comp['es_query'].run().total,
'sql_rows': comp['sql_rows'] if comp['sql_rows'] else 'n/a',
})
return json_response(comparisons)
def count_downloads_for_all_snapshots(domain_id):
"""
domain_id should represent an actual (non-snapshot) domain for this to make sense
but that is not checked. It'll just be 0 otherwise.
"""
try:
return Domain.get_db().view(
'domain/snapshots',
startkey=[domain_id],
endkey=[domain_id, {}],
reduce=True,
include_docs=False,
).one()["value"]
except TypeError:
return 0
def count_downloads_for_all_snapshots(domain_id):
"""
domain_id should represent an actual (non-snapshot) domain for this to make sense
but that is not checked. It'll just be 0 otherwise.
"""
try:
return Domain.get_db().view(
'domain/snapshots',
startkey=[domain_id],
endkey=[domain_id, {}],
reduce=True,
include_docs=False,
).one()["value"]
except TypeError:
return 0
def handle(self, *args, **options):
db = Domain.get_db()
def get_doc_ids():
for result in db.view("domain/domains", reduce=False).all():
yield result["id"]
for result in ApplicationBase.get_db().view(
"app_manager/applications", startkey=[None], endkey=[None, {}], reduce=False
):
yield result["id"]
for doc in iter_docs(db, get_doc_ids()):
if "secure_submissions" not in doc:
print "Updated", doc.get("doc_type"), doc.get("_id")
doc["secure_submissions"] = False
db.save_doc(doc)
def db_comparisons(request):
def _simple_view_couch_query(db, view_name):
return db.view(view_name, reduce=True).one()["value"]
def _count_real_forms():
all_forms = _simple_view_couch_query(XFormInstance.get_db(), "couchforms/by_xmlns")
device_logs = XFormInstance.get_db().view("couchforms/by_xmlns", key=DEVICE_LOG_XMLNS).one()["value"]
return all_forms - device_logs
comparison_config = [
{
"description": 'Users (base_doc is "CouchUser")',
"couch_docs": _simple_view_couch_query(CommCareUser.get_db(), "users/by_username"),
"es_query": UserES().remove_default_filter("active").size(0),
"sql_rows": User.objects.count(),
},
{
"description": 'Domains (doc_type is "Domain")',
"couch_docs": _simple_view_couch_query(Domain.get_db(), "domain/by_status"),
"es_query": DomainES().size(0),
"sql_rows": None,
},
{
"description": 'Forms (doc_type is "XFormInstance")',
"couch_docs": _count_real_forms(),
"es_query": FormES().remove_default_filter("has_xmlns").remove_default_filter("has_user").size(0),
"sql_rows": FormData.objects.count(),
},
{
"description": 'Cases (doc_type is "CommCareCase")',
"couch_docs": _simple_view_couch_query(CommCareCase.get_db(), "case/by_owner"),
"es_query": CaseES().size(0),
"sql_rows": CaseData.objects.count(),
},
]
comparisons = []
for comp in comparison_config:
comparisons.append(
{
"description": comp["description"],
"couch_docs": comp["couch_docs"],
"es_docs": comp["es_query"].run().total,
"sql_rows": comp["sql_rows"] if comp["sql_rows"] else "n/a",
}
)
return json_response(comparisons)
def setUpClass(cls):
super(FormFactIntegrationTest, cls).setUpClass()
delete_all_docs_by_doc_type(Domain.get_db(),
['Domain', 'Domain-Deleted'])
delete_all_docs_by_doc_type(CommCareUser.get_db(),
['CommCareUser', 'WebUser'])
cls.domain_records = [
Domain(name=cls.domain,
hr_name='One',
creating_user_id='abc',
is_active=True),
]
for domain in cls.domain_records:
domain.save()
cls.user_records = [
# TODO: Handle WebUsers who have multiple domains
# WebUser.create(
# cls.domain,
# 'web-user',
# '***',
# date_joined=datetime.utcnow(),
# first_name='A',
# last_name='B',
# email='*****@*****.**',
# is_active=True,
# is_staff=False,
# is_superuser=True,
# ),
CommCareUser.create(
cls.domain,
'commcare-user',
'***',
date_joined=datetime.utcnow(),
email='*****@*****.**',
is_active=True,
is_staff=True,
is_superuser=False,
),
]
cls.form_records = [
create_form_for_test(cls.domain, user_id=cls.user_records[0]._id),
create_form_for_test(cls.domain, user_id=cls.user_records[0]._id),
create_form_for_test(cls.domain, user_id=cls.user_records[0]._id),
]
def repl_domains(self):
prod_db = get_prod_db(self.source_uri)
domain_docs = [prod_db.view('domain/domains', key=x, include_docs=True, limit=1, reduce=False).one() for x in self.domain_list]
domain_docs = filter(lambda x: x is not None, domain_docs)
domain_doc_ids = [x['doc']['_id'] for x in domain_docs]
if len(domain_doc_ids) == 0:
print "No domains matching your request found, exiting to prevent a full db replication"
sys.exit()
params = {
'continuous': True,
'doc_ids': domain_doc_ids
}
target_server = Domain.get_db().server
self.do_repl(target_server, params)
def tearDownModule():
if isinstance(Domain.get_db(), Mock):
# needed to skip setUp for javascript tests thread on Travis
return
_call_center_domain_mock = mock.patch(
'corehq.apps.callcenter.data_source.call_center_data_source_configuration_provider'
)
_call_center_domain_mock.start()
configs = StaticDataSourceConfiguration.by_domain('champ-cameroon')
adapters = [get_indicator_adapter(config) for config in configs]
for adapter in adapters:
adapter.drop_table()
Domain.get_by_name('champ-cameroon').delete()
_call_center_domain_mock.stop()
def tearDownModule():
if isinstance(Domain.get_db(), Mock):
# needed to skip setUp for javascript tests thread on Travis
return
_call_center_domain_mock = mock.patch(
'corehq.apps.callcenter.data_source.call_center_data_source_configuration_provider'
)
_call_center_domain_mock.start()
configs = StaticDataSourceConfiguration.by_domain('champ-cameroon')
adapters = [get_indicator_adapter(config) for config in configs]
for adapter in adapters:
adapter.drop_table()
Domain.get_by_name('champ-cameroon').delete()
_call_center_domain_mock.stop()
def handle(self, dry_run=False, log_file=False, *args, **options):
if not dry_run:
confirm_force_reset = raw_input(
"Are you sure you want to apply downgrades and upgrades to "
"ALL domains? Type 'yes' to continue. \n")
if confirm_force_reset != 'yes':
return
if log_file:
orig_stdout = sys.stdout
f = file('subscription_changes.txt', 'w')
sys.stdout = f
all_domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)]
for domain_doc in iter_docs(Domain.get_db(), all_domain_ids):
domain = Domain.wrap(domain_doc)
logging.info("%s START" % domain.name)
print("\n")
plan_version, subscription = Subscription.get_subscribed_plan_by_domain(
domain)
if not subscription:
subscriber = Subscriber.objects.get_or_create(
domain=domain.name)[0]
print("Active subscription not found for domain %s" %
domain.name)
else:
subscriber = subscription.subscriber
if not dry_run:
print("%s => %s" % (domain.name, plan_version.plan.name))
subscriber.apply_upgrades_and_downgrades(
new_plan_version=plan_version,
verbose=True,
new_subscription=subscription,
)
else:
print("[DRY RUN] %s => %s" %
(domain.name, plan_version.plan.name))
logging.info("%s END" % domain.name)
if log_file:
sys.stdout = orig_stdout
f.close()
def handle(self, dry_run=False, log_file=False, *args, **options):
if not dry_run:
confirm_force_reset = raw_input(
"Are you sure you want to apply downgrades and upgrades to "
"ALL domains? Type 'yes' to continue. \n"
)
if confirm_force_reset != 'yes':
return
if log_file:
orig_stdout = sys.stdout
f = file('subscription_changes.txt', 'w')
sys.stdout = f
all_domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)]
for domain_doc in iter_docs(Domain.get_db(), all_domain_ids):
domain = Domain.wrap(domain_doc)
logging.info("%s START" % domain.name)
print ("\n")
plan_version, subscription = Subscription.get_subscribed_plan_by_domain(domain)
if not subscription:
subscriber = Subscriber.objects.get_or_create(domain=domain.name)[0]
print ("Active subscription not found for domain %s"
% domain.name)
else:
subscriber = subscription.subscriber
if not dry_run:
print ("%s => %s" %
(domain.name, plan_version.plan.name))
subscriber.apply_upgrades_and_downgrades(
new_plan_version=plan_version,
verbose=True,
new_subscription=subscription,
)
else:
print ("[DRY RUN] %s => %s" %
(domain.name, plan_version.plan.name))
logging.info("%s END" % domain.name)
if log_file:
sys.stdout = orig_stdout
f.close()
def handle(self, doc_types, *args, **options):
input = raw_input('\n'.join([
'\n\nReally delete documents of the following types: {}?',
'This operation is not reversible. Enter a number N to delete the first '
'N found, or type "delete all" to delete everything.',
'',
]).format(doc_types))
if input == 'delete all':
remaining = None
else:
try:
remaining = int(input)
except ValueError:
print 'aborting'
sys.exit()
doc_types = doc_types.split(',')
deleted = 0
# unfortunately the only couch view we have for this needs to go by domain
# will be a bit slow
domain_names = Domain.get_all_names()
database = Domain.get_db()
for domain in domain_names:
for doc_type in doc_types:
docs = [row['doc'] for row in database.view(
'domain/docs',
startkey=[domain, doc_type],
endkey=[domain, doc_type, {}],
reduce=False,
include_docs=True,
)][:remaining]
if docs:
count = len(docs)
print 'deleting {} {}s from {}'.format(count, doc_type, domain)
database.delete_docs(docs)
deleted += count
if remaining is not None:
remaining -= count
if remaining <= 0:
return
print 'successfully deleted {} documents'.format(deleted)
def db_comparisons(request):
def _simple_view_couch_query(db, view_name):
return db.view(view_name, reduce=True).one()['value']
comparison_config = [
{
'description': 'Users (base_doc is "CouchUser")',
'couch_docs': _simple_view_couch_query(CommCareUser.get_db(), 'users/by_username'),
'es_query': UserES().remove_default_filter('active').size(0),
'sql_rows': User.objects.count(),
},
{
'description': 'Domains (doc_type is "Domain")',
'couch_docs': _simple_view_couch_query(Domain.get_db(), 'domain/by_status'),
'es_query': DomainES().size(0),
'sql_rows': None,
},
{
'description': 'Forms (doc_type is "XFormInstance")',
'couch_docs': get_number_of_forms_all_domains_in_couch(),
'es_query': FormES().remove_default_filter('has_xmlns')
.remove_default_filter('has_user')
.size(0),
'sql_rows': FormData.objects.exclude(domain__isnull=True).count(),
},
{
'description': 'Cases (doc_type is "CommCareCase")',
'couch_docs': get_total_case_count(),
'es_query': CaseES().size(0),
'sql_rows': CaseData.objects.exclude(domain__isnull=True).count(),
}
]
comparisons = []
for comp in comparison_config:
comparisons.append({
'description': comp['description'],
'couch_docs': comp['couch_docs'],
'es_docs': comp['es_query'].run().total,
'sql_rows': comp['sql_rows'] if comp['sql_rows'] else 'n/a',
})
return json_response(comparisons)
def setUpClass(cls):
super(FormFactIntegrationTest, cls).setUpClass()
delete_all_docs_by_doc_type(Domain.get_db(), ['Domain', 'Domain-Deleted'])
delete_all_docs_by_doc_type(CommCareUser.get_db(), ['CommCareUser', 'WebUser'])
cls.domain_records = [
Domain(name=cls.domain, hr_name='One', creating_user_id='abc', is_active=True),
]
for domain in cls.domain_records:
domain.save()
cls.user_records = [
# TODO: Handle WebUsers who have multiple domains
# WebUser.create(
# cls.domain,
# 'web-user',
# '***',
# date_joined=datetime.utcnow(),
# first_name='A',
# last_name='B',
# email='*****@*****.**',
# is_active=True,
# is_staff=False,
# is_superuser=True,
# ),
CommCareUser.create(
cls.domain,
'commcare-user',
'***',
date_joined=datetime.utcnow(),
email='*****@*****.**',
is_active=True,
is_staff=True,
is_superuser=False,
),
]
cls.form_records = [
create_form_for_test(cls.domain, user_id=cls.user_records[0]._id),
create_form_for_test(cls.domain, user_id=cls.user_records[0]._id),
create_form_for_test(cls.domain, user_id=cls.user_records[0]._id),
]
cls.batch = create_batch(cls.slug)
def tearDownModule():
if isinstance(Domain.get_db(), Mock):
# needed to skip setUp for javascript tests thread on Travis
return
_call_center_domain_mock = mock.patch(
'corehq.apps.callcenter.data_source.call_center_data_source_configuration_provider'
)
domain = Domain.get_by_name('test-pna')
engine = connection_manager.get_engine(UCR_ENGINE_ID)
metadata = sqlalchemy.MetaData(bind=engine)
metadata.reflect(bind=engine, extend_existing=True)
path = os.path.join(os.path.dirname(__file__), 'fixtures')
for file_name in os.listdir(path):
table_name = get_table_name(domain.name, file_name[:-4])
table = metadata.tables[table_name]
table.drop()
_call_center_domain_mock.start()
domain.delete()
_call_center_domain_mock.stop()
