def setUpClass(cls):
super(TestExplicitCommunitySubscriptions, cls).setUpClass()
# TODO - remove once messy tests are cleaned up
for domain in Domain.get_all():
domain.delete()
assert len(list(Domain.get_all())) == 0
cls.domain = Domain(name='test')
cls.domain.save()
cls.from_date = date.today()
def setUpClass(cls):
super(TestExplicitCommunitySubscriptions, cls).setUpClass()
# TODO - remove once messy tests are cleaned up
for domain in Domain.get_all():
domain.delete()
assert len(list(Domain.get_all())) == 0
cls.domain = Domain(name='test')
cls.domain.save()
cls.from_date = date.today()
def handle(self, *args, **options):
domains = Domain.get_all()
error_domains = []
for d in domains:
if d.commtrack_enabled:
outlet = self.find_outlet_type(
d.commtrack_settings.location_types)
if outlet:
if self.is_still_default_outlet_config(outlet):
if self.any_bad_outlets(d):
self.stdout.write(
"\nDomain " + d.name +
" still has outlets mapped to blocks")
else:
outlet.allowed_parents = ['village']
d.commtrack_settings.save()
self.stdout.write("\nFixed domain " + d.name)
elif len(outlet.allowed_parents) > 1:
# if it was no longer the default setup, but does
# have multiple parent options, we will need to
# fix manually
error_domains.append(d.name)
if error_domains:
self.stdout.write(
"\nThe following domains had outlet types that could not "
"be corrected:\n" + '\n'.join(error_domains))
def generate_invoices(based_on_date=None):
"""
Generates all invoices for the past month.
"""
today = based_on_date or datetime.date.today()
invoice_start, invoice_end = get_previous_month_date_range(today)
log_accounting_info(
"Starting up invoices for %(start)s - %(end)s" % {
'start': invoice_start.strftime(USER_DATE_FORMAT),
'end': invoice_end.strftime(USER_DATE_FORMAT),
})
all_domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)]
for domain_doc in iter_docs(Domain.get_db(), all_domain_ids):
domain = Domain.wrap(domain_doc)
try:
invoice_factory = DomainInvoiceFactory(invoice_start, invoice_end,
domain)
invoice_factory.create_invoices()
log_accounting_info("Sent invoices for domain %s" % domain.name)
except CreditLineError as e:
log_accounting_error("There was an error utilizing credits for "
"domain %s: %s" % (domain.name, e))
except InvoiceError as e:
log_accounting_error("Could not create invoice for domain %s: %s" %
(domain.name, e))
except InvoiceAlreadyCreatedError as e:
log_accounting_error("Invoice already existed for domain %s: %s" %
(domain.name, e))
except Exception as e:
log_accounting_error("Error occurred while creating invoice for "
"domain %s: %s" % (domain.name, e))
if not settings.UNIT_TESTING:
_invoicing_complete_soft_assert(False, "Invoicing is complete!")
def handle(self, *args, **options):
billables_created = 0
for domain in Domain.get_all():
key = [domain.name, 'SMSLog']
start_date = [datetime.datetime(2014, 1, 1).isoformat()]
end_date = [datetime.datetime(2014, 1, 24).isoformat()]
sms_docs = SMSLog.get_db().view('sms/by_domain',
reduce=False,
startkey=key + start_date,
endkey=key + end_date + [{}])
for sms_doc in sms_docs:
sms_log = SMSLog.get(sms_doc['id'])
try:
if sms_log.phone_number is not None:
parse_phone_number(sms_log.phone_number)
except PhoneNumberParseException:
billables = SmsBillable.objects.filter(log_id=sms_log._id)
if len(billables) == 0:
SmsBillable.create(sms_log)
billables_created += 1
print 'created SmsBillable for invalid number %s in domain %s, id=%s'\
% (sms_log.phone_number, domain.name, sms_log._id)
elif len(billables) > 1:
print "Warning: >1 SmsBillable exists for SMSLog with id=%" % sms_log._id
print 'Number of SmsBillables created: %d' % billables_created
print 'Completed retrobilling.'
def handle(self, *args, **options):
for domain in Domain.get_all():
if domain['commtrack_enabled']:
fields_definition = CustomDataFieldsDefinition.get_or_create(
domain['name'],
'ProductFields'
)
product_ids = Product.ids_by_domain(domain['name'])
existing_field_slugs = set(
[field.slug for field in fields_definition.fields]
)
for product in iter_docs(Product.get_db(), product_ids):
product_data = product.get('product_data', {})
for key in product_data.keys():
if key and key not in existing_field_slugs:
existing_field_slugs.add(key)
fields_definition.fields.append(CustomDataField(
slug=key,
label=key,
is_required=False
))
# Only save a definition for domains which use custom product data
if fields_definition.fields:
fields_definition.save()
def handle(self, *args, **options):
toggle_map = dict([(t.slug, t) for t in all_toggles()])
domains = [row['key'] for row in Domain.get_all(include_docs=False)]
for domain in domains:
if toggle_map['rich_text'].enabled(domain) or toggle_map['experimental_ui'].enabled(domain):
#logger.info('migrating domain {}'.format(domain))
apps = get_apps_in_domain(domain, include_remote=False)
for app in apps:
app_dirty = False
builder = ParentCasePropertyBuilder(app)
relationships = builder.get_parent_type_map(app.get_case_types(), allow_multiple_parents=True)
for module in app.modules:
for form in module.forms:
if form.doc_type == 'Form' and form.requires_case():
#logger.info('migrating form {}'.format(form.name.get('en', form.name)))
base_case_type = form.get_module().case_type
self._replace_in_form(form, relationships, base_case_type, 0)
prefixes = re.findall(r'#case/\w+/', form.source)
prefixes = set(prefixes)
for p in prefixes:
if p != "#case/parent/" and p != "#case/grandparent/":
self._form_error(form, "Unknown prefix remaining: {}".format(p))
if options['save']:
try:
save_xform(form.get_app(), form, form.source)
app_dirty = True
except:
self._form_error(form, "Form xml invalid")
if app_dirty:
app.save()
logger.info('done with cmitfb_migrate_syntax')
def handle(self, *args, **options):
for domain in Domain.get_all():
if (
self.domain_has_active_reminders(domain) and
not domain_has_privilege(domain, privileges.REMINDERS_FRAMEWORK)
):
print "%s has active reminders without a subscription" % domain.name
def domain_list(request):
# one wonders if this will eventually have to paginate
domains = Domain.get_all()
all_stats = _all_domain_stats()
for dom in domains:
dom.web_users = int(all_stats["web_users"][dom.name])
dom.commcare_users = int(all_stats["commcare_users"][dom.name])
dom.cases = int(all_stats["cases"][dom.name])
dom.forms = int(all_stats["forms"][dom.name])
dom.admins = [
row["doc"]["email"]
for row in get_db().view("users/admins_by_domain", key=dom.name, reduce=False, include_docs=True).all()
]
context = get_hqadmin_base_context(request)
context.update({"domains": domains})
context["layout_flush_content"] = True
headers = DataTablesHeader(
DataTablesColumn("Domain"),
DataTablesColumn("# Web Users", sort_type=DTSortType.NUMERIC),
DataTablesColumn("# Mobile Workers", sort_type=DTSortType.NUMERIC),
DataTablesColumn("# Cases", sort_type=DTSortType.NUMERIC),
DataTablesColumn("# Submitted Forms", sort_type=DTSortType.NUMERIC),
DataTablesColumn("Domain Admins"),
)
context["headers"] = headers
context["aoColumns"] = headers.render_aoColumns
return render_to_response(request, "hqadmin/domain_list.html", context)
def handle(self, *args, **options):
num_sms = 0
start_datetime = datetime.datetime(*str_to_int_tuple(args[0:6]))
end_datetime = datetime.datetime(*str_to_int_tuple(args[6:12]))
for domain in Domain.get_all():
key = [domain.name, 'SMSLog']
sms_docs = SMSLog.get_db().view('sms/by_domain',
reduce=False,
startkey=key + [start_datetime.isoformat()],
endkey=key + [end_datetime.isoformat(), {}],
)
for sms_doc in sms_docs:
sms_log = SMSLog.get(sms_doc['id'])
if options.get('create', False):
SmsBillable.create(sms_log)
print 'Created billable for SMSLog %s in domain %s from %s' \
% (sms_doc['id'], domain.name, sms_log.date)
else:
print 'Found SMSLog %s in domain %s from %s' \
% (sms_doc['id'], domain.name, sms_log.date)
num_sms += 1
print 'Number of SMSs in datetime range: %d' % num_sms
def gauge_pending_user_confirmations():
metric_name = 'commcare.pending_user_confirmations'
from corehq.apps.users.models import Invitation
for stats in (Invitation.objects.filter(is_accepted=False).all()
.values('domain').annotate(Count('domain'))):
metrics_gauge(
metric_name, stats['domain__count'], tags={
'domain': stats['domain'],
'user_type': 'web',
},
multiprocess_mode=MPM_MAX
)
from corehq.apps.users.analytics import get_inactive_commcare_users_in_domain
for doc in Domain.get_all(include_docs=False):
domain_name = doc['key']
users = get_inactive_commcare_users_in_domain(domain_name)
num_unconfirmed = sum(1 for u in users if not u.is_account_confirmed)
if num_unconfirmed:
metrics_gauge(
metric_name, num_unconfirmed, tags={
'domain': domain_name,
'user_type': 'mobile',
},
multiprocess_mode=MPM_MAX
)
def tearDownClass(cls):
for config in DataSourceConfiguration.all():
config.delete()
delete_all_report_configs()
for domain in Domain.get_all():
domain.delete()
super(ReportConfigurationDbTest, cls).tearDownClass()
def handle(self, *args, **options):
domain_names = args or [
d["key"] for d in Domain.get_all(include_docs=False)
]
start_at = options.get("startat")
if start_at and start_at in domain_names:
domain_names = domain_names[domain_names.index(start_at):]
for d in domain_names:
print "Migrating cases in project space: %s" % d
key = ["all", d]
results = get_db().view(
'case/all_cases',
startkey=key,
endkey=key + [{}],
reduce=False,
include_docs=False,
).all()
cases_to_save = []
for raw_case in iter_docs(get_db(), [r['id'] for r in results]):
if add_to_case(raw_case):
cases_to_save.append(raw_case)
if len(cases_to_save) > 100:
get_db().bulk_save(cases_to_save)
cases_to_save = []
get_db().bulk_save(cases_to_save)
def handle(self, **options):
logger.setLevel('DEBUG')
domains = [row['key'] for row in Domain.get_all(include_docs=False)]
settings = {
s['id']: s['default']
for s in get_custom_commcare_settings() if 'default' in s
}
deviant_counts = {id: 0 for id in settings.keys()}
app_count = 0
for domain in domains:
for app in get_apps_in_domain(domain, include_remote=False):
#logger.info("looking at app {}".format(app.id))
if ('properties' in app.profile):
app_count = app_count + 1
for id, default in six.iteritems(settings):
if (id not in app.profile['properties']):
#logger.info("{}: not found".format(id))
pass
elif (app.profile['properties'][id] != default):
#logger.info("{}: {} != {}".format(id, app.profile['properties'][id], default))
deviant_counts[id] = deviant_counts[id] + 1
else:
#logger.info("{}: {} == {}".format(id, app.profile['properties'][id], default))
pass
for id, count in six.iteritems(deviant_counts):
logger.info("{}\t{}".format(count, id))
logger.info(
'done with audit_app_profile_properties, examined {} apps in {} domains'
.format(app_count, len(domains)))
def handle(self, *args, **options):
self.stdout.write("Fixing default programs...\n")
for domain in Domain.get_all():
if not domain.commtrack_enabled:
continue
if Program.default_for_domain(domain.name):
continue
programs = Program.by_domain(domain.name)
# filter anything named 'default' or 'Default'
current_default = [
p for p in programs
if p.name == 'Default' or p.name == 'default'
]
# if they never changed their default programs
# name, we don't want to add a confusing new one
# so just flip this to the default
if len(current_default) == 1:
p.default = True
p.save()
else:
get_or_create_default_program(domain.name)
def handle(self, *args, **options):
domains = Domain.get_all()
error_domains = []
for d in domains:
if d.commtrack_enabled:
outlet = self.find_outlet_type(d.location_types)
if outlet:
if self.is_still_default_outlet_config(outlet):
if self.any_bad_outlets(d):
self.stdout.write(
"\nDomain " + d.name + " still has outlets mapped to blocks"
)
else:
outlet.allowed_parents = ['village']
d.commtrack_settings.save()
self.stdout.write("\nFixed domain " + d.name)
elif len(outlet.allowed_parents) > 1:
# if it was no longer the default setup, but does
# have multiple parent options, we will need to
# fix manually
error_domains.append(d.name)
if error_domains:
self.stdout.write(
"\nThe following domains had outlet types that could not "
"be corrected:\n" + '\n'.join(error_domains)
)
def handle(self, *args, **options):
print "Migrating Domain countries"
country_lookup = {x[1].lower(): x[0] for x in COUNTRIES}
#Special cases
country_lookup["USA"] = country_lookup["united states"]
country_lookup["California"] = country_lookup["united states"]
country_lookup["Wales"] = country_lookup["united kingdom"]
for domain in Domain.get_all():
try:
if isinstance(domain.deployment.country, basestring):
if ',' in domain.deployment.country:
countries = domain.deployment.country.split(',')
elif ' and ' in domain.deployment.country:
countries = domain.deployment.country.split(' and ')
else:
countries = [domain.deployment.country]
abbr = []
for country in countries:
country = country.strip().lower()
if country in country_lookup.keys():
abbr.append(country_lookup[country])
domain.deployment.countries = abbr
domain.save()
except Exception as e:
print "There was an error migrating the domain named %s." % domain.name
print "Error: %s", e
def _get_matching_couch_domains(run_config):
if run_config.domain is ALL_DOMAINS:
return [domain.name for domain in Domain.get_all() if not _should_use_sql_backend(domain)]
elif _should_use_sql_backend(run_config.domain):
return []
else:
return [run_config.domain]
def handle(self, *args, **options):
for domain in Domain.get_all():
if domain['commtrack_enabled']:
fields_definition = cdm.CustomDataFieldsDefinition.get_or_create(
domain['name'], 'ProductFields')
had_fields = bool(fields_definition.fields)
product_ids = Product.ids_by_domain(domain['name'])
existing_field_slugs = set(
[field.slug for field in fields_definition.fields])
for product in iter_docs(Product.get_db(), product_ids):
product_data = product.get('product_data', {})
for key in product_data.keys():
if key and key not in existing_field_slugs:
existing_field_slugs.add(key)
fields_definition.fields.append(
cdm.CustomDataField(
slug=key,
label=key,
is_required=False,
))
for field in fields_definition.fields:
if cdm.is_system_key(field.slug):
fields_definition.fields.remove(field)
# Only save a definition for domains which use custom product data
if fields_definition.fields or had_fields:
fields_definition.save()
print 'finished domain "{}"'.format(domain['name'])
def page_context(self):
toggle_meta = self.toggle_meta()
toggle = self.get_toggle()
context = {
'toggle_meta':
toggle_meta,
'toggle':
toggle,
'expanded':
self.expanded,
'namespaces': [
NAMESPACE_USER if n is None else n
for n in toggle_meta.namespaces
],
'usage_info':
self.usage_info,
}
if self.expanded:
context['domain_toggle_list'] = sorted(
[(row['key'], toggle_meta.enabled(row['key']))
for row in Domain.get_all(include_docs=False)],
key=lambda domain_tup: (not domain_tup[1], domain_tup[0]))
if self.usage_info:
context['last_used'] = _get_usage_info(toggle)
return context
def handle(self, *args, **options):
self.stdout.write("Fixing default programs...\n")
for domain in Domain.get_all():
if not domain.commtrack_enabled:
continue
if Program.default_for_domain(domain.name):
continue
programs = Program.by_domain(domain.name)
# filter anything named 'default' or 'Default'
current_default = [
p for p in programs
if p.name == 'Default' or p.name == 'default'
]
# if they never changed their default programs
# name, we don't want to add a confusing new one
# so just flip this to the default
if len(current_default) == 1:
p.default = True
p.save()
else:
get_or_create_default_program(domain.name)
def handle(self, **options):
if options.get('domain'):
domains = [Domain.get_by_name(options.get('domain'))]
else:
domains = Domain.get_all()
since = options.get('since')
seen_since = False
for d in domains:
if since and not seen_since:
if d.name == since:
seen_since = True
else:
continue
if d.name in DOMS_TO_IGNORE:
continue
try:
for app in d.full_applications():
try:
if app.is_remote_app():
continue
for _, m in app.get_media_objects(remove_unused=True):
if app.domain not in m.valid_domains:
m.valid_domains.append(app.domain)
self.stdout.write(
"adding domain %s to media file %s" %
(app.domain, m._id))
m.save()
except Exception as e:
self.stdout.write("Error in app %s-%s: %s" %
(app.domain, app._id, e))
except Exception as e:
self.stdout.write("Error in domain %s: %s" % (d.name, e))
def handle(self, *args, **options):
for domain in Domain.get_all():
UserRole.get_or_create_with_permissions(
domain.name,
UserRolePresets.get_permissions(UserRolePresets.BILLING_ADMIN),
UserRolePresets.BILLING_ADMIN
)
def handle(self, *args, **options):
if options.get('domain'):
domains = [Domain.get_by_name(options.get('domain'))]
else:
domains = Domain.get_all()
since = options.get('since')
seen_since = False
for d in domains:
if since and not seen_since:
if d.name == since:
seen_since = True
else:
continue
if d.name in DOMS_TO_IGNORE:
continue
try:
for app in d.full_applications():
try:
if app.is_remote_app():
continue
for _, m in app.get_media_objects():
if app.domain not in m.valid_domains:
m.valid_domains.append(app.domain)
self.stdout.write("adding domain %s to media file %s" % (app.domain, m._id))
m.save()
except Exception as e:
self.stdout.write("Error in app %s-%s: %s" % (app.domain, app._id, e))
except Exception as e:
self.stdout.write("Error in domain %s: %s" % (d.name, e))
def handle(self, month_year, file_path, **options):
month_year_parsed = dateutil.parser.parse('1-' + month_year)
start_date = month_year_parsed.replace(day=1)
end_date = start_date + relativedelta(
day=1, months=+1, microseconds=-1)
with open(file_path, 'wb') as file_object:
writer = csv.writer(file_object)
writer.writerow([
'domain name', 'user id',
'total number of forms submitted in a month',
'used management case?', 'multiple_form_types?'
])
for domain_object in Domain.get_all():
domain_name = domain_object.name
user_ids = CommCareUser.ids_by_domain(domain=domain_name)
buckets_dict = get_forms_for_users(domain_object.name,
user_ids, start_date,
end_date)
for user_id, bucket in buckets_dict.iteritems():
has_case = bool(
filter(lambda h: h.get('form', {}).get('case'),
bucket.top_hits_user_submissions.hits))
has_two_or_more_different_forms_submitted = len([
hit.get('form', {}).get('@xmlns')
for hit in bucket.top_hits_user_submissions.hits
if hit.get('form', {}).get('@xmlns')
]) >= 2
writer.writerow([
domain_name, user_id, bucket.doc_count, has_case,
has_two_or_more_different_forms_submitted
])
def message_log_report(request):
show_dates = True
datespan = request.datespan
domains = Domain.get_all()
for dom in domains:
dom.sms_incoming = SMSLog.count_incoming_by_domain(
dom.name, datespan.startdate_param, datespan.enddate_param)
dom.sms_outgoing = SMSLog.count_outgoing_by_domain(
dom.name, datespan.startdate_param, datespan.enddate_param)
dom.sms_total = SMSLog.count_by_domain(dom.name,
datespan.startdate_param,
datespan.enddate_param)
context = get_hqadmin_base_context(request)
headers = DataTablesHeader(
DataTablesColumn("Domain"),
DataTablesColumn("Incoming Messages", sort_type=DTSortType.NUMERIC),
DataTablesColumn("Outgoing Messages", sort_type=DTSortType.NUMERIC),
DataTablesColumn("Total Messages", sort_type=DTSortType.NUMERIC))
context["headers"] = headers
context["aoColumns"] = headers.render_aoColumns
context.update({
"domains": domains,
"show_dates": show_dates,
"datespan": datespan
})
context['layout_flush_content'] = True
return render(request, "hqadmin/message_log_report.html", context)
def tearDownClass(cls):
for config in DataSourceConfiguration.all():
config.delete()
delete_all_report_configs()
for domain in Domain.get_all():
domain.delete()
super(ReportConfigurationDbTest, cls).tearDownClass()
def handle(self, *args, **options):
domain_names = args or [d["key"] for d in Domain.get_all(include_docs=False)]
start_at = options.get("startat")
if start_at and start_at in domain_names:
domain_names = domain_names[domain_names.index(start_at):]
for d in domain_names:
print "Migrating cases in project space: %s" % d
key = ["all", d]
results = get_db().view('case/all_cases',
startkey=key,
endkey=key + [{}],
reduce=False,
include_docs=False,
).all()
cases_to_save = []
for raw_case in iter_docs(get_db(), [r['id'] for r in results]):
if add_to_case(raw_case):
cases_to_save.append(raw_case)
if len(cases_to_save) > 100:
get_db().bulk_save(cases_to_save)
cases_to_save = []
get_db().bulk_save(cases_to_save)
def message_log_report(request):
show_dates = True
datespan = request.datespan
domains = Domain.get_all()
for dom in domains:
dom.sms_incoming = SMSLog.count_incoming_by_domain(dom.name, datespan.startdate_param, datespan.enddate_param)
dom.sms_outgoing = SMSLog.count_outgoing_by_domain(dom.name, datespan.startdate_param, datespan.enddate_param)
dom.sms_total = SMSLog.count_by_domain(dom.name, datespan.startdate_param, datespan.enddate_param)
context = get_hqadmin_base_context(request)
headers = DataTablesHeader(
DataTablesColumn("Domain"),
DataTablesColumn("Incoming Messages", sort_type=DTSortType.NUMERIC),
DataTablesColumn("Outgoing Messages", sort_type=DTSortType.NUMERIC),
DataTablesColumn("Total Messages", sort_type=DTSortType.NUMERIC),
)
context["headers"] = headers
context["aoColumns"] = headers.render_aoColumns
context.update({"domains": domains, "show_dates": show_dates, "datespan": datespan})
context["layout_flush_content"] = True
return render(request, "hqadmin/message_log_report.html", context)
def handle(self, *args, **options):
with open('parentage_results.csv', 'wb+') as csvfile:
csv_writer = csv.writer(csvfile,
delimiter=',',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
csv_writer.writerow([
'id',
'name',
'is_test',
'location_type',
'number_of_offending_locations',
])
domains = Domain.get_all()
for d in domains:
if d.commtrack_enabled:
for loc_type in d.commtrack_settings.location_types:
if len(loc_type.allowed_parents) > 1:
count = len(
list(
Location.filter_by_type(
d.name,
loc_type.name,
)))
csv_writer.writerow([
d._id, d.name, d.is_test, loc_type.name, count
])
def handle(self, *args, **options):
print "Migrating Domain countries"
country_lookup = {x[1].lower(): x[0] for x in COUNTRIES}
#Special cases
country_lookup["USA"] = country_lookup["united states"]
country_lookup["California"] = country_lookup["united states"]
country_lookup["Wales"] = country_lookup["united kingdom"]
for domain in Domain.get_all():
try:
if isinstance(domain.deployment.country, basestring):
if ',' in domain.deployment.country:
countries = domain.deployment.country.split(',')
elif ' and ' in domain.deployment.country:
countries = domain.deployment.country.split(' and ')
else:
countries = [domain.deployment.country]
abbr = []
for country in countries:
country = country.strip().lower()
if country in country_lookup.keys():
abbr.append(country_lookup[country])
domain.deployment.countries = abbr
domain.save()
except Exception as e:
print "There was an error migrating the domain named %s." % domain.name
print "Error: %s", e
def handle(self, *args, **options):
domains = Domain.get_all()
seen = set([])
dups = set([])
for domain in domains:
if domain.name in seen:
dups.add(domain.name)
else:
seen.add(domain.name)
if not dups:
self.stdout.write('Found no duplicate domains\n')
for domain in list(dups):
real_dom = Domain.get_by_name(domain)
total_doms = Domain.view("domain/domains",
key=domain,
reduce=False,
include_docs=True,
).all()
fake_doms = [d for d in total_doms if d.get_id != real_dom.get_id]
self.stdout.write('Found Dup: %s\n' % domain)
self.stdout.write(" -- _id of correct domain: %s\n" % real_dom.get_id)
self.stdout.write(" -- ids of duplicate domains: %s\n" % [d.get_id for d in fake_doms])
for dom in fake_doms:
dom.doc_type = 'Domain-DUPLICATE'
dom.save()
def handle(self, *args, **options):
dryrun = options.pop('dryrun')
if dryrun:
print '*** Running in dryrun mode. Will not save any conversion ***'
for doc in Domain.get_all(include_docs=False):
domain = doc['key']
migrate_domain(domain, dryrun)
def build_table(self):
rows = []
for domain in Domain.get_all():
for monthspan in self.monthspan_list:
gir_dict = GIRTableGenerator.get_gir_dict_for_domain_and_monthspan(
domain, monthspan)
rows.append(gir_dict)
GIRRow.objects.bulk_create([GIRRow(**gir_row) for gir_row in rows])
def handle(self, filename, **options):
with open(filename, 'wb') as f:
writer = csv.writer(f)
writer.writerow(['domain', 'section', 'type', 'report'])
for domain in Domain.get_all():
for report_config in domain.dynamic_reports:
for report in report_config.reports:
writer.writerow([domain.name, report_config.section_title, report.report, report.name])
def generate_invoices(based_on_date=None, check_existing=False, is_test=False):
"""
Generates all invoices for the past month.
"""
today = based_on_date or datetime.date.today()
invoice_start, invoice_end = utils.get_previous_month_date_range(today)
logger.info("[Billing] Starting up invoices for %(start)s - %(end)s" % {
'start': invoice_start.strftime("%d %B %Y"),
'end': invoice_end.strftime("%d %B %Y"),
})
all_domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)]
for domain_doc in iter_docs(Domain.get_db(), all_domain_ids):
domain = Domain.wrap(domain_doc)
if (check_existing and
Invoice.objects.filter(
subscription__subscriber__domain=domain,
date_created__gte=today).count() != 0):
pass
elif is_test:
logger.info("[Billing] Ready to create invoice for domain %s"
% domain.name)
else:
try:
invoice_factory = DomainInvoiceFactory(
invoice_start, invoice_end, domain)
invoice_factory.create_invoices()
logger.info("[BILLING] Sent invoices for domain %s"
% domain.name)
except CreditLineError as e:
logger.error(
"[BILLING] There was an error utilizing credits for "
"domain %s: %s" % (domain.name, e)
)
except BillingContactInfoError as e:
subject = "[%s] Invoice Generation Issue" % domain.name
email_content = render_to_string(
'accounting/invoice_error_email.html', {
'project': domain.name,
'error_msg': 'BillingContactInfoError: %s' % e,
}
)
send_HTML_email(
subject, settings.BILLING_EMAIL, email_content,
email_from="Dimagi Billing Bot <%s>" % settings.DEFAULT_FROM_EMAIL
)
except InvoiceError as e:
logger.error(
"[BILLING] Could not create invoice for domain %s: %s" % (
domain.name, e
))
except Exception as e:
logger.error(
"[BILLING] Error occurred while creating invoice for "
"domain %s: %s" % (domain.name, e)
)
# And finally...
if not is_test:
send_bookkeeper_email()
def domain_response(self):
domain_names = [
domain['key'] for domain in Domain.get_all(include_docs=False)
]
if self.search_string:
domain_names = filter(
lambda x: x.lower().startswith(self.search_string.lower()),
domain_names)
return [(d, d) for d in domain_names]
def setUp(self):
for domain in Domain.get_all():
domain.delete()
super(TimeZoneMigrationTest, self).setUp()
self.domain = 'foo'
self.domain_object = create_domain(self.domain)
tzp, _ = DomainMigrationProgress.objects.get_or_create(domain=self.domain, migration_slug=TZMIGRATION_SLUG)
tzp.migration_status = MigrationStatus.NOT_STARTED
tzp.save()
def tearDownClass(cls):
cls.api_key.delete()
cls.user.delete()
for domain in Domain.get_all():
Subscription._get_active_subscription_by_domain.clear(Subscription, domain.name)
domain.delete()
super(APIResourceTest, cls).tearDownClass()
def handle(self, **options):
for doc in Domain.get_all(include_docs=False):
domain = doc['key']
if not use_new_exports(domain):
set_toggle(toggles.OLD_EXPORTS.slug,
domain,
True,
namespace=toggles.NAMESPACE_DOMAIN)
toggle_js_domain_cachebuster.clear(domain)
def setUp(self):
for domain in Domain.get_all():
domain.delete()
super(TimeZoneMigrationTest, self).setUp()
self.domain = 'foo'
self.domain_object = create_domain(self.domain)
tzp, _ = DomainMigrationProgress.objects.get_or_create(domain=self.domain, migration_slug=TZMIGRATION_SLUG)
tzp.migration_status = MigrationStatus.NOT_STARTED
tzp.save()
def handle(self, itext_type, log_file, **options):
hdlr = logging.FileHandler(log_file)
formatter = logging.Formatter('%(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
domains = [row['key'] for row in Domain.get_all(include_docs=False)]
domain_data = {}
errors = []
for domain in domains:
sys.stdout.write(".")
app_data = {}
for app in get_apps_in_domain(domain, include_remote=False):
form_data = {}
for module in app.get_modules():
for form in module.get_forms():
try:
xform = form.wrapped_xform()
except Exception:
errors.append(
ItextFetchError(domain, app.get_id,
form.get_unique_id()))
continue
total_refs = len(
xform.media_references(form=itext_type))
if total_refs > 0:
form_data[form.unique_id] = total_refs
if len(form_data) > 0:
app_data[app.get_id] = {
'data': form_data,
'total_refs': sum(form_data.values()),
'total_forms': len(form_data),
}
if len(app_data) > 0:
domain_data[domain] = {
'app_data':
app_data,
'total_apps':
len(app_data),
'total_refs':
sum([a['total_refs'] for a in app_data.values()]),
'total_forms':
sum([a['total_forms'] for a in app_data.values()]),
}
logger.info('DOMAINS USING "{}"'.format(itext_type))
logger.info('domain name\t# apps\t# forms\t# references')
for domain, data in domain_data.items():
logger.info('{}\t{}\t{}\t{}'.format(domain, data['total_apps'],
data['total_forms'],
data['total_refs']))
logger.info('\n\nERRORS')
for error in errors:
logger.info(
'Error getting form {} in app {} from domain {}'.format(
error.form, error.app, error.domain))
def generate_monthly_bills(billing_range=None, domain_name=None):
logging.info("[Billing] Generating Monthly Bills")
from corehq.apps.domain.models import Domain
if domain_name is not None:
domains = [domain_name]
else:
domains = [domain['key'] for domain in Domain.get_all(include_docs=False)]
for domain in domains:
HQMonthlyBill.create_bill_for_domain(domain, billing_range=billing_range)
def handle(self, *args, **kwargs):
domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)]
with IterativeSaver(Domain.get_db()) as iter_db:
for domain_doc in iter_docs(Domain.get_db(), domain_ids):
if (domain_doc.pop('location_types', None)
or domain_doc.pop('obsolete_location_types', None)):
print("Removing location_types from domain {} - {}".format(
domain_doc['name'], domain_doc['_id']))
iter_db.save(domain_doc)
def build_table(self):
rows = []
for domain in Domain.get_all():
for monthspan in self.monthspan_list:
gir_dict = GIRTableGenerator.get_gir_dict_for_domain_and_monthspan(domain, monthspan)
rows.append(gir_dict)
GIRRow.objects.bulk_create(
[GIRRow(**gir_row) for gir_row in rows]
)
def check_requisition_updates():
projects = Domain.get_all()
for project in projects:
if project.commtrack_enabled and project.commtrack_settings.openlmis_enabled:
endpoint = OpenLMISEndpoint.from_config(project.commtrack_settings.openlmis_config)
requisitions = endpoint.get_all_requisition_statuses()
for requisition in requisitions:
cases, send_notification = sync_requisition_from_openlmis(project.name, requisition.requisition_id, endpoint)
if cases and send_notification:
send_notifications(xform=None, cases=cases)
def handle(self, *args, **options):
csvWriter = csv.writer(sys.stdout)
for domain in Domain.get_all():
for app in domain.full_applications(include_builds=False):
for module in app.modules:
for form in module.forms:
intents = form.wrapped_xform().odk_intents
if len(intents):
csvWriter.writerow([domain.name, app.name,
module.name, form.name, intents])
def domain_response(self):
domain_names = [
domain['key'] for domain in Domain.get_all(include_docs=False)
]
if self.search_string:
search_string = self.search_string.lower()
domain_names = [
x for x in domain_names if x.lower().startswith(search_string)
]
return [(d, d) for d in domain_names]
def handle(self, *args, **kwargs):
domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)]
with IterativeSaver(Domain.get_db()) as iter_db:
for domain_doc in iter_docs(Domain.get_db(), domain_ids):
if (
domain_doc.pop('location_types', None) or
domain_doc.pop('obsolete_location_types', None)
):
print ("Removing location_types from domain {} - {}"
.format(domain_doc['name'], domain_doc['_id']))
iter_db.save(domain_doc)
def resend_pending_invitations():
from corehq.apps.users.models import DomainInvitation
days_to_resend = (15, 29)
days_to_expire = 30
domains = Domain.get_all()
for domain in domains:
invitations = DomainInvitation.by_domain(domain.name)
for invitation in invitations:
days = (datetime.now() - invitation.invited_on).days
if days in days_to_resend:
invitation.send_activation_email(days_to_expire - days)
def mobile_user_reports(request):
template = "hqadmin/mobile_user_reports.html"
rows = []
if request.GET.get('old') == 'true': # old reports
for domain in Domain.get_all():
data = get_db().view("phonelog/devicelog_data",
reduce=False,
startkey=[domain.name, "tag", "user-report"],
endkey=[domain.name, "tag", "user-report", {}],
#stale=settings.COUCH_STALE_QUERY,
).all()
for report in data:
val = report.get('value')
version = val.get('version', 'unknown')
formatted_date = tz_utils.string_to_prertty_time(val['@date'], timezone(domain.default_timezone), fmt="%b %d, %Y %H:%M:%S")
rows.append(dict(domain=domain.name,
time=formatted_date,
user=val['user'],
device_users=val['device_users'],
message=val['msg'],
version=version.split(' ')[0],
detailed_version=html.escape(version),
report_id=report['id']))
else:
logs = Log.objects.filter(type__exact="user-report").order_by('domain')
for log in logs:
rows.append(dict(domain=log.domain,
time=log.date,
user=log.username,
device_users=[u.username for u in log.device_users.all()],
message=log.msg,
version=(log.app_version or 'unknown').split(' ')[0],
detailed_version=html.escape(log.app_version or 'unknown'),
report_id=log.xform_id))
headers = DataTablesHeader(
DataTablesColumn(_("View Form")),
DataTablesColumn(_("Domain")),
DataTablesColumn(_("Time"), sort_type=DTSortType.NUMERIC),
DataTablesColumn(_("User"), sort_type=DTSortType.NUMERIC),
DataTablesColumn(_("Device Users"), sort_type=DTSortType.NUMERIC),
DataTablesColumn(_("Message"), sort_type=DTSortType.NUMERIC),
DataTablesColumn(_("Version"), sort_type=DTSortType.NUMERIC)
)
context = get_hqadmin_base_context(request)
context["headers"] = headers
context["aoColumns"] = headers.render_aoColumns
context["rows"] = rows
return render(request, template, context)
def handle(self, *args, **options):
for doc in Domain.get_all(include_docs=False):
domain = doc['key']
if not use_new_exports(domain):
set_toggle(
toggles.OLD_EXPORTS.slug,
domain,
True,
namespace=toggles.NAMESPACE_DOMAIN
)
toggle_js_domain_cachebuster.clear(domain)
def handle(self, itext_type, log_file, **options):
hdlr = logging.FileHandler(log_file)
formatter = logging.Formatter('%(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
domains = [row['key'] for row in Domain.get_all(include_docs=False)]
domain_data = {}
errors = []
for domain in domains:
sys.stdout.write(".")
app_data = {}
for app in get_apps_in_domain(domain, include_remote=False):
form_data = {}
for module in app.get_modules():
for form in module.get_forms():
try:
xform = form.wrapped_xform()
except Exception:
errors.append(ItextFetchError(domain, app.get_id, form.get_unique_id()))
continue
total_refs = len(xform.media_references(form=itext_type))
if total_refs > 0:
form_data[form.unique_id] = total_refs
if len(form_data) > 0:
app_data[app.get_id] = {
'data': form_data,
'total_refs': sum(form_data.values()),
'total_forms': len(form_data),
}
if len(app_data) > 0:
domain_data[domain] = {
'app_data': app_data,
'total_apps': len(app_data),
'total_refs': sum([a['total_refs'] for a in app_data.values()]),
'total_forms': sum([a['total_forms'] for a in app_data.values()]),
}
logger.info('DOMAINS USING "{}"'.format(itext_type))
logger.info('domain name\t# apps\t# forms\t# references')
for domain, data in domain_data.items():
logger.info('{}\t{}\t{}\t{}'.format(
domain,
data['total_apps'],
data['total_forms'],
data['total_refs']
))
logger.info('\n\nERRORS')
for error in errors:
logger.info('Error getting form {} in app {} from domain {}'.format(
error.form, error.app, error.domain
))
def handle(self, *args, **options):
if len(args) != 1:
print 'usage is ./manage.py list_dynamic_reports filename'
sys.exit(1)
filename = args[0]
with open(filename, 'wb') as f:
writer = csv.writer(f)
writer.writerow(['domain', 'section', 'type', 'report'])
for domain in Domain.get_all():
for report_config in domain.dynamic_reports:
for report in report_config.reports:
writer.writerow([domain.name, report_config.section_title, report.report, report.name])
