def get_data(self):
# todo: this will probably have to paginate eventually
if self.all_relevant_forms:
sp_ids = get_relevant_supply_point_ids(
self.domain,
self.active_location,
)
form_xmlnses = [form['xmlns'] for form in self.all_relevant_forms.values()]
spoint_loc_map = {
doc['_id']: doc['location_id']
for doc in iter_docs(SupplyPointCase.get_db(), sp_ids)
}
locations = {
doc['_id']: Location.wrap(doc)
for doc in iter_docs(Location.get_db(), spoint_loc_map.values())
}
for spoint_id, loc_id in spoint_loc_map.items():
loc = locations[loc_id]
form_ids = StockReport.objects.filter(
stocktransaction__case_id=spoint_id
).exclude(
date__lte=self.start_date
).exclude(
date__gte=self.end_date
).values_list(
'form_id', flat=True
).order_by('-date').distinct() # not truly distinct due to ordering
matched = False
for form_id in form_ids:
try:
if XFormInstance.get(form_id).xmlns in form_xmlnses:
yield {
'loc_id': loc._id,
'loc_path': loc.path,
'name': loc.name,
'type': loc.location_type,
'reporting_status': 'reporting',
'geo': loc._geopoint,
}
matched = True
break
except ResourceNotFound:
logging.error('Stock report for location {} in {} references non-existent form {}'.format(
loc._id, loc.domain, form_id
))
if not matched:
yield {
'loc_id': loc._id,
'loc_path': loc.path,
'name': loc.name,
'type': loc.location_type,
'reporting_status': 'nonreporting',
'geo': loc._geopoint,
}
def get_data(ids):
"""
returns the data in the format:
{
'2015-03': {
'domain1': {
'KOOKOO': {'calls': 40, 'minutes': 45}
},
'domain2': {
'KOOKOO': {'calls': 20, 'minutes': 25}
'TELERIVET': {'calls': 5, 'minutes': 0}
}
}
}
"""
data = {}
for doc in iter_docs(CallLog.get_db(), ids):
call = CallLog.wrap(doc)
month_data = get_month_data(data, call.date)
domain_data = get_domain_data(month_data, call.domain)
backend_api = get_backend_api(call)
backend_data = get_backend_data(domain_data, backend_api)
backend_data['calls'] += 1
duration = (call.duration or 0) / 60.0
duration = int(ceil(duration))
backend_data['minutes'] += duration
return data
def by_domain(cls, domain):
relevant_ids = set([r['id'] for r in cls.get_db().view('locations/by_type',
reduce=False,
startkey=[domain],
endkey=[domain, {}],
).all()])
return (cls.wrap(l) for l in iter_docs(cls.get_db(), list(relevant_ids)))
def handle(self, ids_file, **options):
with open(ids_file, encoding='utf-8') as f:
doc_ids = [line.strip() for line in f]
total_doc_ids = len(doc_ids)
doc_ids = set(doc_ids)
print("{} total doc ids, {} unique".format(total_doc_ids, len(doc_ids)))
db = XFormInstance.get_db() # Both forms and cases are in here
with IterDB(db) as iter_db:
for doc in iter_docs(db, with_progress_bar(doc_ids)):
iter_db.save(doc)
print("{} docs saved".format(len(iter_db.saved_ids)))
print("{} docs errored".format(len(iter_db.error_ids)))
not_found = len(doc_ids) - len(iter_db.saved_ids) - len(iter_db.error_ids)
print("{} docs not found".format(not_found))
filename = '{}_{}.csv'.format(ids_file.split('/')[-1],
datetime.datetime.now().isoformat())
with open(filename, 'w', encoding='utf-8') as f:
writer = csv.writer(f)
writer.writerow(['doc_id', 'status'])
for doc_id in doc_ids:
if doc_id in iter_db.saved_ids:
status = "saved"
elif doc_id in iter_db.error_ids:
status = "errored"
else:
status = "not_found"
writer.writerow([doc_id, status])
print("Saved results to {}".format(filename))
def export_events(request):
"""
Nothing fancy, just a simple csv dump of all the WisePill event
data stored for debugging. This can't really be a domain-specific
report because we may not be able to tie an event to a domain if
the device was not configured properly in CommCareHQ.
"""
attrs = [
'_id',
'domain',
'data',
'received_on',
'case_id',
'processed',
'serial_number',
'timestamp',
]
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="device_events.csv"'
writer = csv.writer(response)
writer.writerow(attrs)
ids = WisePillDeviceEvent.get_all_ids()
for doc in iter_docs(WisePillDeviceEvent.get_db(), ids):
event = WisePillDeviceEvent.wrap(doc)
writer.writerow([getattr(event, attr) for attr in attrs])
return response
def handle(self, *args, **options):
for domain in Domain.get_all_names():
fields_definition = cdm.CustomDataFieldsDefinition.get_or_create(
domain,
'UserFields'
)
had_fields = bool(fields_definition.fields)
user_ids = (CommCareUser.ids_by_domain(domain) +
CommCareUser.ids_by_domain(domain, is_active=False))
existing_field_slugs = set([field.slug for field in fields_definition.fields])
for user in iter_docs(CommCareUser.get_db(), user_ids):
user_data = user.get('user_data', {})
for key in user_data.keys():
if (key and key not in existing_field_slugs
and not cdm.is_system_key(key)):
existing_field_slugs.add(key)
fields_definition.fields.append(cdm.CustomDataField(
slug=key,
label=key,
is_required=False,
))
for field in fields_definition.fields:
if cdm.is_system_key(field.slug):
fields_definition.fields.remove(field)
# Only save a definition for domains which use custom user data
if fields_definition.fields or had_fields:
fields_definition.save()
print 'finished domain "{}"'.format(domain.name)
def archive_forms(domain, user, uploaded_data):
response = {"errors": [], "success": []}
form_ids = [row.get("form_id") for row in uploaded_data]
missing_forms = set(form_ids)
for xform_doc in iter_docs(XFormInstance.get_db(), form_ids):
xform = XFormInstance.wrap(xform_doc)
missing_forms.discard(xform["_id"])
if xform["domain"] != domain:
response["errors"].append(
_(u"XFORM {form_id} does not belong to domain {domain}").format(
form_id=xform["_id"], domain=xform["domain"]
)
)
continue
xform_string = _(u"XFORM {form_id} for domain {domain} by user '{username}'").format(
form_id=xform["_id"], domain=xform["domain"], username=user.username
)
try:
xform.archive(user=user.username)
response["success"].append(_(u"Successfully archived {form}").format(form=xform_string))
except Exception as e:
response["errors"].append(_(u"Could not archive {form}: {error}").format(form=xform_string, error=e))
for missing_form_id in missing_forms:
response["errors"].append(_(u"Could not find XForm {form_id}").format(form_id=missing_form_id))
return response
def handle(self, *args, **options):
self.stdout.write("Processing locations...\n")
relevant_ids = set([r['id'] for r in Location.get_db().view(
'commtrack/locations_by_code',
reduce=False,
).all()])
to_save = []
for location in iter_docs(Location.get_db(), relevant_ids):
# exclude any psi domain to make this take a realistic
# amount fo time
if (
not location.get('last_modified', False) and
'psi' not in location.get('domain', '')
):
location['last_modified'] = datetime.now().isoformat()
to_save.append(location)
if len(to_save) > 500:
Location.get_db().bulk_save(to_save)
to_save = []
if to_save:
Location.get_db().bulk_save(to_save)
def handle(self, *args, **options):
start = options['startdate']
end = options['enddate']
print 'Starting...\n'
ids = get_build_ids(start, end)
print 'Checking {} builds\n'.format(len(ids))
case_types_by_domain = {}
all_form_xmlns = set()
for build in iter_docs(Application.get_db(), ids):
domain = build.get('domain')
errors = forms_with_empty_case_block(build)
if not errors:
continue
case_types, form_xmlns = errors
all_form_xmlns |= form_xmlns
domain_case_counts = case_types_by_domain.setdefault(domain, {})
case_counts = {
case_type: get_number_of_cases_in_domain(domain, case_type)
for case_type in case_types
if case_type not in domain_case_counts
}
domain_case_counts.update(case_counts)
import pprint
pprint.pprint(case_types_by_domain)
print
print all_form_xmlns
def handle(self, **options):
roles = UserRole.view(
'users/roles_by_domain',
include_docs=False,
reduce=False
).all()
for role_doc in iter_docs(UserRole.get_db(), [r['id'] for r in roles]):
role = UserRole.wrap(role_doc)
save_role = False
if role.permissions.edit_web_users:
role.permissions.view_web_users = True
role.permissions.view_roles = True
save_role = True
if role.permissions.edit_commcare_users:
role.permissions.view_commcare_users = True
role.permissions.edit_groups = True
role.permissions.view_groups = True
save_role = True
if role.permissions.edit_locations:
role.permissions.view_locations = True
save_role = True
if save_role:
role.save()
def get_latest_schema(self):
last_export = self.last_checkpoint()
schema = self.cleanup(dict(last_export.schema) if last_export else None)
doc_ids = last_export.get_new_ids(self.database) if last_export else self.all_doc_ids
for doc in iter_docs(self.database, doc_ids):
schema = extend_schema(schema, self.cleanup(doc))
return schema
def handle(self, *args, **options):
self.stdout.write("...\n")
relevant_ids = set([r['id'] for r in CouchUser.get_db().view(
'users/by_username',
reduce=False,
).all()])
to_save = []
for user_doc in iter_docs(CouchUser.get_db(), relevant_ids):
if 'commtrack_location' in user_doc:
user = CommCareUser.get(user_doc['_id'])
try:
original_location_object = Location.get(user['commtrack_location'])
except ResourceNotFound:
# if there was bad data in there before, we can ignore it
continue
user.set_locations([original_location_object])
del user_doc['commtrack_location']
to_save.append(user_doc)
if len(to_save) > 500:
CouchUser.get_db().bulk_save(to_save)
to_save = []
if to_save:
CouchUser.get_db().bulk_save(to_save)
def response_cloudcare(self):
"""
CloudCare enabled apps will have cloudcare_enabled set to false on downgrade.
"""
key = [self.domain.name]
db = Application.get_db()
domain_apps = db.view(
'app_manager/applications_brief',
reduce=False,
startkey=key,
endkey=key + [{}],
).all()
cloudcare_enabled_apps = []
for app_doc in iter_docs(db, [a['id'] for a in domain_apps]):
if app_doc.get('cloudcare_enabled', False):
cloudcare_enabled_apps.append((app_doc['_id'], app_doc['name']))
if not cloudcare_enabled_apps:
return None
num_apps = len(cloudcare_enabled_apps)
return self._fmt_alert(
ungettext(
"You have %(num_apps)d application that will lose CloudCare access if you select this plan.",
"You have %(num_apps)d applications that will lose CloudCare access if you select this plan.",
num_apps
) % {
'num_apps': num_apps,
},
[mark_safe('<a href="%(url)s">%(title)s</a>') % {
'title': a[1],
'url': reverse('view_app', args=[self.domain.name, a[0]])
} for a in cloudcare_enabled_apps],
)
def handle(self, *args, **options):
if len(args) < 2:
print "please specify a filepath and an archiving_user"
return
filepath = args[0]
archiving_user = args[1]
try:
form_ids = open(filepath)
except Exception as e:
print "there was an issue opening the file: %s" % e
return
try:
form_ids = [f[0] for f in csv.reader(form_ids)]
except Exception as e:
print "there was an issue reading the file %s" % e
return
for xform_doc in iter_docs(XFormInstance.get_db(), form_ids):
try:
xform = XFormInstance.wrap(xform_doc)
xform.archive(user_id=archiving_user)
print "Archived form %s in domain %s" % (
xform._id, xform.domain
)
except Exception as e:
print "Issue archiving XFORM %s for domain %s: %s" % (
xform_doc['_id'], xform_doc['domain'], e
)
def copy_docs(self, sourcedb, domain, simulate, startkey=None, endkey=None, doc_ids=None,
type=None, since=None, exclude_types=None, postgres_db=None, exclude_attachments=False):
if not doc_ids:
doc_ids = [result["id"] for result in sourcedb.view("domain/docs", startkey=startkey,
endkey=endkey, reduce=False)]
total = len(doc_ids)
count = 0
msg = "Found %s matching documents in domain: %s" % (total, domain)
msg += " of type: %s" % (type) if type else ""
msg += " since: %s" % (since) if since else ""
print msg
err_log = self._get_err_log()
queue = Queue(150)
for i in range(NUM_PROCESSES):
Worker(queue, sourcedb, self.targetdb, exclude_types, total, simulate, err_log, exclude_attachments).start()
for doc in iter_docs(sourcedb, doc_ids, chunksize=100):
count += 1
queue.put((doc, count))
# shutdown workers
for i in range(NUM_PROCESSES):
queue.put(None)
err_log.close()
if os.stat(err_log.name)[6] == 0:
os.remove(err_log.name)
else:
print 'Failed document IDs written to %s' % err_log.name
if postgres_db:
copy_postgres_data_for_docs(postgres_db, doc_ids=doc_ids, simulate=simulate)
def get_forms(self):
"""
Gets the form docs associated with a case. If it can't find a form
it won't be included.
"""
forms = iter_docs(self.get_db(), self.xform_ids)
return [XFormInstance(form) for form in forms]
def get_cases(cls, ids):
return [
CommCareCase.wrap(doc) for doc in iter_docs(
CommCareCase.get_db(),
ids
)
]
def _build_indicators(indicator_config_id, relevant_ids):
config = _get_config_by_id(indicator_config_id)
adapter = IndicatorSqlAdapter(config)
couchdb = _get_db(config.referenced_doc_type)
redis_client = get_redis_client().client.get_client()
redis_key = _get_redis_key_for_config(config)
for doc in iter_docs(couchdb, relevant_ids, chunksize=500):
try:
# save is a noop if the filter doesn't match
adapter.save(doc)
redis_client.srem(redis_key, doc.get('_id'))
except Exception as e:
logging.exception('problem saving document {} to table. {}'.format(doc['_id'], e))
if not is_static(indicator_config_id):
redis_client.delete(redis_key)
config.meta.build.finished = True
try:
config.save()
except ResourceConflict:
current_config = DataSourceConfiguration.get(config._id)
# check that a new build has not yet started
if config.meta.build.initiated == current_config.meta.build.initiated:
current_config.meta.build.finished = True
current_config.save()
def update_schema(self):
key = [self.domain, self.app_id]
all_apps = Application.get_db().view(
'app_manager/saved_app',
startkey=key + [self.last_processed_version],
endkey=key + [{}],
reduce=False,
include_docs=False,
skip=(1 if self.last_processed_version else 0)).all()
all_seen_apps = self.apps_with_errors | self.processed_apps
to_process = [
app['id'] for app in all_apps if app['id'] not in all_seen_apps
]
if self.app_id not in all_seen_apps:
to_process.append(self.app_id)
for app_doc in iter_docs(Application.get_db(), to_process):
if app_doc['doc_type'] == 'RemoteApp':
continue
app = Application.wrap(app_doc)
try:
self.update_for_app(app)
except AppManagerException:
self.apps_with_errors.add(app.get_id)
self.last_processed_version = app.version
if to_process:
self.save()
def handle(self, *args, **options):
domain, group_name = args
group = Group.by_name(domain, name=group_name)
owner_ids = get_all_owner_ids_from_group(group)
pillow = CareBiharFluffPillow()
db = CommCareCase.get_db()
greenlets = []
def process_case(case):
pillow.change_transport(pillow.change_transform(case))
for i, owner_id in enumerate(owner_ids):
print '{0}/{1} owner_ids'.format(i, len(owner_ids))
rows = CommCareCase.view(
'hqcase/by_owner',
startkey=[domain, owner_id],
endkey=[domain, owner_id, {}],
reduce=False,
).all()
case_ids = [row['id'] for row in rows]
print '{0} case_ids'.format(len(case_ids))
for case in iter_docs(db, case_ids):
g = gevent.Greenlet.spawn(process_case, case)
greenlets.append(g)
gevent.joinall(greenlets)
def get_cases(case_ids, ordered=False):
return [
CommCareCase.wrap(doc) for doc in iter_docs(
CommCareCase.get_db(),
case_ids
)
]
def all(cls):
ids = [
res["id"]
for res in cls.get_db().view("userreports/data_sources_by_domain", reduce=False, include_docs=False)
]
for result in iter_docs(cls.get_db(), ids):
yield cls.wrap(result)
def get_rows(self, datespan):
def get_awc_filter(awcs):
return get_nested_terms_filter("awc_name.#value", awcs)
def get_gp_filter(gp):
owner_ids = [user._id for user in self.users
if getattr(user, 'user_data', {}).get('gp') in self.gp]
return es_filters.term("owner_id", owner_ids)
def get_block_filter(block):
return es_filters.term("block_name.#value", block.lower())
query = case_es.CaseES().domain(self.domain)\
.fields([])\
.opened_range(lte=self.datespan.enddate_utc)\
.term("type.exact", self.default_case_type)
query.index = 'report_cases'
if self.display_open_cases_only:
query = query.filter(es_filters.OR(
case_es.is_closed(False),
case_es.closed_range(gte=self.datespan.enddate_utc)
))
elif self.display_closed_cases_only:
query = query.filter(case_es.closed_range(lte=self.datespan.enddate_utc))
if self.awcs:
query = query.filter(get_awc_filter(self.awcs))
elif self.gp:
query = query.filter(get_gp_filter(self.gp))
elif self.block:
query = query.filter(get_block_filter(self.block))
result = query.run()
return map(CommCareCase, iter_docs(CommCareCase.get_db(), result.ids))
def handle_one(self, domain, case_type, chunk_size):
self.log('Copying {case_type} cases in {domain}'
.format(case_type=case_type, domain=domain))
old_db = CommCareCase.get_db()
new_db = IndicatorCase.get_db()
assert old_db.uri != new_db.uri
# this dbaccessor pulls from old_db
case_ids = get_case_ids_in_domain(domain, case_type)
self.delete_bad_doc_types(case_ids, chunk_size)
case_dict_chunks = chunked(iter_docs(old_db, case_ids, chunk_size),
chunk_size)
for case_dicts in case_dict_chunks:
for case_dict in case_dicts:
del case_dict['_rev']
case_dict.pop('_attachments', None)
case_dict['doc_type'] = "IndicatorCase"
try:
results = new_db.bulk_save(case_dicts)
except BulkSaveError as error:
results = error.results
for result in results:
if result.get('error') == 'conflict':
self.log('- OK: [{id}] is already in the indicator db'
.format(id=result.get('id')))
elif 'error' in result:
self.log('- ERROR: [{id}] ({result})'.format(
id=result.get('id'),
result=json.dumps(result)
))
else:
self.log('- ADDED: [{id}] saved to indicator db'.format(
id=result.get('id')
))
def cache_users_at_location(self, selected_users):
user_cache_list = []
for doc in iter_docs(CommCareUser.get_db(), selected_users):
display_username = user_display_string(
doc['username'], doc.get('first_name', ''), doc.get('last_name', ''))
user_cache_list.append({'text': display_username, 'id': doc['_id']})
self.get_users_at_location.set_cached_value(self).to(user_cache_list)
def _delete_couch_data(self):
for doc_class, doc_ids in get_doc_ids_to_dump(self.domain_name):
db = doc_class.get_db()
for docs in chunked(iter_docs(db, doc_ids), 100):
db.bulk_delete(docs)
self.assertEqual(0, len(get_docs(db, doc_ids)))
def generate_schema_from_builds(domain, case_type):
"""Builds a schema from Application builds for a given identifier
:param domain: The domain that the export belongs to
:param unique_form_id: The unique identifier of the item being exported
:returns: Returns a ExportDataSchema instance
"""
app_build_ids = get_all_app_ids(domain)
all_case_schema = CaseExportDataSchema()
for app_doc in iter_docs(Application.get_db(), app_build_ids):
app = Application.wrap(app_doc)
case_property_mapping = get_case_properties(
app,
[case_type],
include_parent_properties=False
)
case_schema = CaseExportDataSchema._generate_schema_from_case_property_mapping(
case_property_mapping,
app.version,
)
case_history_schema = CaseExportDataSchema._generate_schema_for_case_history(
case_property_mapping,
app.version,
)
all_case_schema = CaseExportDataSchema._merge_schemas(
all_case_schema,
case_schema,
case_history_schema
)
return all_case_schema
def rebuild_indicators(indicator_config_id):
is_static = indicator_config_id.startswith(CustomDataSourceConfiguration._datasource_id_prefix)
if is_static:
config = CustomDataSourceConfiguration.by_id(indicator_config_id)
else:
config = DataSourceConfiguration.get(indicator_config_id)
# Save the start time now in case anything goes wrong. This way we'll be
# able to see if the rebuild started a long time ago without finishing.
config.meta.build.initiated = datetime.datetime.utcnow()
config.save()
adapter = IndicatorSqlAdapter(config)
adapter.rebuild_table()
couchdb = _get_db(config.referenced_doc_type)
relevant_ids = get_doc_ids(config.domain, config.referenced_doc_type,
database=couchdb)
for doc in iter_docs(couchdb, relevant_ids, chunksize=500):
try:
# save is a noop if the filter doesn't match
adapter.save(doc)
except DataError as e:
logging.exception('problem saving document {} to table. {}'.format(doc['_id'], e))
if not is_static:
config.meta.build.finished = True
config.save()
def cases(self):
if 'debug_case' in self.request.GET:
case = CommCareCase.get(self.request.GET['debug_case'])
if case.domain != DOMAIN:
raise Http404()
return [case]
query = case_es.CaseES().domain(self.domain)\
.fields([])\
.opened_range(lte=self.datespan.enddate_utc)\
.case_type(self.default_case_type)
query.index = 'report_cases'
if self.case_status == 'open':
query = query.filter(es_filters.OR(
case_es.is_closed(False),
case_es.closed_range(gte=self.datespan.enddate_utc)
))
elif self.case_status == 'closed':
query = query.filter(case_es.closed_range(lte=self.datespan.enddate_utc))
query = query.owner([user['doc_id'] for user in self.users_matching_filter])
result = query.run()
return [
CommCareCase.wrap(doc)
for doc in iter_docs(CommCareCase.get_db(), result.ids)
]
def get_media_objects(self, languages=None):
"""
Gets all the media objects stored in the multimedia map.
If passed a profile, will only get those that are used
in a language in the profile.
"""
found_missing_mm = False
filter_multimedia = languages and self.media_language_map
if filter_multimedia:
requested_media = set()
for lang in languages:
requested_media.update(self.media_language_map[lang].media_refs)
# preload all the docs to avoid excessive couch queries.
# these will all be needed in memory anyway so this is ok.
expected_ids = [map_item.multimedia_id for map_item in self.multimedia_map.values()]
raw_docs = dict((d["_id"], d) for d in iter_docs(CommCareMultimedia.get_db(), expected_ids))
for path, map_item in self.multimedia_map.items():
if not filter_multimedia or not map_item.form_media or path in requested_media:
media_item = raw_docs.get(map_item.multimedia_id)
if media_item:
media_cls = CommCareMultimedia.get_doc_class(map_item.media_type)
yield path, media_cls.wrap(media_item)
else:
# delete media reference from multimedia map so this doesn't pop up again!
del self.multimedia_map[path]
found_missing_mm = True
if found_missing_mm:
self.save()
def get_cases(case_ids, ordered=False, prefetched_indices=None):
# prefetched_indices is ignored sinces cases already have them
return [
CommCareCase.wrap(doc)
for doc in iter_docs(CommCareCase.get_db(), case_ids)
]
def _get_products(domain):
for p_doc in iter_docs(Product.get_db(), Product.ids_by_domain(domain)):
# filter out archived products from export
if not ('is_archived' in p_doc and p_doc['is_archived']):
yield Product.wrap(p_doc)
def record_iter(cls, start_datetime, end_datetime):
group_ids = get_group_ids_by_last_modified(start_datetime,
end_datetime)
return iter_docs(Group.get_db(), group_ids)
def all(cls):
for result in iter_docs(cls.get_db(), cls.all_ids()):
yield cls.wrap(result)
def assign_users(self, users):
for doc in iter_docs(CommCareUser.get_db(), users):
CommCareUser.wrap(doc).add_to_assigned_locations(self.location)
def unassign_users(self, users):
for doc in iter_docs(CommCareUser.get_db(), users):
# This could probably be sped up by bulk saving, but there's a lot
# of stuff going on - seems tricky.
CommCareUser.wrap(doc).unset_location_by_id(
self.location.location_id, fall_back_to_next=True)
def get_new_docs(self, database=None):
return iter_docs(self.get_new_ids(database))
def filter_cases(request, domain, app_id, module_id):
app = Application.get(app_id)
module = app.get_module(module_id)
delegation = request.GET.get('task-list') == 'true'
auth_cookie = request.COOKIES.get('sessionid')
suite_gen = SuiteGenerator(app)
xpath = suite_gen.get_filter_xpath(module, delegation=delegation)
extra_instances = [{
'id': inst.id,
'src': inst.src
} for inst in suite_gen.get_extra_instances(module)]
# touchforms doesn't like this to be escaped
xpath = HTMLParser.HTMLParser().unescape(xpath)
if delegation:
case_type = DELEGATION_STUB_CASE_TYPE
else:
case_type = module.case_type
if xpath:
# if we need to do a custom filter, send it to touchforms for processing
additional_filters = {
"properties/case_type": case_type,
"footprint": True
}
helper = SessionDataHelper(domain, request.couch_user)
result = helper.filter_cases(xpath,
additional_filters,
DjangoAuth(auth_cookie),
extra_instances=extra_instances)
if result.get('status', None) == 'error':
return HttpResponseServerError(
result.get("message",
_("Something went wrong filtering your cases.")))
case_ids = result.get("cases", [])
else:
# otherwise just use our built in api with the defaults
case_ids = [
res.id
for res in get_filtered_cases(domain,
status=CASE_STATUS_OPEN,
case_type=case_type,
user_id=request.couch_user._id,
ids_only=True)
]
cases = [
CommCareCase.wrap(doc)
for doc in iter_docs(CommCareCase.get_db(), case_ids)
]
# refilter these because we might have accidentally included footprint cases
# in the results from touchforms. this is a little hacky but the easiest
# (quick) workaround. should be revisted when we optimize the case list.
cases = filter(lambda c: c.type == case_type, cases)
cases = [c.get_json(lite=True) for c in cases if c]
parents = []
if delegation:
for case in cases:
parent_id = case['indices']['parent']['case_id']
parents.append(CommCareCase.get(parent_id))
return json_response({'cases': cases, 'parents': parents})
else:
return json_response(cases)
def get_all_commcare_users_by_domain(domain):
"""Returns all CommCareUsers by domain regardless of their active status"""
ids = get_all_user_ids_by_domain(domain, include_web_users=False)
return map(CommCareUser.wrap, iter_docs(CommCareUser.get_db(), ids))
def get_data(self):
# todo: this will probably have to paginate eventually
if self.all_relevant_forms:
sp_ids = get_relevant_supply_point_ids(
self.domain,
self.active_location,
)
form_xmlnses = [
form['xmlns'] for form in self.all_relevant_forms.values()
]
spoint_loc_map = {
doc['_id']: doc['location_id']
for doc in iter_docs(SupplyPointCase.get_db(), sp_ids)
}
locations = _location_map(spoint_loc_map.values())
for spoint_id, loc_id in spoint_loc_map.items():
if loc_id not in locations:
continue # it's archived, skip
loc = locations[loc_id]
results = StockReport.objects.filter(
stocktransaction__case_id=spoint_id).filter(
date__gte=self.converted_start_datetime,
date__lte=self.converted_end_datetime).values_list(
'form_id', 'date').distinct(
) # not truly distinct due to ordering
matched = False
for form_id, date in results:
try:
if XFormInstance.get(form_id).xmlns in form_xmlnses:
yield {
'parent_name':
loc.parent.name if loc.parent else '',
'loc_id': loc.location_id,
'loc_path': loc.path,
'name': loc.name,
'type': loc.location_type.name,
'reporting_status': 'reporting',
'geo': geopoint(loc),
'last_reporting_date': date,
}
matched = True
break
except ResourceNotFound:
logging.error(
'Stock report for location {} in {} references non-existent form {}'
.format(loc.location_id, loc.domain, form_id))
if not matched:
result = StockReport.objects.filter(
stocktransaction__case_id=spoint_id).values_list(
'date').order_by('-date')[:1]
yield {
'parent_name': loc.parent.name if loc.parent else '',
'loc_id': loc.location_id,
'loc_path': loc.path,
'name': loc.name,
'type': loc.location_type.name,
'reporting_status': 'nonreporting',
'geo': geopoint(loc),
'last_reporting_date': result[0][0] if result else ''
}
def choices_by_domain(cls, domain):
group_ids = cls.ids_by_domain(domain)
group_choices = []
for group_doc in iter_docs(cls.get_db(), group_ids):
group_choices.append((group_doc['_id'], group_doc['name']))
return group_choices
def iter_documents(self, ids):
return iter_docs(self._couch_db, ids, chunksize=500)
def iter_cases(case_ids, wrap=True):
from casexml.apps.case.models import CommCareCase
for doc in iter_docs(CommCareCase.get_db(), case_ids):
yield CommCareCase.wrap(doc) if wrap else doc
def get_users(self):
user_ids = CommCareUser.ids_by_domain(self.domain)
for user_doc in iter_docs(CommCareUser.get_db(), user_ids):
yield CommCareUser.wrap(user_doc)
def _get_related_cases(results):
ids = filter(None, [_related_id(res) for res in results])
return dict((c['_id'], c) for c in iter_docs(CommCareCase.get_db(), ids))
def generate_invoices(based_on_date=None):
"""
Generates all invoices for the past month.
"""
today = based_on_date or datetime.date.today()
invoice_start, invoice_end = get_previous_month_date_range(today)
log_accounting_info("Starting up invoices for %(start)s - %(end)s" % {
'start': invoice_start.strftime(USER_DATE_FORMAT),
'end': invoice_end.strftime(USER_DATE_FORMAT),
})
all_domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)]
for domain_doc in iter_docs(Domain.get_db(), all_domain_ids):
domain_obj = Domain.wrap(domain_doc)
if not domain_obj.is_active:
continue
try:
invoice_factory = DomainInvoiceFactory(invoice_start, invoice_end, domain_obj)
invoice_factory.create_invoices()
log_accounting_info("Sent invoices for domain %s" % domain_obj.name)
except CreditLineError as e:
log_accounting_error(
"There was an error utilizing credits for "
"domain %s: %s" % (domain_obj.name, e),
show_stack_trace=True,
)
except InvoiceError as e:
log_accounting_error(
"Could not create invoice for domain %s: %s" % (domain_obj.name, e),
show_stack_trace=True,
)
except Exception as e:
log_accounting_error(
"Error occurred while creating invoice for "
"domain %s: %s" % (domain_obj.name, e),
show_stack_trace=True,
)
all_customer_billing_accounts = BillingAccount.objects.filter(is_customer_billing_account=True)
for account in all_customer_billing_accounts:
try:
if account.invoicing_plan == InvoicingPlan.QUARTERLY:
customer_invoice_start = invoice_start - relativedelta(months=2)
elif account.invoicing_plan == InvoicingPlan.YEARLY:
customer_invoice_start = invoice_start - relativedelta(months=11)
else:
customer_invoice_start = invoice_start
invoice_factory = CustomerAccountInvoiceFactory(
account=account,
date_start=customer_invoice_start,
date_end=invoice_end
)
invoice_factory.create_invoice()
except CreditLineError as e:
log_accounting_error(
"There was an error utilizing credits for "
"domain %s: %s" % (domain_obj.name, e),
show_stack_trace=True,
)
except InvoiceError as e:
log_accounting_error(
"Could not create invoice for domain %s: %s" % (domain_obj.name, e),
show_stack_trace=True,
)
except Exception as e:
log_accounting_error(
"Error occurred while creating invoice for "
"domain %s: %s" % (domain_obj.name, e),
show_stack_trace=True,
)
if not settings.UNIT_TESTING:
_invoicing_complete_soft_assert(False, "Invoicing is complete!")
def build_form_multimedia_zip(domain, xmlns, startdate, enddate, app_id, export_id, zip_name, download_id):
def find_question_id(form, value):
for k, v in form.iteritems():
if isinstance(v, dict):
ret = find_question_id(v, value)
if ret:
return [k] + ret
else:
if v == value:
return [k]
return None
def filename(form_info, question_id, extension):
fname = u"%s-%s-%s-%s%s"
if form_info['cases']:
fname = u'-'.join(form_info['cases']) + u'-' + fname
return fname % (form_info['name'],
unidecode(question_id),
form_info['user'],
form_info['id'], extension)
case_ids = set()
def extract_form_info(form, properties=None, case_ids=case_ids):
unknown_number = 0
meta = form['form'].get('meta', dict())
# get case ids
case_blocks = extract_case_blocks(form)
cases = {c['@case_id'] for c in case_blocks}
case_ids |= cases
form_info = {
'form': form,
'attachments': list(),
'name': form['form'].get('@name', 'unknown form'),
'user': meta.get('username', 'unknown_user'),
'cases': cases,
'id': form['_id']
}
for k, v in form['_attachments'].iteritems():
if v['content_type'] == 'text/xml':
continue
try:
question_id = unicode(u'-'.join(find_question_id(form['form'], k)))
except TypeError:
question_id = unicode(u'unknown' + unicode(unknown_number))
unknown_number += 1
if not properties or question_id in properties:
extension = unicode(os.path.splitext(k)[1])
form_info['attachments'].append({
'size': v['length'],
'name': k,
'question_id': question_id,
'extension': extension,
'timestamp': parse(form['received_on']).timetuple(),
})
return form_info
key = [domain, app_id, xmlns]
form_ids = {f['id'] for f in XFormInstance.get_db().view("attachments/attachments",
start_key=key + [startdate],
end_key=key + [enddate, {}],
reduce=False)}
properties = set()
if export_id:
schema = FormExportSchema.get(export_id)
for table in schema.tables:
# - in question id is replaced by . in excel exports
properties |= {c.display.replace('.', '-') for c in table.columns}
if not app_id:
zip_name = 'Unrelated Form'
forms_info = list()
for form in iter_docs(XFormInstance.get_db(), form_ids):
if not zip_name:
zip_name = unidecode(form['form'].get('@name', 'unknown form'))
forms_info.append(extract_form_info(form, properties))
num_forms = len(forms_info)
DownloadBase.set_progress(build_form_multimedia_zip, 0, num_forms)
# get case names
case_id_to_name = {c: c for c in case_ids}
for case in iter_docs(CommCareCase.get_db(), case_ids):
if case['name']:
case_id_to_name[case['_id']] = case['name']
use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled
if use_transfer:
params = '_'.join(map(str, [xmlns, startdate, enddate, export_id, num_forms]))
fname = '{}-{}'.format(app_id, hashlib.md5(params).hexdigest())
fpath = os.path.join(settings.SHARED_DRIVE_CONF.transfer_dir, fname)
else:
_, fpath = tempfile.mkstemp()
if not (os.path.isfile(fpath) and use_transfer): # Don't rebuild the file if it is already there
with open(fpath, 'wb') as zfile:
with zipfile.ZipFile(zfile, 'w') as z:
for form_number, form_info in enumerate(forms_info):
f = XFormInstance.wrap(form_info['form'])
form_info['cases'] = {case_id_to_name[case_id] for case_id in form_info['cases']}
for a in form_info['attachments']:
fname = filename(form_info, a['question_id'], a['extension'])
zi = zipfile.ZipInfo(fname, a['timestamp'])
z.writestr(zi, f.fetch_attachment(a['name'], stream=True).read(), zipfile.ZIP_STORED)
DownloadBase.set_progress(build_form_multimedia_zip, form_number + 1, num_forms)
common_kwargs = dict(
mimetype='application/zip',
content_disposition='attachment; filename="{fname}.zip"'.format(fname=zip_name),
download_id=download_id,
)
if use_transfer:
expose_file_download(
fpath,
use_transfer=use_transfer,
**common_kwargs
)
else:
expose_cached_download(
FileWrapper(open(fpath)),
expiry=(1 * 60 * 60),
**common_kwargs
)
DownloadBase.set_progress(build_form_multimedia_zip, num_forms, num_forms)
def all(cls):
ids = [res['id'] for res in cls.get_db().view('userreports/data_sources_by_build_info',
reduce=False, include_docs=False)]
for result in iter_docs(cls.get_db(), ids):
yield cls.wrap(result)
def get_open_case_docs_in_domain(domain, type=None, owner_id=None):
from casexml.apps.case.models import CommCareCase
case_ids = get_open_case_ids_in_domain(domain, type, owner_id)
for doc in iter_docs(CommCareCase.get_db(), case_ids):
yield doc
def rendered_content(self):
from corehq.apps.users.views.mobile.users import EditCommCareUserView
users = get_users_by_location_id(self.config['domain'],
self.config['location_id'])
in_charges = FacilityInCharge.objects.filter(
location=self.location).values_list('user_id', flat=True)
if self.location.parent.location_type.name == 'district':
children = self.location.parent.get_descendants()
availaible_in_charges = list(
chain.from_iterable([
filter(
lambda u: 'In Charge' in u.user_data.get('role', []),
get_users_by_location_id(self.config['domain'],
child.location_id))
for child in children
]))
else:
availaible_in_charges = filter(
lambda u: 'In Charge' in u.user_data.get('role', []),
get_users_by_location_id(self.domain, self.location_id))
user_to_dict = lambda sms_user: {
'id':
sms_user.get_id,
'full_name':
sms_user.full_name,
'phone_numbers':
sms_user.phone_numbers,
'in_charge':
sms_user.get_id in in_charges,
'location_name':
sms_user.location.sql_location.name,
'url':
reverse(EditCommCareUserView.urlname,
args=[self.config['domain'], sms_user.get_id])
}
web_users_from_extension = list(
iter_docs(
WebUser.get_db(),
EWSExtension.objects.filter(
domain=self.domain,
location_id=self.location_id).values_list('user_id',
flat=True)))
WebUserInfo = collections.namedtuple('WebUserInfo',
'id first_name last_name email')
web_users = {
WebUserInfo(id=web_user['_id'],
first_name=web_user['first_name'],
last_name=web_user['last_name'],
email=web_user['email'])
for web_user in (UserES().web_users().domain(
self.config['domain']).term("domain_memberships.location_id",
self.location_id).run().hits +
web_users_from_extension)
}
return render_to_string(
'ewsghana/partials/users_tables.html', {
'users': [user_to_dict(user) for user in users],
'domain':
self.domain,
'location_id':
self.location_id,
'web_users':
web_users,
'district_in_charges':
[user_to_dict(user) for user in availaible_in_charges]
})
def _get_products(domain):
product_ids = SQLProduct.objects.filter(domain=domain).product_ids()
for p_doc in iter_docs(Product.get_db(), product_ids):
# filter out archived products from export
if not ('is_archived' in p_doc and p_doc['is_archived']):
yield Product.wrap(p_doc)
def record_iter(cls, start_datetime, end_datetime):
domain_ids = get_domain_ids_by_last_modified(start_datetime,
end_datetime)
return iter_docs(Domain.get_db(), domain_ids)
def record_iter(cls, start_datetime, end_datetime):
application_ids = get_application_ids_by_last_modified(
start_datetime, end_datetime)
return iter_docs(Application.get_db(), application_ids)
def get_all_data_sources(self):
active_ids = get_all_registry_data_source_ids(is_active=True)
for result in iter_docs(RegistryDataSourceConfiguration.get_db(),
active_ids):
yield RegistryDataSourceConfiguration.wrap(result)
def record_iter(cls, start_datetime, end_datetime):
user_ids = get_user_ids_by_last_modified(start_datetime, end_datetime)
return iter_docs(CouchUser.get_db(), user_ids)