def lead_time(self, closed_date, opened_date):
try:
closed_date = iso_string_to_datetime(closed_date)
opened_date = iso_string_to_datetime(opened_date)
days_rest_delta = (((closed_date - opened_date).seconds / 3600)*10)/24
return "%s.%s" % ((closed_date - opened_date).days, days_rest_delta)
except TypeError:
return _("---")
def has_acceptable_date_diff(old_obj, new_obj, field, delta=timedelta(days=1)):
old = old_obj.get(field)
new = new_obj.get(field)
if _both_dates(old, new):
old = iso_string_to_datetime(old)
new = iso_string_to_datetime(new)
return abs(old - new) < delta
return False
def is_datetime_string(string):
if not isinstance(string, basestring):
return False
try:
iso_string_to_datetime(string, strict=True)
except (ValueError, OverflowError, TypeError):
return False
else:
return True
def is_datetime_string(string):
if not isinstance(string, str):
return False
try:
iso_string_to_datetime(string, strict=True)
except (ValueError, OverflowError, TypeError):
return False
else:
return True
def is_datetime_string(string):
if not isinstance(string, six.string_types):
return False
soft_assert_type_text(string)
try:
iso_string_to_datetime(string, strict=True)
except (ValueError, OverflowError, TypeError):
return False
else:
return True
def is_datetime_string(string):
if not isinstance(string, six.string_types):
return False
soft_assert_type_text(string)
try:
iso_string_to_datetime(string, strict=True)
except (ValueError, OverflowError, TypeError):
return False
else:
return True
def get_schedule(cls, chw_username, override_date=None):
"""
Generate schedule object for a given username
"""
cached_schedules = None
if override_date == None:
nowdate = datetime.utcnow()
else:
nowdate = override_date
day_intervaltree = {}
if cached_schedules == None:
#no documents, then we need to load them up
db = CommCareCase.get_db()
chw_schedules = db.view('pact/chw_dot_schedules',
key=chw_username).all()
to_cache = []
for item in chw_schedules:
single_sched = item['value']
to_cache.append(single_sched)
cache.set("%s_schedule" % (chw_username), json.dumps(to_cache),
3600)
cached_arr = to_cache
else:
cached_arr = json.loads(cached_schedules)
for single_sched in cached_arr:
day_of_week = int(single_sched['day_of_week'])
if day_of_week in day_intervaltree:
daytree = day_intervaltree[day_of_week]
else:
#if there's no day of week indication for this, then it's just a null interval node. To start this node, we make it REALLY old.
daytree = IntervalNode(
get_seconds(datetime.min),
get_seconds(nowdate + timedelta(days=10)))
if single_sched['ended_date'] == None:
enddate = nowdate + timedelta(days=9)
else:
enddate = iso_string_to_datetime(single_sched['ended_date'])
startdate = iso_string_to_datetime(single_sched['active_date'])
case_id = single_sched['case_id']
if 'error' in single_sched:
#this is a non-showstopping issue due to quirks with older submissions
logging.error("Error, no pactid: %s" % single_sched['error'])
daytree.insert(get_seconds(startdate),
get_seconds(enddate),
other=case_id)
day_intervaltree[day_of_week] = daytree
return cls(chw_username, day_intervaltree, cached_arr)
def _get_es_modified_dates(case_ids):
results = (CaseES(es_instance_alias=ES_EXPORT_INSTANCE).case_ids(
case_ids).values_list('_id', 'server_modified_on', 'domain'))
return {
_id: (iso_string_to_datetime(server_modified_on), domain)
for _id, server_modified_on, domain in results
}
def _get_es_modified_dates_for_forms(form_ids):
results = (FormES(for_export=True).remove_default_filters().form_ids(
form_ids).values_list('_id', 'received_on', 'doc_type', 'domain'))
return {
_id: (iso_string_to_datetime(received_on), doc_type, domain)
for _id, received_on, doc_type, domain in results
}
def get_last_form_submission_received(domain):
result = _received_on_query(domain, desc=True)[0]
if not result:
return
return iso_string_to_datetime(result[0]['received_on'])
def submission_or_completion_time(self):
time = iso_string_to_datetime(safe_index(self.form, self.report.time_field.split('.')))
if self.report.by_submission_time:
user_time = ServerTime(time).user_time(self.report.timezone)
else:
user_time = PhoneTime(time, self.report.timezone).user_time(self.report.timezone)
return user_time.ui_string(USER_DATETIME_FORMAT_WITH_SEC)
def display_time(row, display=True):
submission_time = row["key"][2]
if display:
return iso_string_to_datetime(submission_time).strftime(
DISPLAY_DATE_FORMAT)
else:
return submission_time
def make_rows(self, es_results, with_checkbox):
submissions = [
res['_source']
for res in self.es_results.get('hits', {}).get('hits', [])
]
for form in submissions:
data = calculate_form_data(self, form)
row = [
json_format_date(
iso_string_to_datetime(form["form"]["meta"]["timeEnd"])),
self._get_case_name_html(data.get('case'), with_checkbox),
self._get_service_type_html(form, data.get('service_type'),
with_checkbox),
data.get('location_name'),
get_property(data.get('case'), "card_number", EMPTY_FIELD),
data.get('location_parent_name'),
get_property(data.get('case'), "phone_number", EMPTY_FIELD),
data.get('amount_due'),
get_property(data.get('case'), "visits", EMPTY_FIELD)
]
if with_checkbox:
checkbox = mark_safe(
'<input type="checkbox" class="selected-element" '
'data-formid="%(form_id)s" '
'data-caseid="%(case_id)s" data-servicetype="%(service_type)s"/>'
)
row.append(checkbox %
dict(form_id=data.get('form_id'),
case_id=data.get('case_id'),
service_type=data.get('service_type')))
else:
row.append(self.display_status)
yield row
def _get_data_for_couch_backend(run_config):
if run_config.case_type:
raise CommandError(
'Case type argument is not supported for couch domains!')
domain = run_config.domain
start_time = datetime.utcnow()
chunk_size = 1000
chunked_iterator = chunked(
paginate_view(CommCareCase.get_db(),
'cases_by_server_date/by_server_modified_on',
chunk_size=chunk_size,
startkey=[domain],
endkey=[domain, {}],
include_docs=False,
reduce=False), chunk_size)
for chunk in chunked_iterator:
case_ids = [row['id'] for row in chunk]
es_modified_on_by_ids = _get_es_modified_dates(domain, case_ids)
for row in chunk:
case_id, couch_modified_on = row['id'], row['value']
if iso_string_to_datetime(couch_modified_on) > start_time:
# skip cases modified after the script started
continue
es_modified_on = es_modified_on_by_ids.get(case_id)
if not es_modified_on or (es_modified_on != couch_modified_on):
yield (case_id, 'COUCH_TYPE_NOT_SUPPORTED', es_modified_on,
couch_modified_on)
def submit_xform_rf(self, action, submission_xml_string):
# received_on = request.META.get('HTTP_X_SUBMIT_TIME')
# date_header = request.META.get('HTTP_DATE')
# if received_on:
# doc.received_on = string_to_datetime(received_on)
# if date_header:
# comes in as:
# Mon, 11 Apr 2011 18:24:43 GMT
# goes out as:
# 2011-04-11T18:24:43Z
# try:
# date = datetime.strptime(date_header, "%a, %d %b %Y %H:%M:%S GMT")
# date = datetime.strftime(date, "%Y-%m-%dT%H:%M:%SZ")
server_date = action.get("server_date", None)
phone_date = action.get("date", None)
extra_meta = {}
if phone_date:
date = iso_string_to_datetime(phone_date)
date = datetime.strftime(date, "%a, %d %b %Y %H:%M:%S GMT")
extra_meta["HTTP_DATE"] = date
if server_date:
extra_meta["HTTP_X_SUBMIT_TIME"] = server_date
else:
if phone_date is not None:
extra_meta["HTTP_X_SUBMIT_TIME"] = phone_date
submit_xform("/a/pact/receiver", PACT_DOMAIN, submission_xml_string, extra_meta=extra_meta)
def get_300th_form_submission_received(domain):
result = FormES().domain(domain).start(300).size(1).sort(
'received_on').fields(['received_on']).run().hits
if not result:
return
return iso_string_to_datetime(result[0]['received_on'])
def make_rows(self, es_results, with_checkbox):
submissions = [res['_source'] for res in self.es_results.get('hits', {}).get('hits', [])]
for form in submissions:
data = calculate_form_data(self, form)
try:
status_data = McctStatus.objects.get(domain=self.domain, form_id=data.get('form_id'))
reason = status_data.reason
except McctStatus.DoesNotExist:
reason = None
row = [
iso_string_to_datetime(form["form"]["meta"]["timeEnd"]).strftime(SERVER_DATETIME_FORMAT_NO_SEC),
self._get_case_name_html(data.get('case'), with_checkbox),
self._get_service_type_html(form, data.get('service_type'), with_checkbox),
data.get('location_name'),
get_property(data.get('case'), "card_number", EMPTY_FIELD),
data.get('location_parent_name'),
get_property(data.get('case'), "phone_number", EMPTY_FIELD),
data.get('amount_due'),
REJECTION_REASON_DISPLAY_NAMES[reason] if reason is not None else '',
form["form"]["meta"]["username"]
]
if with_checkbox:
checkbox = mark_safe('<input type="checkbox" class="selected-element" '
'data-formid="%(form_id)s" '
'data-caseid="%(case_id)s" data-servicetype="%(service_type)s"/>')
row.append(checkbox % dict(form_id=data.get('form_id'), case_id=data.get('case_id'),
service_type=data.get('service_type')))
else:
row.insert(8, self.display_status)
yield row
def submission_or_completion_time(self):
time = iso_string_to_datetime(safe_index(self.form, self.report.time_field.split('.')))
if self.report.by_submission_time:
user_time = ServerTime(time).user_time(self.report.timezone)
else:
user_time = PhoneTime(time, self.report.timezone).user_time(self.report.timezone)
return user_time.ui_string(USER_DATETIME_FORMAT_WITH_SEC)
def make_rows(self, es_results, with_checkbox):
submissions = [res['_source'] for res in self.es_results.get('hits', {}).get('hits', [])]
for form in submissions:
data = calculate_form_data(self, form)
try:
status_data = McctStatus.objects.get(domain=self.domain, form_id=data.get('form_id'))
reason = status_data.reason
except McctStatus.DoesNotExist:
reason = None
row = [
iso_string_to_datetime(form["form"]["meta"]["timeEnd"]).strftime(SERVER_DATETIME_FORMAT_NO_SEC),
self._get_case_name_html(data.get('case'), with_checkbox),
self._get_service_type_html(form, data.get('service_type'), with_checkbox),
data.get('location_name'),
get_property(data.get('case'), "card_number", EMPTY_FIELD),
data.get('location_parent_name'),
get_property(data.get('case'), "phone_number", EMPTY_FIELD),
data.get('amount_due'),
REJECTION_REASON_DISPLAY_NAMES[reason] if reason is not None else '',
form["form"]["meta"]["username"]
]
if with_checkbox:
checkbox = mark_safe('<input type="checkbox" class="selected-element" '
'data-formid="%(form_id)s" '
'data-caseid="%(case_id)s" data-servicetype="%(service_type)s"/>')
row.append(checkbox % dict(form_id=data.get('form_id'), case_id=data.get('case_id'),
service_type=data.get('service_type')))
else:
row.insert(8, self.display_status)
yield row
def get_last_form_submission_received(domain):
result = _received_on_query(domain, desc=True)[0]
if not result:
return
return iso_string_to_datetime(result[0]['received_on'])
def submit_xform_rf(self, action, submission_xml_string):
# received_on = request.META.get('HTTP_X_SUBMIT_TIME')
# date_header = request.META.get('HTTP_DATE')
# if received_on:
# doc.received_on = string_to_datetime(received_on)
# if date_header:
# comes in as:
# Mon, 11 Apr 2011 18:24:43 GMT
# goes out as:
# 2011-04-11T18:24:43Z
# try:
# date = datetime.strptime(date_header, "%a, %d %b %Y %H:%M:%S GMT")
# date = datetime.strftime(date, "%Y-%m-%dT%H:%M:%SZ")
server_date = action.get('server_date', None)
phone_date = action.get('date', None)
extra_meta = {}
if phone_date:
date = iso_string_to_datetime(phone_date)
date = datetime.strftime(date, "%a, %d %b %Y %H:%M:%S GMT")
extra_meta['HTTP_DATE'] = date
if server_date:
extra_meta['HTTP_X_SUBMIT_TIME'] = server_date
else:
if phone_date is not None:
extra_meta['HTTP_X_SUBMIT_TIME'] = phone_date
submit_xform('/a/pact/receiver',
PACT_DOMAIN,
submission_xml_string,
extra_meta=extra_meta)
def get_schedule(cls, chw_username, override_date=None):
"""
Generate schedule object for a given username
"""
cached_schedules = None
if override_date == None:
nowdate = datetime.utcnow()
else:
nowdate = override_date
day_intervaltree = {}
if cached_schedules == None:
#no documents, then we need to load them up
db = CommCareCase.get_db()
chw_schedules = db.view('pact/chw_dot_schedules', key=chw_username).all()
to_cache = []
for item in chw_schedules:
single_sched = item['value']
to_cache.append(single_sched)
cache.set("%s_schedule" % (chw_username), json.dumps(to_cache), 3600)
cached_arr = to_cache
else:
cached_arr = json.loads(cached_schedules)
for single_sched in cached_arr:
day_of_week = int(single_sched['day_of_week'])
if day_of_week in day_intervaltree:
daytree = day_intervaltree[day_of_week]
else:
#if there's no day of week indication for this, then it's just a null interval node. To start this node, we make it REALLY old.
daytree = IntervalNode(get_seconds(datetime.min),
get_seconds(nowdate + timedelta(days=10)))
if single_sched['ended_date'] == None:
enddate = nowdate + timedelta(days=9)
else:
enddate = iso_string_to_datetime(single_sched['ended_date'])
startdate = iso_string_to_datetime(single_sched['active_date'])
case_id = single_sched['case_id']
if 'error' in single_sched:
#this is a non-showstopping issue due to quirks with older submissions
logging.error("Error, no pactid: %s" % single_sched['error'])
daytree.insert(get_seconds(startdate), get_seconds(enddate), other=case_id)
day_intervaltree[day_of_week] = daytree
return cls(chw_username, day_intervaltree, cached_arr)
def _skip_updating_domain_stats(last_updated=None, last_form_submission=None):
"""
Skip domain if no forms submitted in the last day
AND stats were updated less than a week ago.
:return: True to skip domain
"""
if not last_updated:
return False
last_updated_ago = datetime.utcnow() - iso_string_to_datetime(last_updated)
if last_form_submission:
last_form_ago = datetime.utcnow() - iso_string_to_datetime(last_form_submission)
new_data = last_form_ago < timedelta(days=1)
else:
new_data = False
return last_updated_ago < timedelta(days=7) and not new_data
def _get_es_modified_dates_for_forms(form_ids):
results = (
FormES(es_instance_alias=ES_EXPORT_INSTANCE).remove_default_filters()
.form_ids(form_ids)
.values_list('_id', 'received_on', 'doc_type', 'domain')
)
return {_id: (iso_string_to_datetime(received_on), doc_type, domain)
for _id, received_on, doc_type, domain in results}
def _get_es_modified_dates(case_ids):
results = (CaseES(for_export=True).case_ids(case_ids).values_list(
'_id', 'server_modified_on', 'domain'))
return {
_id: (iso_string_to_datetime(server_modified_on)
if server_modified_on else None, domain)
for _id, server_modified_on, domain in results
}
def _skip_updating_domain_stats(last_updated=None, last_form_submission=None):
"""
Skip domain if no forms submitted in the last day
AND stats were updated less than a week ago.
:return: True to skip domain
"""
if not last_updated:
return False
last_updated_ago = datetime.utcnow() - iso_string_to_datetime(last_updated)
if last_form_submission:
last_form_ago = datetime.utcnow() - iso_string_to_datetime(last_form_submission)
new_data = last_form_ago < timedelta(days=1)
else:
new_data = False
return last_updated_ago < timedelta(days=7) and not new_data
def missing_ledger_patch(diff):
ref = diff.ref
data = diff.old_value["ledger"]
return Balance(
entity_id=ref.case_id,
date=iso_string_to_datetime(data["last_modified"]).date(),
section_id=ref.section_id,
entry=Entry(id=ref.entry_id, quantity=data["balance"]),
)
def app_has_been_submitted_to_in_last_30_days(domain, app_id):
now = datetime.datetime.utcnow()
_30_days = datetime.timedelta(days=30)
result = _received_on_query(domain, desc=True).app(app_id)[0]
if not result:
return
return iso_string_to_datetime(result[0]['received_on']) > (now - _30_days)
def transform_datetime(item):
if item:
if isinstance(item, string_types):
try:
return iso_string_to_datetime(item, strict=True)
except ValueError:
pass
elif isinstance(item, datetime):
return item
return None
def transform_datetime(item):
if item:
if isinstance(item, str):
return iso_string_to_datetime(item, strict=False)
elif isinstance(item, datetime):
return item
elif isinstance(item, date):
return datetime.combine(item, time(0, 0, 0))
return None
def app_has_been_submitted_to_in_last_30_days(domain, app_id):
now = datetime.datetime.utcnow()
_30_days = datetime.timedelta(days=30)
result = _received_on_query(domain, desc=True).app(app_id)[0]
if not result:
return
return iso_string_to_datetime(result[0]['received_on']) > (now - _30_days)
def _process_main_forms(domain):
last_received_on = datetime.min
# process main forms (including cases and ledgers)
for change in _get_main_form_iterator(domain).iter_all_changes():
form = change.get_document()
wrapped_form = XFormInstance.wrap(form)
form_received = iso_string_to_datetime(form['received_on'])
assert last_received_on <= form_received
last_received_on = form_received
print 'processing form {}: {}'.format(form['_id'], form_received)
_migrate_form_and_associated_models(domain, wrapped_form)
def rows(self):
def form_data_link(instance_id):
return "<a class='ajax_dialog' target='_new' href='%(url)s'>%(text)s</a>" % {
"url":
absolute_reverse('render_form_data',
args=[self.domain, instance_id]),
"text":
_("View Form")
}
submissions = [
res['_source']
for res in self.es_results.get('hits', {}).get('hits', [])
]
for form in submissions:
uid = form["form"]["meta"]["userID"]
username = form["form"]["meta"].get("username")
try:
if username not in ['demo_user', 'admin']:
full_name = get_cached_property(CouchUser,
uid,
'full_name',
expiry=7 * 24 * 60 * 60)
name = '"%s"' % full_name if full_name else ""
else:
name = ""
except (ResourceNotFound, IncompatibleDocument):
name = "<b>[unregistered]</b>"
time = iso_string_to_datetime(
safe_index(form, self.time_field.split('.')))
if self.by_submission_time:
user_time = ServerTime(time).user_time(self.timezone)
else:
user_time = PhoneTime(time,
self.timezone).user_time(self.timezone)
init_cells = [
form_data_link(form["_id"]),
(username or _('No data for username')) +
(" %s" % name if name else ""),
user_time.ui_string(USER_DATETIME_FORMAT_WITH_SEC),
xmlns_to_name(self.domain,
form.get("xmlns"),
app_id=form.get("app_id")),
]
def cell(field):
return form["form"].get(field)
init_cells.extend([cell(field) for field in self.other_fields])
yield init_cells
def get_last_modified_dates(domain, case_ids):
"""
Given a list of case IDs, return a dict where the ids are keys and the
values are the last server modified date of that case.
"""
keys = [[domain, case_id] for case_id in case_ids]
return dict([(row['id'], iso_string_to_datetime(row['value']))
for row in CommCareCase.get_db().view(
'cases_by_server_date/by_server_modified_on',
keys=keys,
include_docs=False,
reduce=False)])
def parse_date(self, date_string):
try:
return iso_string_to_datetime(date_string)
except:
try:
date_obj = dateutil.parser.parse(date_string)
if isinstance(date_obj, datetime.datetime):
return date_obj.replace(tzinfo=None)
else:
return date_obj
except:
return date_string
def parse_date(self, date_string):
try:
return iso_string_to_datetime(date_string)
except:
try:
date_obj = dateutil.parser.parse(date_string)
if isinstance(date_obj, datetime.datetime):
return date_obj.replace(tzinfo=None)
else:
return date_obj
except:
return date_string
def _get_couch_form_chunks(run_config):
db = XFormInstance.get_db()
view_name = 'by_domain_doc_type_date/view'
keys = [{
'startkey': [
couch_domain, doc_type,
json_format_datetime(run_config.start_date)
],
'endkey': [
couch_domain, doc_type,
json_format_datetime(run_config.end_date)
],
} for couch_domain in _get_matching_couch_domains(run_config)
for doc_type in ['XFormArchived', 'XFormInstance']]
if not keys:
return
def _get_length():
length = 0
for key in keys:
result = db.view(view_name, reduce=True, **key).one()
if result:
length += result['value']
return length
iteration_key = f'couch_forms-{run_config.iteration_key}'
iterable = resumable_view_iterator(XFormInstance.get_db(),
iteration_key,
view_name,
keys,
chunk_size=CHUNK_SIZE,
full_row=True)
progress = ProgressManager(iterable,
total=_get_length(),
reset=False,
chunk_size=CHUNK_SIZE,
logger=ProcessorProgressLogger(
'[Couch Forms] ', sys.stderr))
with progress:
for chunk in chunked(iterable, CHUNK_SIZE):
records = []
for row in chunk:
form_id = row['id']
domain, doc_type, received_on = row['key']
received_on = iso_string_to_datetime(received_on)
assert run_config.domain in (domain, ALL_DOMAINS)
records.append(
(form_id, doc_type, 'COUCH_XMLNS_NOT_SUPPORTED',
received_on, domain))
yield records
progress.add(len(chunk))
def datespan_export_filter(doc, datespan):
if isinstance(datespan, dict):
datespan = DateSpan(**datespan)
try:
received_on = iso_string_to_datetime(doc['received_on']).replace(tzinfo=pytz.utc)
except Exception:
if settings.DEBUG:
raise
return False
if datespan.startdate <= received_on < (datespan.enddate + timedelta(days=1)):
return True
return False
def parse_date(self, date_string):
if date_string != EMPTY_FIELD and date_string != '' and date_string is not None:
try:
# assuming it's a date string or datetime string,
# DefaultProperty will wrap it as the correct type
# todo: there has to be a better way
return str(self.report.date_to_json(iso_string_to_datetime(date_string)))
except AttributeError:
return _("Bad date format!")
except TypeError:
return _("Bad date format!")
else:
return EMPTY_FIELD
def parse_date(self, date_string):
if date_string != EMPTY_FIELD and date_string != '' and date_string is not None:
try:
# assuming it's a date string or datetime string,
# DefaultProperty will wrap it as the correct type
# todo: there has to be a better way
return str(self.report.date_to_json(iso_string_to_datetime(date_string)))
except AttributeError:
return _("Bad date format!")
except TypeError:
return _("Bad date format!")
else:
return EMPTY_FIELD
def datespan_export_filter(doc, datespan):
if isinstance(datespan, dict):
datespan = DateSpan(**datespan)
try:
received_on = iso_string_to_datetime(doc['received_on']).replace(tzinfo=pytz.utc)
except Exception:
if settings.DEBUG:
raise
return False
if datespan.startdate <= received_on < (datespan.enddate + timedelta(days=1)):
return True
return False
def get_first_form_submission_received(domain):
from corehq.apps.reports.util import make_form_couch_key
key = make_form_couch_key(domain)
row = (
XFormInstance.get_db()
.view("reports_forms/all_forms", reduce=False, startkey=key, endkey=key + [{}], limit=1, stale=stale_ok())
.first()
)
if row:
submission_time = iso_string_to_datetime(row["key"][2])
else:
submission_time = None
return submission_time
def paginate_mobile_workers(request, domain):
limit = int(request.GET.get('limit', 10))
page = int(request.GET.get('page', 1))
query = request.GET.get('query')
deactivated_only = json.loads(
request.GET.get('showDeactivatedUsers', "false"))
def _user_query(search_string, page, limit):
user_es = get_search_users_in_domain_es_query(
domain=domain,
search_string=search_string,
offset=page * limit,
limit=limit)
if not request.couch_user.has_permission(domain,
'access_all_locations'):
loc_ids = (SQLLocation.objects.accessible_to_user(
domain, request.couch_user).location_ids())
user_es = user_es.location(list(loc_ids))
return user_es.mobile_users()
# backend pages start at 0
users_query = _user_query(query, page - 1, limit)
# run with a blank query to fetch total records with same scope as in search
if deactivated_only:
users_query = users_query.show_only_inactive()
users_data = users_query.source([
'_id',
'first_name',
'last_name',
'base_username',
'created_on',
'is_active',
]).run()
users = users_data.hits
for user in users:
date_registered = user.pop('created_on', '')
if date_registered:
date_registered = iso_string_to_datetime(date_registered).strftime(
USER_DATE_FORMAT)
user.update({
'username': user.pop('base_username', ''),
'user_id': user.pop('_id'),
'date_registered': date_registered,
})
return json_response({
'users': users,
'total': users_data.total,
})
def app_exchange(request, domain):
template = "app_manager/app_exchange.html"
records = []
for obj in ExchangeApplication.objects.all():
results = get_all_built_app_results(obj.domain, app_id=obj.app_id)
results = [r['value'] for r in results if r['value']['is_released']]
if not results:
continue
results.reverse()
first = results[0]
def _version_text(result):
if result['_id'] == first['_id']:
return _("Latest Version")
built_on = iso_string_to_datetime(
result['built_on']).strftime("%B %d, %Y")
return _("{} version").format(built_on)
records.append({
"id":
first['_id'],
"name":
first['name'],
"help_link":
obj.help_link,
"changelog_link":
obj.changelog_link,
"last_released":
iso_string_to_datetime(first['built_on']).date(),
"versions": [{
"id": r['_id'],
"text": _version_text(r),
} for r in results],
})
context = {
"domain": domain,
"records": records,
}
if request.method == "POST":
clear_app_cache(request, domain)
from_app_id = request.POST.get('from_app_id')
app_copy = import_app_util(from_app_id, domain, {
'created_from_template': from_app_id,
})
return back_to_main(request, domain, app_id=app_copy._id)
return render(request, template, context)
def last_submit_time(user_id):
#need to call it directly due to reversed not liking the keys set the regular way
key = make_form_couch_key(self.domain, user_id=user_id)
v = XFormInstance.get_db().view('all_forms/view',
endkey=key,
startkey=key + [{}],
reduce=False,
include_docs=False,
descending=True, limit=1
)
res = v.one()
if res is None:
return None
else:
return iso_string_to_datetime(res['key'][3]).strftime("%m/%d/%Y")
def last_submit_time(user_id):
#need to call it directly due to reversed not liking the keys set the regular way
key = make_form_couch_key(self.domain, user_id=user_id)
v = XFormInstance.get_db().view('reports_forms/all_forms',
endkey=key,
startkey=key + [{}],
reduce=False,
include_docs=False,
descending=True, limit=1
)
res = v.one()
if res is None:
return None
else:
return iso_string_to_datetime(res['key'][3]).strftime("%m/%d/%Y")
def get_last_modified_dates(domain, case_ids):
"""
Given a list of case IDs, return a dict where the ids are keys and the
values are the last server modified date of that case.
"""
keys = [[domain, case_id] for case_id in case_ids]
return dict([
(row['id'], iso_string_to_datetime(row['value']))
for row in CommCareCase.get_db().view(
'cases_by_server_date/by_server_modified_on',
keys=keys,
include_docs=False,
reduce=False
)
])
def transform_date(item):
# postgres crashes on empty strings, but is happy to take null dates
if item:
if isinstance(item, string_types):
try:
return iso_string_to_date(item)
except ValueError:
try:
return iso_string_to_datetime(item, strict=True).date()
except ValueError:
return None
elif isinstance(item, datetime):
return item.date()
elif isinstance(item, date):
return item
return None
def transform_datetime(item):
if item:
if isinstance(item, string_types):
try:
return iso_string_to_datetime(item, strict=True)
except ValueError:
try:
parsed_item = iso_string_to_date(item)
return datetime.combine(parsed_item, time(0, 0, 0))
except ValueError:
pass
elif isinstance(item, datetime):
return item
elif isinstance(item, date):
return datetime.combine(item, time(0, 0, 0))
return None
def paginate_mobile_workers(request, domain):
limit = int(request.GET.get('limit', 10))
page = int(request.GET.get('page', 1))
query = request.GET.get('query')
deactivated_only = json.loads(request.GET.get('showDeactivatedUsers', "false"))
def _user_query(search_string, page, limit):
user_es = get_search_users_in_domain_es_query(
domain=domain, search_string=search_string,
offset=page * limit, limit=limit)
if not request.couch_user.has_permission(domain, 'access_all_locations'):
loc_ids = (SQLLocation.objects.accessible_to_user(domain, request.couch_user)
.location_ids())
user_es = user_es.location(list(loc_ids))
return user_es.mobile_users()
# backend pages start at 0
users_query = _user_query(query, page - 1, limit)
# run with a blank query to fetch total records with same scope as in search
if deactivated_only:
users_query = users_query.show_only_inactive()
users_data = users_query.source([
'_id',
'first_name',
'last_name',
'base_username',
'created_on',
'is_active',
]).run()
users = users_data.hits
for user in users:
date_registered = user.pop('created_on', '')
if date_registered:
date_registered = iso_string_to_datetime(date_registered).strftime(USER_DATE_FORMAT)
user.update({
'username': user.pop('base_username', ''),
'user_id': user.pop('_id'),
'date_registered': date_registered,
})
return json_response({
'users': users,
'total': users_data.total,
})
def recent_submissions(self):
from corehq.apps.reports.util import make_form_couch_key
key = make_form_couch_key(self.name)
res = get_db().view(
'reports_forms/all_forms',
startkey=key + [{}],
endkey=key,
descending=True,
reduce=False,
include_docs=False,
limit=1
).all()
# if there have been any submissions in the past 30 days
if len(res) > 0:
received_on = iso_string_to_datetime(res[0]['key'][2])
return datetime.utcnow() <= received_on + timedelta(days=30)
else:
return False
def date_format(date_str):
"""
>>> date_format('2015-02-10 11:54:24')
'02/10/2015'
>>> date_format('2015-02-10 11:54:24.004000')
'02/10/2015'
"""
if date_str:
# this comes in with a ' ' instead of 'T' for some reason
# would be nice to go back and figure out where that happens
# probably `date_str = unicode(dt)` happens somewhere
if ' ' in date_str and not date_str.endswith('Z'):
date_str = date_str.replace(' ', 'T') + 'Z'
date = iso_string_to_datetime(date_str)
return date.strftime(OUTPUT_DATE_FORMAT)
else:
return EMPTY_FIELD
def _clean(meta_block):
ret = copy(dict(meta_block))
for key in ret.keys():
# remove attributes from the meta block
if key.startswith('@'):
del ret[key]
# couchdbkit erroneously converts appVersion to a Decimal just because it is possible (due to it being within a "dynamic" property)
# (see https://github.com/benoitc/couchdbkit/blob/a23343e539370cffcf8b0ce483c712911bb022c1/couchdbkit/schema/properties.py#L1038)
ret['appVersion'] = get_text(meta_block.get('appVersion'))
ret['location'] = get_text(meta_block.get('location'))
# couchdbkit chokes on dates that aren't actually dates
# so check their validity before passing them up
if meta_block:
for key in ("timeStart", "timeEnd"):
if key in meta_block:
if meta_block[key]:
if re_date.match(meta_block[key]):
# this kind of leniency is pretty bad
# and making it midnight in UTC
# is totally arbitrary
# here for backwards compatibility
meta_block[key] += 'T00:00:00.000000Z'
try:
# try to parse to ensure correctness
parsed = iso_string_to_datetime(meta_block[key])
# and set back in the right format in case it was a date, not a datetime
ret[key] = json_format_datetime(parsed)
except BadValueError:
# we couldn't parse it
del ret[key]
else:
# it was empty, also a failure
del ret[key]
# also clean dicts on the return value, since those are not allowed
for key in ret:
if isinstance(ret[key], dict):
ret[key] = ", ".join(\
"%s:%s" % (k, v) \
for k, v in ret[key].items())
return ret
def parse():
if not val:
return None
# datetime is a subclass of date
if isinstance(val, datetime.date):
return val
try:
dt = iso_string_to_datetime(val)
except BadValueError:
try:
return DateProperty().wrap(val)
except BadValueError:
return val
else:
if not any([dt.hour, dt.minute, dt.second, dt.microsecond]):
return dt.date()
else:
return dt
def rows(self):
def form_data_link(instance_id):
return "<a class='ajax_dialog' target='_new' href='%(url)s'>%(text)s</a>" % {
"url": absolute_reverse('render_form_data', args=[self.domain, instance_id]),
"text": _("View Form")
}
submissions = [res['_source'] for res in self.es_results.get('hits', {}).get('hits', [])]
for form in submissions:
uid = form["form"]["meta"]["userID"]
username = form["form"]["meta"].get("username")
try:
if username not in ['demo_user', 'admin']:
full_name = get_cached_property(CouchUser, uid, 'full_name', expiry=7*24*60*60)
name = '"%s"' % full_name if full_name else ""
else:
name = ""
except (ResourceNotFound, IncompatibleDocument):
name = "<b>[unregistered]</b>"
time = iso_string_to_datetime(safe_index(form, self.time_field.split('.')))
if self.by_submission_time:
user_time = ServerTime(time).user_time(self.timezone)
else:
user_time = PhoneTime(time, self.timezone).user_time(self.timezone)
init_cells = [
form_data_link(form["_id"]),
(username or _('No data for username')) + (" %s" % name if name else ""),
user_time.ui_string(USER_DATETIME_FORMAT_WITH_SEC),
xmlns_to_name(self.domain, form.get("xmlns"), app_id=form.get("app_id")),
]
def cell(field):
return form["form"].get(field)
init_cells.extend([cell(field) for field in self.other_fields])
yield init_cells
def _parse_meta_times(meta_block):
for key in ("timeStart", "timeEnd"):
if key not in meta_block:
continue
if meta_block[key]:
if re_date.match(meta_block[key]):
# this kind of leniency is pretty bad and making it midnight in UTC
# is totally arbitrary here for backwards compatibility
meta_block[key] += 'T00:00:00.000000Z'
try:
# try to parse to ensure correctness
parsed = iso_string_to_datetime(meta_block[key])
# and set back in the right format in case it was a date, not a datetime
meta_block[key] = json_format_datetime(parsed)
except Exception:
logging.exception('Could not parse meta_block')
# we couldn't parse it
del meta_block[key]
else:
# it was empty, also a failure
del meta_block[key]
return meta_block
def make_rows(self, es_results, with_checkbox):
submissions = [res['_source'] for res in self.es_results.get('hits', {}).get('hits', [])]
for form in submissions:
data = calculate_form_data(self, form)
row = [
json_format_date(iso_string_to_datetime(form["form"]["meta"]["timeEnd"])),
self._get_case_name_html(data.get('case'), with_checkbox),
self._get_service_type_html(form, data.get('service_type'), with_checkbox),
data.get('location_name'),
get_property(data.get('case'), "card_number", EMPTY_FIELD),
data.get('location_parent_name'),
get_property(data.get('case'), "phone_number", EMPTY_FIELD),
data.get('amount_due'),
get_property(data.get('case'), "visits", EMPTY_FIELD)
]
if with_checkbox:
checkbox = mark_safe('<input type="checkbox" class="selected-element" '
'data-formid="%(form_id)s" '
'data-caseid="%(case_id)s" data-servicetype="%(service_type)s"/>')
row.append(checkbox % dict(form_id=data.get('form_id'), case_id=data.get('case_id'),
service_type=data.get('service_type')))
else:
row.append(self.display_status)
yield row
def display_time(row, display=True):
submission_time = row["key"][2]
if display:
return iso_string_to_datetime(submission_time).strftime(DISPLAY_DATE_FORMAT)
else:
return submission_time
def get_300th_form_submission_received(domain):
result = FormES().domain(domain).start(300).size(1).sort('received_on').fields(['received_on']).run().hits
if not result:
return
return iso_string_to_datetime(result[0]['received_on'])
def format_date(dstr, default):
if dstr:
return iso_string_to_datetime(dstr).strftime('%Y/%m/%d %H:%M:%S')
else:
return default
