def _sms_count(user, startdate, enddate, message_type="SMSLog"): """ Returns a dictionary of messages seen for a given type, user, and date range of the format: { I: inbound_count, O: outbound_count } """ # utilizable if we want to stick it somewhere else start_timestamp = json_format_datetime(startdate) end_timestamp = json_format_datetime(enddate) ret = {} for direction in [INCOMING, OUTGOING]: results = ( SMSLog.get_db() .view( "sms/by_recipient", startkey=[user.doc_type, user._id, message_type, direction, start_timestamp], endkey=[user.doc_type, user._id, message_type, direction, end_timestamp], reduce=True, ) .all() ) ret[direction] = results[0]["value"] if results else 0 return ret
def get_first_survey_response(self, case, dt): timestamp_start = datetime.combine(dt, time(20, 45)) timestamp_start = tz_utils.adjust_datetime_to_timezone( timestamp_start, self.domain_obj.default_timezone, pytz.utc.zone) timestamp_start = timestamp_start.replace(tzinfo=None) timestamp_start = json_format_datetime(timestamp_start) timestamp_end = datetime.combine(dt + timedelta(days=1), time(11, 0)) timestamp_end = tz_utils.adjust_datetime_to_timezone( timestamp_end, self.domain_obj.default_timezone, pytz.utc.zone) timestamp_end = timestamp_end.replace(tzinfo=None) if timestamp_end > datetime.utcnow(): return RESPONSE_NOT_APPLICABLE timestamp_end = json_format_datetime(timestamp_end) all_inbound = FRISMSLog.view( "sms/by_recipient", startkey=["CommCareCase", case._id, "SMSLog", INCOMING, timestamp_start], endkey=["CommCareCase", case._id, "SMSLog", INCOMING, timestamp_end], reduce=False, include_docs=True ).all() survey_responses = filter(lambda s: s.xforms_session_couch_id is not None, all_inbound) if len(survey_responses) > 0: return survey_responses[0] else: return NO_RESPONSE
def get_number_cases(self, user_id, modified_after=None, modified_before=None, closed=None): key = [self.domain, {} if closed is None else closed, self.case_type or {}, user_id] if modified_after is None: start = "" else: start = json_format_datetime(modified_after) if modified_before is None: end = {} else: end = json_format_datetime(modified_before) return ( get_db() .view( "case/by_date_modified", startkey=key + [start], endkey=key + [end], group=True, group_level=0, wrapper=lambda row: row["value"], ) .one() or 0 )
def test_ota_consumption(self): self.ct_settings.consumption_config = ConsumptionConfig(min_transactions=0, min_window=0, optimal_window=60) self.ct_settings.ota_restore_config = StockRestoreConfig(section_to_consumption_types={"stock": "consumption"}) set_default_monthly_consumption_for_domain(self.domain.name, 5 * DAYS_IN_MONTH) self._save_settings_and_clear_cache() amounts = [(p._id, i * 10) for i, p in enumerate(self.products)] report = _report_soh(amounts, self.sp.case_id, "stock") balance_blocks = _get_ota_balance_blocks(self.domain, self.user) self.assertEqual(2, len(balance_blocks)) stock_block, consumption_block = balance_blocks check_xml_line_by_line( self, balance_ota_block(self.sp, "stock", amounts, datestring=json_format_datetime(report.date)), stock_block, ) check_xml_line_by_line( self, balance_ota_block( self.sp, "consumption", [(p._id, 150) for p in self.products], datestring=json_format_datetime(report.date), ), consumption_block, )
def _cacheable_domain_activity_report(request): landmarks = json.loads(request.GET.get('landmarks') or "[7, 30, 90]") landmarks.sort() now = datetime.utcnow() dates = [] for landmark in landmarks: dates.append(now - timedelta(days=landmark)) domains = [{'name': domain.name, 'display_name': domain.display_name()} for domain in Domain.get_all()] for domain in domains: domain['users'] = dict([(user.user_id, {'raw_username': user.raw_username}) for user in CommCareUser.by_domain(domain['name'])]) if not domain['users']: continue key = make_form_couch_key(domain['name']) forms = [r['value'] for r in get_db().view('reports_forms/all_forms', reduce=False, startkey=key+[json_format_datetime(dates[-1])], endkey=key+[json_format_datetime(now)], ).all()] domain['user_sets'] = [dict() for landmark in landmarks] for form in forms: user_id = form.get('user_id') try: time = string_to_datetime(form['submission_time']).replace(tzinfo = None) except ValueError: continue if user_id in domain['users']: for i, date in enumerate(dates): if time > date: domain['user_sets'][i][user_id] = domain['users'][user_id] return HttpResponse(json.dumps({'domains': domains, 'landmarks': landmarks}))
def get_pre_migration_copy(app): from corehq.apps.app_manager.util import get_correct_app_class def date_key(doc): return doc.get("built_on") or mindate mindate = json_format_datetime(datetime(1980, 1, 1)) migrate_date = json_format_datetime(ORIGINAL_MIGRATION_DATE) skip = 0 docs = None while docs is None or date_key(docs[-1]) > migrate_date: docs = saved_apps = [row['doc'] for row in Application.get_db().view( 'app_manager/saved_app', startkey=[app.domain, app._id, {}], endkey=[app.domain, app._id], descending=True, skip=skip, limit=5, include_docs=True, )] if not docs: break skip += len(docs) docs = sorted(saved_apps, key=date_key, reverse=True) for doc in docs: if date_key(doc) < migrate_date: copy = get_correct_app_class(doc).wrap(doc) if copy.version < app.version: return copy return None
def process_view(self, request, view_func, view_args, view_kwargs): if not request.user.is_authenticated(): return secure_session = request.session.get('secure_session') domain = getattr(request, "domain", None) now = datetime.datetime.utcnow() if not secure_session and ( (domain and Domain.is_secure_session_required(domain)) or self._user_requires_secure_session(request.couch_user)): if self._session_expired(settings.SECURE_TIMEOUT, request.user.last_login, now): django_logout(request, template_name=settings.BASE_TEMPLATE) # this must be after logout so it is attached to the new session request.session['secure_session'] = True return HttpResponseRedirect(reverse('login') + '?next=' + request.path) else: request.session['secure_session'] = True request.session['last_request'] = json_format_datetime(now) return else: last_request = request.session.get('last_request') timeout = settings.SECURE_TIMEOUT if secure_session else settings.INACTIVITY_TIMEOUT if self._session_expired(timeout, last_request, now): django_logout(request, template_name=settings.BASE_TEMPLATE) return HttpResponseRedirect(reverse('login') + '?next=' + request.path) request.session['last_request'] = json_format_datetime(now)
def adjust_datetimes(data, parent=None, key=None): """ find all datetime-like strings within data (deserialized json) and format them uniformly, in place. """ # this strips the timezone like we've always done # todo: in the future this will convert to UTC if isinstance(data, basestring): if re_loose_datetime.match(data): if phone_timezones_should_be_processed(): parent[key] = json_format_datetime( iso8601.parse_date(data).astimezone(pytz.utc) .replace(tzinfo=None) ) else: parent[key] = json_format_datetime( iso8601.parse_date(data).replace(tzinfo=None)) elif isinstance(data, dict): for key, value in data.items(): adjust_datetimes(value, parent=data, key=key) elif isinstance(data, list): for i, value in enumerate(data): adjust_datetimes(value, parent=data, key=i) # return data, just for convenience in testing # this is the original input, modified, not a new data structure return data
def is_app_active(app_id, domain): now = datetime.utcnow() then = json_format_datetime(now - timedelta(days=30)) now = json_format_datetime(now) key = ['submission app', domain, app_id] row = get_db().view("reports_forms/all_forms", startkey=key+[then], endkey=key+[now]).all() return True if row else False
def get_list(cls, environment, startdate, enddate, limit=50): return HqDeploy.view( 'hqadmin/deploy_history', startkey=[environment, json_format_datetime(startdate)], endkey=[environment, json_format_datetime(enddate)], reduce=False, limit=limit, include_docs=False ).all()
def rows(self): startdate = json_format_datetime(self.datespan.startdate_utc) enddate = json_format_datetime(self.datespan.enddate_utc) data = FRISMSLog.view("sms/by_domain", startkey=[self.domain, "SMSLog", startdate], endkey=[self.domain, "SMSLog", enddate], include_docs=True, reduce=False).all() result = [] direction_map = { INCOMING: _("Incoming"), OUTGOING: _("Outgoing"), } message_bank_messages = get_message_bank(self.domain, for_comparing=True) FormProcessorInterface(self.domain).casedb_cache( domain=self.domain, strip_history=False, deleted_ok=True ) user_cache = UserCache() show_only_survey_traffic = self.show_only_survey_traffic() for message in data: if message.direction == OUTGOING and not message.processed: continue if show_only_survey_traffic and message.xforms_session_couch_id is None: continue # Add metadata from the message bank if it has not been added already if (message.direction == OUTGOING) and (not message.fri_message_bank_lookup_completed): add_metadata(message, message_bank_messages) if message.couch_recipient_doc_type == "CommCareCase": recipient = case_cache.get(message.couch_recipient) else: recipient = user_cache.get(message.couch_recipient) if message.chat_user_id: sender = user_cache.get(message.chat_user_id) else: sender = None study_arm = None if message.couch_recipient_doc_type == "CommCareCase": study_arm = case_cache.get(message.couch_recipient).get_case_property("study_arm") timestamp = ServerTime(message.date).user_time(self.timezone).done() result.append([ self._fmt(self._participant_id(recipient)), self._fmt(study_arm or "-"), self._fmt(self._originator(message, recipient, sender)), self._fmt_timestamp(timestamp), self._fmt(message.text), self._fmt(message.fri_id or "-"), self._fmt(direction_map.get(message.direction,"-")), ]) return result
def _formatXForm(self, doc_id, raw_xml, attachment_block): final_xml = raw_xml % ({ "attachments": attachment_block, "time_start": json_format_datetime(datetime.utcnow() - timedelta(minutes=4)), "time_end": json_format_datetime(datetime.utcnow()), "date_modified": json_format_datetime(datetime.utcnow()), "doc_id": doc_id }) return final_xml
def _formatXForm(self, doc_id, raw_xml, attachment_block, date=None): if date is None: date = datetime.utcnow() final_xml = Template(raw_xml).render(Context({ "attachments": attachment_block, "time_start": json_format_datetime(date - timedelta(minutes=4)), "time_end": json_format_datetime(date), "date_modified": json_format_datetime(date), "doc_id": doc_id })) return final_xml
def update_task(domain, subcase_guid, submitting_user_id, task_owner_id, form_unique_id, task_activation_datetime, task_deactivation_datetime, incentive): context = { "subcase_guid" : subcase_guid, "user_id" : submitting_user_id, "date_modified" : json_format_datetime(datetime.datetime.utcnow()), "task_owner_id" : task_owner_id, "form_unique_id" : form_unique_id, "task_activation_date" : json_format_datetime(task_activation_datetime), "task_deactivation_date" : json_format_datetime(task_deactivation_datetime), "incentive" : incentive, } submit_xml(domain, "sms/xml/update_task.xml", context)
def rows(self): startdate = json_format_datetime(self.datespan.startdate_utc) enddate = json_format_datetime(self.datespan.enddate_utc) data = SMSLog.by_domain_date(self.domain, startdate, enddate) result = [] username_map = {} # Store the results of username lookups for faster loading direction_map = { INCOMING: _("Incoming"), OUTGOING: _("Outgoing"), } # Retrieve message log options message_log_options = getattr(settings, "MESSAGE_LOG_OPTIONS", {}) abbreviated_phone_number_domains = message_log_options.get("abbreviated_phone_number_domains", []) abbreviate_phone_number = (self.domain in abbreviated_phone_number_domains) for message in data: if message.direction == OUTGOING and not message.processed: continue recipient_id = message.couch_recipient if recipient_id in [None, ""]: username = "******" elif recipient_id in username_map: username = username_map.get(recipient_id) else: username = "******" try: if message.couch_recipient_doc_type == "CommCareCase": username = CommCareCase.get(recipient_id).name else: username = CouchUser.get_by_user_id(recipient_id).username except Exception: pass username_map[recipient_id] = username phone_number = message.phone_number if abbreviate_phone_number and phone_number is not None: phone_number = phone_number[0:7] if phone_number[0:1] == "+" else phone_number[0:6] timestamp = tz_utils.adjust_datetime_to_timezone(message.date, pytz.utc.zone, self.timezone.zone) result.append([ self._fmt_timestamp(timestamp), self._fmt(username), self._fmt(phone_number), self._fmt(direction_map.get(message.direction,"-")), self._fmt(message.text), ]) return result
def _household_verification_json( domain="dodoma", last_hvid_path=["household_verification"], next_hvid_path=["followup_id"], xmlns='http://openrosa.org/formdesigner/9DAACA82-A414-499A-9C40-BC43775CEE79', range=None ): if range: start, end = map(string_to_datetime, range) else: now = datetime.utcnow() start, end = now - timedelta(days=7), now key = make_form_couch_key(domain, xmlns=xmlns) submissions = XFormInstance.view('reports_forms/all_forms', reduce=False, startkey=key+[json_format_datetime(start)], endkey=key+[json_format_datetime(end)], include_docs=True, ) stats = get_household_verification_data( submissions=submissions, next_hvid_path=next_hvid_path, last_hvid_path=last_hvid_path, ) stats_by_userID = {} for s in stats: stats_by_userID[s['userID']] = s s['username'] = "******" % s['userID'] users = CommCareUser.by_domain(domain) for user in users: userID = user.user_id username = user_id_to_username(userID) if userID in stats_by_userID: stats_by_userID[userID]['username'] = username else: stats.append({'userID': userID, 'username': username, 'total': 0, 'correct': 0}) stats.sort(key=lambda s: s['username']) return { "headers": ["Username", "Correct", "Total", "Percent Correct"], "rows": [[ s['username'], s['correct'], s['total'], ("%s%%" % int(s['correct']*100/s['total']) if s['total'] else "---") ] for s in stats], }
def rows(self): startdate = json_format_datetime(self.datespan.startdate_utc) enddate = json_format_datetime(self.datespan.enddate_utc) data = SMSLog.by_domain_date(self.domain, startdate, enddate) result = [] direction_map = { INCOMING: _("Incoming"), OUTGOING: _("Outgoing"), } # Retrieve message log options message_log_options = getattr(settings, "MESSAGE_LOG_OPTIONS", {}) abbreviated_phone_number_domains = message_log_options.get("abbreviated_phone_number_domains", []) abbreviate_phone_number = (self.domain in abbreviated_phone_number_domains) contact_cache = {} for message in data: if message.direction == OUTGOING and not message.processed: continue recipient_id = message.couch_recipient doc = None if recipient_id not in [None, ""]: try: if message.couch_recipient_doc_type == "CommCareCase": doc = CommCareCase.get(recipient_id) else: doc = CouchUser.get_by_user_id(recipient_id) except Exception: pass if doc: doc_info = get_doc_info(doc.to_json(), self.domain, contact_cache) else: doc_info = None phone_number = message.phone_number if abbreviate_phone_number and phone_number is not None: phone_number = phone_number[0:7] if phone_number[0:1] == "+" else phone_number[0:6] timestamp = tz_utils.adjust_datetime_to_timezone(message.date, pytz.utc.zone, self.timezone.zone) result.append([ self._fmt_timestamp(timestamp), self._fmt_contact_link(message, doc_info), self._fmt(phone_number), self._fmt(direction_map.get(message.direction,"-")), self._fmt(message.text), ]) return result
def active(domain, *args): now = datetime.utcnow() then = json_format_datetime(now - timedelta(days=30)) now = json_format_datetime(now) key = ['submission', domain] row = get_db().view( "reports_forms/all_forms", startkey=key+[then], endkey=key+[now], limit=1 ).all() return True if row else False
def app_has_been_submitted_to_in_last_30_days(domain, app_id): now = datetime.datetime.utcnow() _30_days = datetime.timedelta(days=30) then = json_format_datetime(now - _30_days) now = json_format_datetime(now) key = ["submission app", domain, app_id] row = ( XFormInstance.get_db() .view("reports_forms/all_forms", startkey=key + [then], endkey=key + [now], limit=1, stale=stale_ok()) .all() ) return True if row else False
def mk_date_range(start=None, end=None, ago=timedelta(days=7), iso=False): if isinstance(end, basestring): end = parse_date(end) if isinstance(start, basestring): start = parse_date(start) if not end: end = datetime.utcnow() if not start: start = end - ago if iso: return json_format_datetime(start), json_format_datetime(end) else: return start, end
def get_reports(cls, domain, location=None, datespan=None): start = datespan.startdate if datespan else datetime(1900, 1, 1) end = datespan.end_of_end_day if datespan else datetime.max timestamp_start = dateparse.json_format_datetime(start) timestamp_end = dateparse.json_format_datetime(end) loc_id = location._id if location else None startkey = [domain, loc_id, timestamp_start] endkey = [domain, loc_id, timestamp_end] return [StockReport(f) for f in \ XFormInstance.view('commtrack/stock_reports', startkey=startkey, endkey=endkey, include_docs=True)]
def test_xml(self): now = datetime.utcnow() domain = "test" now_plus_30 = now + timedelta(days=30) now_minus_30 = now - timedelta(days=30) record = new_key_record(None, None, now=now) xml = get_mobile_auth_payload([record], domain, now=now) check_xml_line_by_line( self, xml, """ <OpenRosaResponse xmlns="http://openrosa.org/http/response"> <message nature="submit_success">Here are your keys!</message> <auth_keys domain="{domain}" issued="{now}"> <key_record valid="{now}" expires="{now_plus_30}"> <uuid>{record.uuid}</uuid> <key type="{record.type}">{record.key}</key> </key_record> </auth_keys> </OpenRosaResponse> """.format( now=json_format_datetime(now), now_plus_30=json_format_datetime(now_plus_30), record=record, domain=domain, ), ) record = new_key_record(None, None, now=now, valid=now_minus_30) xml = get_mobile_auth_payload([record], domain, now=now) check_xml_line_by_line( self, xml, """ <OpenRosaResponse xmlns="http://openrosa.org/http/response"> <message nature="submit_success">Here are your keys!</message> <auth_keys domain="{domain}" issued="{now}"> <key_record valid="{now_minus_30}" expires="{now_plus_30}"> <uuid>{record.uuid}</uuid> <key type="{record.type}">{record.key}</key> </key_record> </auth_keys> </OpenRosaResponse> """.format( now=json_format_datetime(now), now_plus_30=json_format_datetime(now_plus_30), now_minus_30=json_format_datetime(now_minus_30), record=record, domain=domain, ), )
def get_number_of_submissions(domain, user_id, xmlns, app_id, start, end, by_submission_time=True): from corehq.apps.reports.util import make_form_couch_key key = make_form_couch_key(domain, user_id=user_id, xmlns=xmlns, by_submission_time=by_submission_time, app_id=app_id) data = XFormInstance.get_db().view( 'reports_forms/all_forms', reduce=True, startkey=key + [json_format_datetime(start)], endkey=key + [json_format_datetime(end)], stale=stale_ok(), ).first() return data['value'] if data else 0
def get_case_ids_modified_with_owner_since(domain, owner_id, reference_date, until_date=None): """ Gets all cases with a specified owner ID that have been modified since a particular reference_date (using the server's timestamp) """ return [ row['id'] for row in CommCareCase.get_db().view( 'cases_by_server_date/by_owner_server_modified_on', startkey=[domain, owner_id, json_format_datetime(reference_date)], endkey=[domain, owner_id, {} if not until_date else json_format_datetime(until_date)], include_docs=False, reduce=False ) ]
def _report_soh(soh_reports, case_id, domain): report_date = json_format_datetime(datetime.utcnow()) balance_blocks = [ get_single_balance_block( case_id, report.product_id, report.amount, report_date, section_id=report.section_id ) for report in soh_reports ] form = submit_case_blocks(balance_blocks, domain) return json_format_datetime(FormAccessors(domain).get_form(form.form_id).received_on)
def cases_in_last(domain, days): """ Returns the number of open cases that have been modified in the last <days> days """ now = datetime.utcnow() then = json_format_datetime(now - timedelta(days=int(days))) now = json_format_datetime(now) q = {"query": { "range": { "modified_on": { "from": then, "to": now}}}} data = es_query(params={"domain.exact": domain, 'closed': False}, q=q, es_url=CASE_INDEX + '/case/_search', size=1) return data['hits']['total'] if data.get('hits') else 0
def notify_sentry_deploy(duration_mins): from settingshelper import get_release_name headers = {'Authorization': 'Bearer {}'.format(settings.SENTRY_API_KEY), } payload = { 'environment': settings.SERVER_ENVIRONMENT, } if duration_mins: utcnow = datetime.utcnow() payload.update({ 'dateStarted': json_format_datetime(utcnow - timedelta(minutes=duration_mins)), 'dateFinished': json_format_datetime(utcnow), }) version = get_release_name(settings.BASE_DIR, settings.SERVER_ENVIRONMENT) releases_url = 'https://sentry.io/api/0/organizations/dimagi/releases/{}/deploys/'.format(version) requests.post(releases_url, headers=headers, json=payload)
def rows(self): startdate = json_format_datetime(self.datespan.startdate_utc) enddate = json_format_datetime(self.datespan.enddate_utc) data = FRISMSLog.view("sms/by_domain", startkey=[self.domain, "SMSLog", startdate], endkey=[self.domain, "SMSLog", enddate], include_docs=True, reduce=False).all() result = [] direction_map = { INCOMING: _("Incoming"), OUTGOING: _("Outgoing"), } message_bank_messages = get_message_bank(self.domain, for_comparing=True) case_cache = CaseDbCache(domain=self.domain, strip_history=False, deleted_ok=True) user_cache = UserCache() for message in data: # Add metadata from the message bank if it has not been added already if (message.direction == OUTGOING) and (not message.fri_message_bank_lookup_completed): add_metadata(message, message_bank_messages) if message.couch_recipient_doc_type == "CommCareCase": recipient = case_cache.get(message.couch_recipient) else: recipient = user_cache.get(message.couch_recipient) if message.chat_user_id: sender = user_cache.get(message.chat_user_id) else: sender = None study_arm = None if message.couch_recipient_doc_type == "CommCareCase": study_arm = case_cache.get(message.couch_recipient).get_case_property("study_arm") timestamp = tz_utils.adjust_datetime_to_timezone(message.date, pytz.utc.zone, self.timezone.zone) result.append([ self._fmt(self._participant_id(recipient)), self._fmt(study_arm or "-"), self._fmt(self._originator(message, recipient, sender)), self._fmt_timestamp(timestamp), self._fmt(message.text), self._fmt(message.fri_id or "-"), self._fmt(direction_map.get(message.direction,"-")), ]) return result
def get_sample_doc_and_indicators(fake_time_now=None, owner_id='some-user-id'): if fake_time_now is None: fake_time_now = datetime.utcnow() date_opened = datetime(2014, 6, 21) sample_doc = dict( _id=uuid.uuid4().hex, opened_on=json_format_datetime(date_opened), owner_id=owner_id, doc_type="CommCareCase", domain='user-reports', name='sample name', type='ticket', category='bug', tags='easy-win public', is_starred='yes', estimate=2.3, priority=4, ) expected_indicators = { 'doc_id': sample_doc['_id'], 'repeat_iteration': 0, 'date': date_opened, 'owner': owner_id, 'count': 1, 'category_bug': 1, 'category_feature': 0, 'category_app': 0, 'category_schedule': 0, 'tags_easy-win': 1, 'tags_potential-dupe': 0, 'tags_roadmap': 0, 'tags_public': 1, 'is_starred': 1, 'estimate': Decimal(2.3), 'priority': 4, 'inserted_at': fake_time_now, } return sample_doc, expected_indicators
def to_instance(data): """convert the parsed sms stock report into an instance like what would be submitted from a commcare phone""" E = XML() M = XML(const.META_XMLNS, 'jrm') deviceID = get_device_id(data) timestamp = json_format_datetime(data['timestamp']) transactions = data['transactions'] category = set(tx.category for tx in transactions).pop() stock_blocks = convert_transactions_to_blocks(E, transactions) if category == 'stock': root = E.stock_report( M.meta( M.userID(data['user']._id), M.deviceID(deviceID), M.timeStart(timestamp), M.timeEnd(timestamp) ), E.location(data['location']._id), *stock_blocks ) return etree.tostring(root, encoding='utf-8', pretty_print=True) else: return requisition_case_xml(data, stock_blocks)
def get_single_transfer_block(src_id, dest_id, product_id, quantity, date_string=None, section_id='stock'): date_string = date_string or json_format_datetime(datetime.utcnow()) return """ <transfer xmlns="http://commcarehq.org/ledger/v1" {src} {dest} date="{date}" section-id="{section_id}"> <entry id="{product_id}" quantity="{quantity}" /> </transfer >""".format( src='src="{}"'.format(src_id) if src_id is not None else '', dest='dest="{}"'.format(dest_id) if dest_id is not None else '', product_id=product_id, quantity=quantity, date=date_string, section_id=section_id, ).strip()
def transform_case_for_elasticsearch(doc_dict): system_properties = ['case_properties', '_indexed_on'] doc = { desired_property: doc_dict.get(desired_property) for desired_property in CASE_SEARCH_MAPPING['properties'].keys() if desired_property not in system_properties } doc['_id'] = doc_dict.get('_id') doc['_indexed_on'] = json_format_datetime(datetime.utcnow()) doc['case_properties'] = _get_case_properties(doc_dict) return doc
def get_doc_count_in_domain_by_type(domain, doc_type, db, start_date=None, end_date=None): start_key = [domain, doc_type] end_key = [domain, doc_type] if start_date is not None: start_key.append(json_format_datetime(start_date)) if end_date is not None: end_key.append(json_format_datetime(end_date)) end_key.append({}) row = db.view( "by_domain_doc_type_date/view", startkey=start_key, endkey=end_key, reduce=True, ).one() return row["value"] if row else 0
def get_num_forms_since(self, time): if not hasattr(self, 'domain'): self.domain = self.couch_user.domain if self.couch_user else None from corehq.apps.reports.util import make_form_couch_key key = make_form_couch_key(self.domain, user_id=self.userID) r = get_db().view('reports_forms/all_forms', startkey=key + [json_format_datetime(time)], endkey=key + [{}], group=False).one() return r['value'] if r else 0
def test_payload(self): form = self.post_xml(self.xform_xml, self.domain_name).xform repeat_records = self.repeat_records(self.domain_name).all() payload = repeat_records[0].get_payload() cases = cases_referenced_by_xform(form) self.assertEqual( json.loads(payload), { 'received_on': json_format_datetime(form.received_on), 'form_id': form.form_id, 'case_ids': [case.case_id for case in cases] })
def _soft_delete(db, doc_ids, deletion_date=None, deletion_id=None): from dimagi.utils.couch.undo import DELETED_SUFFIX deletion_date = json_format_datetime(deletion_date or datetime.utcnow()) def delete(doc): doc['doc_type'] += DELETED_SUFFIX doc['-deletion_id'] = deletion_id doc['-deletion_date'] = deletion_date return doc return _operate_on_docs(db, doc_ids, delete)
def fmt(value): if value is None: return '' if isinstance(value, datetime): return six.text_type(json_format_datetime(value)) elif isinstance(value, six.string_types + (numbers.Number, date)): return six.text_type(value) else: raise CaseBlockError( "Can't transform to XML: {}; unexpected type {}.".format( value, type(value)))
def key_for_time(cls, domain, user_id, now): now_json = json_format_datetime(now) key_record = cls.view( 'mobile_auth/key_records', startkey=[domain, user_id, now_json], endkey=[domain, user_id, ""], descending=True, limit=1, include_docs=True, ).first() return key_record
def notify_sentry_deploy(duration_mins): from settingshelper import get_release_name headers = { 'Authorization': 'Bearer {}'.format(settings.SENTRY_API_KEY), } payload = { 'environment': settings.SERVER_ENVIRONMENT, } if duration_mins: utcnow = datetime.utcnow() payload.update({ 'dateStarted': json_format_datetime(utcnow - timedelta(minutes=duration_mins)), 'dateFinished': json_format_datetime(utcnow), }) version = get_release_name(settings.BASE_DIR, settings.SERVER_ENVIRONMENT) releases_url = 'https://sentry.io/api/0/organizations/dimagi/releases/{}/deploys/'.format( version) requests.post(releases_url, headers=headers, json=payload)
def notify_event(domain, couch_user, app_id, form_unique_id, message): message_obj = RedisMessage(json.dumps({ 'domain': domain, 'user_id': couch_user._id, 'username': couch_user.username, 'text': message, 'timestamp': json_format_datetime(datetime.datetime.utcnow()), })) RedisPublisher( facility=get_facility_for_form(domain, app_id, form_unique_id), broadcast=True ).publish_message(message_obj)
def get_synclog_ids_by_date(start_datetime, end_datetime): ''' Returns all synclog ids that have been modified within a time range. The start date is exclusive while the end date is inclusive (start_datetime, end_datetime]. ''' from casexml.apps.phone.models import SyncLog json_start_datetime = json_format_datetime(start_datetime) results = SyncLog.view("sync_logs_by_date/view", startkey=[json_start_datetime], endkey=[json_format_datetime(end_datetime)], reduce=False, include_docs=False) for result in results: result_modified_datetime = result['key'][0] # Skip the record if the datetime is equal to the start because this should return # records with an exclusive start date. if result_modified_datetime == json_start_datetime: continue yield result['id']
def all(cls, domain=None, due_before=None, limit=None): json_now = json_format_datetime(due_before or datetime.utcnow()) repeat_records = RepeatRecord.view( "repeaters/repeat_records_by_next_check", startkey=[domain], endkey=[domain, json_now, {}], include_docs=True, reduce=False, limit=limit, ) return repeat_records
def get_overdue_repeat_record_count( overdue_threshold=datetime.timedelta(minutes=10)): from .models import RepeatRecord overdue_datetime = datetime.datetime.utcnow() - overdue_threshold results = RepeatRecord.view( "receiverwrapper/repeat_records_by_next_check", startkey=[None], endkey=[None, json_format_datetime(overdue_datetime)], reduce=True, ).one() return results['value'] if results else 0
def get_case_ids_modified_with_owner_since(domain, owner_id, reference_date, until_date=None): """ Gets all cases with a specified owner ID that have been modified since a particular reference_date (using the server's timestamp) """ return [ row['id'] for row in CommCareCase.get_db().view( 'cases_by_server_date/by_owner_server_modified_on', startkey=[domain, owner_id, json_format_datetime(reference_date)], endkey=[ domain, owner_id, {} if not until_date else json_format_datetime(until_date) ], include_docs=False, reduce=False) ]
def _get_form_metadata_context(domain, form, timezone, support_enabled=False): meta = _top_level_tags(form).get('meta', None) or {} meta['received_on'] = json_format_datetime(form.received_on) meta['server_modified_on'] = json_format_datetime(form.server_modified_on) if form.server_modified_on else '' if support_enabled: meta['last_sync_token'] = form.last_sync_token definition = get_default_definition(sorted_form_metadata_keys(list(meta))) form_meta_data = get_tables_as_columns(meta, definition, timezone=timezone) if getattr(form, 'auth_context', None): auth_context = AuthContext(form.auth_context) auth_context_user_id = auth_context.user_id auth_user_info = get_doc_info_by_id(domain, auth_context_user_id) else: auth_user_info = get_doc_info_by_id(domain, None) auth_context = AuthContext( user_id=None, authenticated=False, domain=domain, ) meta_userID = meta.get('userID') meta_username = meta.get('username') if meta_userID == 'demo_user': user_info = DocInfo( domain=domain, display='demo_user', ) elif meta_username == 'admin': user_info = DocInfo( domain=domain, display='admin', ) else: user_info = get_doc_info_by_id(domain, meta_userID) return { "form_meta_data": form_meta_data, "auth_context": auth_context, "auth_user_info": auth_user_info, "user_info": user_info, }
def enqueue_directly(self, sms): """ This method is used to try to send a QueuedSMS entry directly to the celery queue, without waiting for it to be enqueued by the handle() thread. """ try: self.enqueue(sms.pk, json_format_datetime(sms.datetime_to_process)) except: # If anything goes wrong here, no problem, the handle() thread will # pick it up later and enqueue. pass
def enqueue_directly(self, reminder): """ This method is used to try to send a reminder directly to the celery queue, without waiting for it to be enqueued by the handle() thread. """ try: self.enqueue(reminder._id, json_format_datetime(reminder.next_fire)) except: # If anything goes wrong here, no problem, the handle() thread will # pick it up later and enqueue. pass
def update_contact(domain, case_id, user_id, contact_phone_number=None, contact_phone_number_is_verified=None, contact_backend_id=None, language_code=None, time_zone=None): context = { "case_id" : case_id, "date_modified" : json_format_datetime(datetime.datetime.utcnow()), "user_id" : user_id, "contact_phone_number" : contact_phone_number, "contact_phone_number_is_verified" : contact_phone_number_is_verified, "contact_backend_id" : contact_backend_id, "language_code" : language_code, "time_zone" : time_zone } submit_xml(domain, "sms/xml/update_contact.xml", context)
def adjust_datetimes(data, parent=None, key=None): """ find all datetime-like strings within data (deserialized json) and format them uniformly, in place. this only processes timezones correctly if the call comes from a request with domain information otherwise it will default to not processing timezones. to force timezone processing, it can be called as follows >>> from corehq.apps.tzmigration.api import force_phone_timezones_should_be_processed >>> with force_phone_timezones_should_be_processed(): >>> adjust_datetimes(form_json) """ # this strips the timezone like we've always done # todo: in the future this will convert to UTC if isinstance(data, basestring) and jsonobject.re_loose_datetime.match(data): try: matching_datetime = iso8601.parse_date(data) except iso8601.ParseError: pass else: if phone_timezones_should_be_processed(): parent[key] = unicode(json_format_datetime( matching_datetime.astimezone(pytz.utc).replace(tzinfo=None) )) else: parent[key] = unicode(json_format_datetime( matching_datetime.replace(tzinfo=None))) elif isinstance(data, dict): for key, value in data.items(): adjust_datetimes(value, parent=data, key=key) elif isinstance(data, list): for i, value in enumerate(data): adjust_datetimes(value, parent=data, key=i) # return data, just for convenience in testing # this is the original input, modified, not a new data structure return data
def submit_unfinished_form(session): """ Gets the raw instance of the session's form and submits it. This is used with sms and ivr surveys to save all questions answered so far in a session that needs to close. If session.include_case_updates_in_partial_submissions is False, no case create / update / close actions will be performed, but the form will still be submitted. The form is only submitted if the smsforms session has not yet completed. """ # Get and clean the raw xml try: response = FormplayerInterface(session.session_id, session.domain).get_raw_instance() # Formplayer's ExceptionResponseBean includes the exception message, # stautus ("error"), url, and type ("text") if response.get('status') == 'error': raise TouchformsError(response.get('exception')) xml = response['output'] except InvalidSessionIdException: return root = XML(xml) case_tag_regex = re.compile( r"^(\{.*\}){0,1}case$" ) # Use regex in order to search regardless of namespace meta_tag_regex = re.compile(r"^(\{.*\}){0,1}meta$") timeEnd_tag_regex = re.compile(r"^(\{.*\}){0,1}timeEnd$") current_timstamp = json_format_datetime(utcnow()) for child in root: if case_tag_regex.match(child.tag) is not None: # Found the case tag case_element = child case_element.set("date_modified", current_timstamp) if not session.include_case_updates_in_partial_submissions: # Remove case actions (create, update, close) child_elements = [case_action for case_action in case_element] for case_action in child_elements: case_element.remove(case_action) elif meta_tag_regex.match(child.tag) is not None: # Found the meta tag, now set the value for timeEnd for meta_child in child: if timeEnd_tag_regex.match(meta_child.tag): meta_child.text = current_timstamp cleaned_xml = tostring(root) # Submit the xml result = submit_form_locally(cleaned_xml, session.domain, app_id=session.app_id, partial_submission=True) session.submission_id = result.xform.form_id
def test_get_list(self): """ Any form in the appropriate domain should be in the list from the API. """ # The actual infrastructure involves saving to CouchDB, having PillowTop # read the changes and write it to ElasticSearch. # In order to test just the API code, we set up a fake XFormES (this should # really be a parameter to the XFormInstanceResource constructor) # and write the translated form directly; we are not trying to test # the ptop infrastructure. # the pillow is set to offline mode - elasticsearch not needed to validate fake_xform_es = FakeXFormES() v0_4.MOCK_XFORM_ES = fake_xform_es backend_form = XFormInstance( xmlns='fake-xmlns', domain=self.domain.name, received_on=datetime.utcnow(), edited_on=datetime.utcnow(), form={ '#type': 'fake-type', '@xmlns': 'fake-xmlns', 'meta': { 'userID': 'metadata-user-id' }, }, auth_context={ 'user_id': 'auth-user-id', 'domain': self.domain.name, 'authenticated': True, }, ) backend_form.save() self.addCleanup(backend_form.delete) translated_doc = transform_xform_for_elasticsearch( backend_form.to_json()) fake_xform_es.add_doc(translated_doc['_id'], translated_doc) response = self._assert_auth_get_resource(self.list_endpoint) self.assertEqual(response.status_code, 200) api_forms = json.loads(response.content)['objects'] self.assertEqual(len(api_forms), 1) api_form = api_forms[0] self.assertEqual(api_form['form']['@xmlns'], backend_form.xmlns) self.assertEqual(api_form['received_on'], json_format_datetime(backend_form.received_on)) self.assertEqual(api_form['metadata']['userID'], 'metadata-user-id') self.assertEqual(api_form['edited_by_user_id'], 'auth-user-id')
def test_get_list(self): """ Any case in the appropriate domain should be in the list from the API. """ backend_case = self._setup_fake_es() response = self._assert_auth_get_resource(self.list_endpoint) self.assertEqual(response.status_code, 200) api_cases = json.loads(response.content)['objects'] self.assertEqual(len(api_cases), 1) api_case = api_cases[0] self.assertEqual(api_case['server_date_modified'], json_format_datetime(backend_case.server_modified_on))
def finish(self, save_stats=True, raise_exc=False): try: if (save_stats and self.stats_key and self.stats and self.redis_client): dumpable = {} for k, v in self.stats.items(): dumpable[k] = [json_format_datetime(t) for t in v] self.redis_client.set(self.stats_key, json.dumps(dumpable)) if self.lock: self.lock.release() except: if raise_exc: raise
def fake_form_submission(userID=userID, username=self.username, xmlns=self.xmlns, time=None): submission = submission_template % { "userID": userID, "username": username, "xmlns": xmlns } f = StringIO(submission.encode('utf-8')) f.name = "tempfile.xml" kwargs = dict(HTTP_X_SUBMIT_TIME=json_format_datetime(time)) if time else {} response = c.post("/a/{self.domain}/receiver/".format(self=self), { 'xml_submission_file': f, }, **kwargs) return response
def get_xml(self): updates = self._case._updates case_block = get_case_xml(self._case, updates, version='2.0') diff_block = get_diff_block(self._case) return render_to_string('hqcase/xml/case_block.xml', { 'xmlns': self.xmlns, 'case_block': case_block.decode('utf-8') + diff_block, 'time': json_format_datetime(self.received_on), 'uid': self.form_id, 'username': "", 'user_id': self.user_id or "", 'device_id': self.device_id, })
def json_handler(obj): if callable(getattr(obj, 'to_complete_json', None)): return obj.to_complete_json() elif callable(getattr(obj, 'to_json', None)): return obj.to_json() elif isinstance(obj, datetime): return json_format_datetime(obj) elif isinstance(obj, date): return obj.isoformat() elif isinstance(obj, Decimal): return float(obj) # warning, potential loss of precision else: return json.JSONEncoder().default(obj)
def submit_case_blocks(case_blocks, domain, username="******", user_id=None, xmlns=None, attachments=None, form_id=None, form_extras=None, case_db=None, device_id=None): """ Submits casexml in a manner similar to how they would be submitted from a phone. :param xmlns: Form XMLNS. Format: IRI or URN. Historically this was used in some places to uniquely identify the subsystem that posted the cases; `device_id` is now recommended for that purpose. Going forward, it is recommended to use the default value along with `device_id`, which indicates that the cases were submitted by an internal system process. :param device_id: Identifier for the source of posted cases. Ideally this should uniquely identify the subsystem that is posting cases to make it easier to trace the source. All new code should use this argument. A human recognizable value is recommended outside of test code. Example: "auto-close-rule-<GUID>" returns the UID of the resulting form. """ attachments = attachments or {} now = json_format_datetime(datetime.datetime.utcnow()) if not isinstance(case_blocks, six.string_types): case_blocks = ''.join(case_blocks) soft_assert_type_text(case_blocks) form_id = form_id or uuid.uuid4().hex form_xml = render_to_string( 'hqcase/xml/case_block.xml', { 'xmlns': xmlns or SYSTEM_FORM_XMLNS, 'case_block': case_blocks, 'time': now, 'uid': form_id, 'username': username, 'user_id': user_id or "", 'device_id': device_id or "", }) form_extras = form_extras or {} result = submit_form_locally(instance=form_xml, domain=domain, attachments=attachments, case_db=case_db, **form_extras) return result.xform, result.cases
def iteration_parameters(db, doc_type, domain, view_range, group, chunk_size=1000): if "view" in group: view_name = group["view"] start = end = "-" assert doc_type is None, doc_type if domain is not None: startkey = [domain] endkey = [domain] else: startkey = [] endkey = [] elif domain is not None: view_name = 'by_domain_doc_type_date/view' if doc_type is not None: startkey = [domain, doc_type] endkey = [domain, doc_type] else: startkey = [domain] endkey = [domain] elif doc_type is not None: view_name = 'all_docs/by_doc_type' startkey = [doc_type] endkey = [doc_type] else: view_name = 'all_docs/by_doc_type' startkey = [] endkey = [] if view_range is not None: assert domain or doc_type, (domain, doc_type) if group.get("date_range"): assert domain and doc_type and view_name == 'by_domain_doc_type_date/view', \ (domain, doc_type, view_name, view_range) view_range = [json_format_datetime(parse_date(x)) for x in view_range] start, end = view_range startkey.append(start) endkey.append(end) else: start = end = "-" if startkey == endkey: endkey.append({}) view_params = { 'startkey': startkey, 'endkey': endkey, 'limit': chunk_size, 'include_docs': False, 'reduce': False, } resume_key = f"{db.dbname}.{domain}.{doc_type}.{start}-{end}" return resume_key, view_name, view_params
def iterate_repeat_records(due_before, chunk_size=10000, database=None): from .models import RepeatRecord json_now = json_format_datetime(due_before) view_kwargs = { 'reduce': False, 'startkey': [None], 'endkey': [None, json_now, {}], 'include_docs': True } for doc in paginate_view(RepeatRecord.get_db(), 'receiverwrapper/repeat_records_by_next_check', chunk_size, **view_kwargs): yield RepeatRecord.wrap(doc['doc'])
def serialize(value): """ Serializes a value so it can properly be parsed into XML """ if isinstance(value, datetime.datetime): return json_format_datetime(value) elif isinstance(value, datetime.date): return value.isoformat() elif isinstance(value, datetime.time): return value.strftime('%H:%M:%S') elif isinstance(value, (int, Decimal, float)): return str(value) else: return value if value is not None else ""