def process(domain, instance):
"""process an incoming commtrack stock report instance"""
config = CommtrackConfig.for_domain(domain)
root = etree.fromstring(instance)
transactions = unpack_transactions(root, config)
case_ids = [tx["case_id"] for tx in transactions]
cases = dict((c._id, c) for c in CommCareCase.view("_all_docs", keys=case_ids, include_docs=True))
# ensure transaction types are processed in the correct order
def transaction_order(tx):
return [action.action_name for action in config.actions].index(tx["action"])
transactions.sort(key=transaction_order)
# apply all transactions to each product case in bulk
transactions_by_product = map_reduce(lambda tx: [(tx["case_id"],)], data=transactions, include_docs=True)
for product_id, txs in transactions_by_product.iteritems():
product_case = cases[product_id]
case_block, reconciliations = process_product_transactions(product_case, txs)
for recon in reconciliations:
root.append(recon)
root.append(case_block)
submission = etree.tostring(root)
logger.debug("submitting: %s" % submission)
submit_time = root.find(".//%s" % _("timeStart", META_XMLNS)).text
spoof_submission(get_submit_url(domain), submission, headers={"HTTP_X_SUBMIT_TIME": submit_time})
def process(domain, instance):
"""process an incoming commtrack stock report instance"""
config = CommtrackConfig.for_domain(domain)
root = etree.fromstring(instance)
user_id, transactions = unpack_transactions(root, config)
transactions = list(normalize_transactions(transactions))
def get_transactions(all_tx, type_filter):
"""get all the transactions of the relevant type (filtered by type_filter),
grouped by product (returns a dict of 'product subcase id' => list of transactions),
with each set of transactions sorted in the correct order for processing
"""
return map_reduce(
lambda tx: [(tx.case_id, )],
lambda v: sorted(v, key=lambda tx: tx.priority_order
), # important!
data=filter(type_filter, all_tx),
include_docs=True)
# split transactions by type and product
stock_transactions = get_transactions(transactions,
lambda tx: tx.category == 'stock')
requisition_transactions = get_transactions(
transactions, lambda tx: tx.category == 'requisition')
case_ids = list(
set(itertools.chain(*[tx.get_case_ids() for tx in transactions])))
cases = dict((c._id, c) for c in CommCareCase.view(
'_all_docs', keys=case_ids, include_docs=True))
# TODO: code to auto generate / update requisitions from transactions if
# project is configured for that.
# TODO: when we start receiving commcare-submitted reports, we should be using a server time rather
# than relying on timeStart (however timeStart is set to server time for reports received via sms)
submit_time = root.find('.//%s' % _('timeStart', META_XMLNS)).text
post_processed_transactions = list(transactions)
for product_id, product_case in cases.iteritems():
stock_txs = stock_transactions.get(product_id, [])
if stock_txs:
case_block, reconciliations = process_product_transactions(
user_id, submit_time, product_case, stock_txs)
root.append(case_block)
post_processed_transactions.extend(reconciliations)
req_txs = requisition_transactions.get(product_id, [])
if req_txs and config.requisitions_enabled:
req = RequisitionState.from_transactions(user_id, product_case,
req_txs)
case_block = etree.fromstring(req.to_xml())
root.append(case_block)
replace_transactions(root, post_processed_transactions)
submission = etree.tostring(root)
logger.debug('submitting: %s' % submission)
spoof_submission(get_submit_url(domain),
submission,
headers={'HTTP_X_SUBMIT_TIME': submit_time},
hqsubmission=False)
def handle(self, *args, **options):
if len(args) != 3:
raise CommandError('Usage is duplicate_cases %s' % self.args)
start = datetime.now()
case_id_file, user_id, domain = args
self.submit_url = URL_BASE + get_submit_url(domain)
print self.submit_url
user = CouchUser.get_by_user_id(user_id, domain)
user_id = user._id
if not user.is_member_of(domain):
raise CommandError("%s can't access %s" % (user, domain))
self.read_progress()
with open(case_id_file, 'r') as f:
case_ids = f.readlines()
try:
for id in case_ids:
self.duplicate_case(id.strip(), domain, user_id)
finally:
self.write_progress()
print 'finished in %s seconds' % (datetime.now() - start).seconds
print '{} cases processed'.format(len(self.cases_processing.keys()))
print '{} forms processed'.format(len(self.forms_processing.keys()))
def _test(self, submit_app_id, expected_app_id, expected_build_id):
form_id = spoof_submission(
get_submit_url(self.domain, submit_app_id),
'<data xmlns="http://example.com/"><question1/></data>')
form = FormAccessors(self.domain).get_form(form_id)
self.assertEqual(form.app_id, expected_app_id)
self.assertEqual(form.build_id, expected_build_id)
return form
def _submit_form(self, f=None, domain=DOMAIN):
if f is None:
f = get_form()
url = get_submit_url(domain)
self._send_form_to_es(
domain=domain,
xmlns="http://www.commcarehq.org/export/test",
)
return spoof_submission(url, f)
def _test(self, submit_app_id, expected_app_id, expected_build_id):
form_id = spoof_submission(
get_submit_url(self.domain, submit_app_id),
'<data xmlns="http://example.com/"><question1/></data>'
)
form = FormAccessors(self.domain).get_form(form_id)
self.assertEqual(form.app_id, expected_app_id)
self.assertEqual(form.build_id, expected_build_id)
return form
def _submit_form(self, f=None, domain=DOMAIN):
if f is None:
f = get_form()
url = get_submit_url(domain)
self._send_form_to_es(
domain=domain,
xmlns="http://www.commcarehq.org/export/test",
)
return spoof_submission(url, f)
def process(domain, instance):
"""process an incoming commtrack stock report instance"""
config = CommtrackConfig.for_domain(domain)
root = etree.fromstring(instance)
user_id, transactions = unpack_transactions(root, config)
transactions = list(normalize_transactions(transactions))
def get_transactions(all_tx, type_filter):
"""get all the transactions of the relevant type (filtered by type_filter),
grouped by product (returns a dict of 'product subcase id' => list of transactions),
with each set of transactions sorted in the correct order for processing
"""
return map_reduce(
lambda tx: [(tx.case_id,)],
lambda v: sorted(v, key=lambda tx: tx.priority_order), # important!
data=filter(type_filter, all_tx),
include_docs=True,
)
# split transactions by type and product
stock_transactions = get_transactions(transactions, lambda tx: tx.category == "stock")
requisition_transactions = get_transactions(transactions, lambda tx: tx.category == "requisition")
case_ids = list(set(itertools.chain(*[tx.get_case_ids() for tx in transactions])))
cases = dict((c._id, c) for c in CommCareCase.view("_all_docs", keys=case_ids, include_docs=True))
# TODO: code to auto generate / update requisitions from transactions if
# project is configured for that.
# TODO: when we start receiving commcare-submitted reports, we should be using a server time rather
# than relying on timeStart (however timeStart is set to server time for reports received via sms)
submit_time = root.find(".//%s" % _("timeStart", META_XMLNS)).text
post_processed_transactions = list(transactions)
for product_id, product_case in cases.iteritems():
stock_txs = stock_transactions.get(product_id, [])
if stock_txs:
case_block, reconciliations = process_product_transactions(user_id, submit_time, product_case, stock_txs)
root.append(case_block)
post_processed_transactions.extend(reconciliations)
req_txs = requisition_transactions.get(product_id, [])
if req_txs and config.requisitions_enabled:
req = RequisitionState.from_transactions(user_id, product_case, req_txs)
case_block = etree.fromstring(req.to_xml())
root.append(case_block)
replace_transactions(root, post_processed_transactions)
submission = etree.tostring(root)
logger.debug("submitting: %s" % submission)
spoof_submission(
get_submit_url(domain), submission, headers={"HTTP_X_SUBMIT_TIME": submit_time}, hqsubmission=False
)
def handle_sms_form_complete(sender, session_id, form, **kwargs):
from corehq.apps.smsforms.models import XFormsSession
session = XFormsSession.latest_by_session_id(session_id)
if session:
# i don't know if app_id is the id of the overall app or the id of the specific build of the app
# the thing i want to pass in is the id of the overall app
resp = spoof_submission(get_submit_url(session.domain, session.app_id), form, hqsubmission=False)
xform_id = resp["X-CommCareHQ-FormID"]
session.end(completed=True)
session.submission_id = xform_id
session.save()
def submit_case_blocks(case_blocks, domain):
now = json_format_datetime(datetime.datetime.utcnow())
if not isinstance(case_blocks, basestring):
case_blocks = ''.join(case_blocks)
form_xml = render_to_string('hqcase/xml/case_block.xml', {
'case_block': case_blocks,
'time': now,
'uid': uuid.uuid4().hex
})
spoof_submission(
get_submit_url(domain),
form_xml,
)
def process(domain, data):
import pprint
logger.debug(pprint.pformat(data))
xmlroot = to_instance(data)
submit_time = xmlroot.find('.//%s' % _('timeStart', const.META_XMLNS)).text
submission = etree.tostring(xmlroot, encoding='utf-8', pretty_print=True)
logger.debug(submission)
# submit it
spoof_submission(get_submit_url(domain), submission,
headers={'HTTP_X_SUBMIT_TIME': submit_time},
hqsubmission=False)
def post(request, domain):
"""expects same posts as receiver but munges the forms before passing them on"""
xml = request.raw_post_data
user_map = UserMap(domain)
submit = True
if is_user_registration(xml):
submit = update_migration_users(xml, user_map)
else:
xml = add_user_id(xml, user_map)
if submit:
submit_time = request.META.get('HTTP_X_SUBMIT_TIME', None)
headers = {"HTTP_X_SUBMIT_TIME": submit_time} if submit_time else {}
return spoof_submission(get_submit_url(domain), xml, hqsubmission=False, headers=headers)
else:
return HttpResponse("user already exists")
def process(domain, instance):
"""process an incoming commtrack stock report instance"""
config = CommtrackConfig.for_domain(domain)
root = etree.fromstring(instance)
user_id, transactions = unpack_transactions(root, config)
case_ids = [tx.case_id for tx in transactions]
cases = dict((c._id, c) for c in CommCareCase.view('_all_docs', keys=case_ids, include_docs=True))
def get_transactions(all_tx, type_filter):
"""get all the transactions of the relevant type (filtered by type_filter),
grouped by product (returns a dict of 'product subcase id' => list of transactions),
with each set of transactions sorted in the correct order for processing
"""
return map_reduce(lambda tx: [(tx.case_id,)],
lambda v: sorted(v, key=lambda tx: tx.priority_order), # important!
data=filter(type_filter, all_tx),
include_docs=True)
# split transactions by type and product
stock_transactions = get_transactions(transactions, lambda tx: tx.category == 'stock')
requisition_transactions = get_transactions(transactions, lambda tx: tx.category == 'requisition')
# TODO: code to auto generate / update requisitions from transactions if
# project is configured for that.
post_processed_transactions = list(transactions)
for product_id, product_case in cases.iteritems():
stock_txs = stock_transactions.get(product_id, [])
if stock_txs:
case_block, reconciliations = process_product_transactions(user_id, product_case, stock_txs)
root.append(case_block)
post_processed_transactions.extend(reconciliations)
req_txs = requisition_transactions.get(product_id, [])
if req_txs and config.requisitions_enabled:
req = RequisitionState.from_transactions(user_id, product_case, req_txs)
case_block = etree.fromstring(req.to_xml())
root.append(case_block)
replace_transactions(root, post_processed_transactions)
submission = etree.tostring(root)
logger.debug('submitting: %s' % submission)
submit_time = root.find('.//%s' % _('timeStart', META_XMLNS)).text
spoof_submission(get_submit_url(domain), submission, headers={'HTTP_X_SUBMIT_TIME': submit_time}, hqsubmission=False)
def handle_sms_form_complete(sender, session_id, form, **kwargs):
from corehq.apps.smsforms.models import XFormsSession
session = XFormsSession.latest_by_session_id(session_id)
if session:
# i don't know if app_id is the id of the overall app or the id of the specific build of the app
# the thing i want to pass in is the id of the overall app
resp = spoof_submission(get_submit_url(session.domain, session.app_id),
form,
hqsubmission=False)
xform_id = resp['X-CommCareHQ-FormID']
session.end(completed=True)
session.submission_id = xform_id
session.save()
xform = XFormInstance.get(xform_id)
xform.survey_incentive = session.survey_incentive
xform.save()
def submit_unfinished_form(session_id, include_case_side_effects=False):
session = XFormsSession.latest_by_session_id(session_id)
if session is not None and session.end_time is None:
# Get and clean the raw xml
xml = get_raw_instance(session_id)
root = XML(xml)
case_tag_regex = re.compile(
"^(\{.*\}){0,1}case$"
) # Use regex in order to search regardless of namespace
meta_tag_regex = re.compile("^(\{.*\}){0,1}meta$")
timeEnd_tag_regex = re.compile("^(\{.*\}){0,1}timeEnd$")
current_timstamp = json_format_datetime(datetime.utcnow())
for child in root:
if case_tag_regex.match(child.tag) is not None:
# Found the case tag
case_element = child
case_element.set("date_modified", current_timstamp)
if not include_case_side_effects:
# Remove case actions (create, update, close)
child_elements = [
case_action for case_action in case_element
]
for case_action in child_elements:
case_element.remove(case_action)
elif meta_tag_regex.match(child.tag) is not None:
# Found the meta tag, now set the value for timeEnd
for meta_child in child:
if timeEnd_tag_regex.match(meta_child.tag):
meta_child.text = current_timstamp
cleaned_xml = tostring(root)
# Submit the xml and end the session
resp = spoof_submission(get_submit_url(session.domain, session.app_id),
cleaned_xml,
hqsubmission=False)
xform_id = resp['X-CommCareHQ-FormID']
session.end(completed=False)
session.submission_id = xform_id
session.save()
# Tag the submission as a partial submission
xform = XFormInstance.get(xform_id)
xform.partial_submission = True
xform.survey_incentive = session.survey_incentive
xform.save()
def submit_case_blocks(case_blocks, domain, username="******", user_id="",
xmlns='http://commcarehq.org/case'):
now = json_format_datetime(datetime.datetime.utcnow())
if not isinstance(case_blocks, basestring):
case_blocks = ''.join(case_blocks)
form_xml = render_to_string('hqcase/xml/case_block.xml', {
'xmlns': xmlns,
'case_block': case_blocks,
'time': now,
'uid': uuid.uuid4().hex,
'username': username,
'user_id': user_id,
})
spoof_submission(
get_submit_url(domain),
form_xml,
hqsubmission=False,
)
def submit_unfinished_form(session_id, include_case_side_effects=False):
session = XFormsSession.latest_by_session_id(session_id)
if session is not None and session.end_time is None:
# Get and clean the raw xml
try:
xml = get_raw_instance(session_id)
except InvalidSessionIdException:
session.end(completed=False)
session.save()
return
root = XML(xml)
case_tag_regex = re.compile("^(\{.*\}){0,1}case$") # Use regex in order to search regardless of namespace
meta_tag_regex = re.compile("^(\{.*\}){0,1}meta$")
timeEnd_tag_regex = re.compile("^(\{.*\}){0,1}timeEnd$")
current_timstamp = json_format_datetime(datetime.utcnow())
for child in root:
if case_tag_regex.match(child.tag) is not None:
# Found the case tag
case_element = child
case_element.set("date_modified", current_timstamp)
if not include_case_side_effects:
# Remove case actions (create, update, close)
child_elements = [case_action for case_action in case_element]
for case_action in child_elements:
case_element.remove(case_action)
elif meta_tag_regex.match(child.tag) is not None:
# Found the meta tag, now set the value for timeEnd
for meta_child in child:
if timeEnd_tag_regex.match(meta_child.tag):
meta_child.text = current_timstamp
cleaned_xml = tostring(root)
# Submit the xml and end the session
resp = spoof_submission(get_submit_url(session.domain, session.app_id), cleaned_xml, hqsubmission=False)
xform_id = resp['X-CommCareHQ-FormID']
session.end(completed=False)
session.submission_id = xform_id
session.save()
# Tag the submission as a partial submission
xform = XFormInstance.get(xform_id)
xform.partial_submission = True
xform.survey_incentive = session.survey_incentive
xform.save()
def post(request, domain):
"""expects same posts as receiver but munges the forms before passing them on"""
xml = request.raw_post_data
user_map = UserMap(domain)
submit = True
if is_user_registration(xml):
submit = update_migration_users(xml, user_map)
else:
xml = add_user_id(xml, user_map)
if submit:
submit_time = request.META.get('HTTP_X_SUBMIT_TIME', None)
headers = {"HTTP_X_SUBMIT_TIME": submit_time} if submit_time else {}
return spoof_submission(get_submit_url(domain),
xml,
hqsubmission=False,
headers=headers)
else:
return HttpResponse("user already exists")
def submit_case_blocks(case_blocks,
domain,
username="******",
user_id="",
xmlns='http://commcarehq.org/case'):
now = json_format_datetime(datetime.datetime.utcnow())
if not isinstance(case_blocks, basestring):
case_blocks = ''.join(case_blocks)
form_xml = render_to_string(
'hqcase/xml/case_block.xml', {
'xmlns': xmlns,
'case_block': case_blocks,
'time': now,
'uid': uuid.uuid4().hex,
'username': username,
'user_id': user_id,
})
spoof_submission(
get_submit_url(domain),
form_xml,
hqsubmission=False,
)
def submit_form(f=None, domain=DOMAIN):
if f is None:
f = get_form()
url = get_submit_url(domain)
return spoof_submission(url, f, hqsubmission=False)
def _submit_form(f=None, domain=DOMAIN):
if f is None:
f = get_form()
url = get_submit_url(domain)
return spoof_submission(url, f)
def actually_close_cases(cases):
for case in cases:
for referral in case.referrals:
case.force_close_referral(get_submit_url(domain), referral)
case.force_close(get_submit_url(domain))
