def handle(self, *args, **options):
ids = get_form_ids_by_type('ipm-senegal', 'XFormInstance')
to_save = []
for doc in iter_docs(XFormInstance.get_db(), ids):
try:
if 'location_id' in doc[
'form'] and not doc['form']['location_id']:
case = SupplyPointCase.get(doc['form']['case']['@case_id'])
if case.type == 'supply-point':
instance = XFormInstance.get(doc['_id'])
# fix the XFormInstance
instance.form['location_id'] = case.location_id
# fix the actual form.xml
xml_object = etree.fromstring(instance.get_xml())
location_id_node = xml_object.find(
re.sub('}.*', '}location_id', xml_object.tag))
location_id_node.text = case.location_id
updated_xml = etree.tostring(xml_object)
attachment_builder = CouchAttachmentsBuilder(
instance._attachments)
attachment_builder.add(
name='form.xml',
content=updated_xml,
content_type=instance._attachments['form.xml']
['content_type'])
instance._attachments = attachment_builder.to_json()
print 'Updating XFormInstance:', doc['_id']
to_save.append(instance)
def store_attachments(cls, xform, attachments):
builder = CouchAttachmentsBuilder()
for attachment in attachments:
builder.add(
content=attachment.content,
name=attachment.name,
content_type=attachment.content_type,
)
xform._attachments = builder.to_json()
def from_instance(cls, instance, message):
"""
Create an instance of this record from a submission body
"""
attachments_builder = CouchAttachmentsBuilder()
attachments_builder.add(content=instance, name=ATTACHMENT_NAME, content_type="text/xml")
return SubmissionErrorLog(
received_on=datetime.datetime.utcnow(),
md5=hashlib.md5(instance).hexdigest(),
problem=message,
_attachments=attachments_builder.to_json(),
)
def handle(self, *args, **options):
ids = get_form_ids_by_type('ipm-senegal', 'XFormInstance')
to_save = []
locations = SQLLocation.objects.filter(domain='ipm-senegal').values_list('location_id', 'name')
locations_map = {location_id: name for (location_id, name) in locations}
for doc in iter_docs(XFormInstance.get_db(), ids):
try:
if 'PPS_name' in doc['form'] and not doc['form']['PPS_name']:
case = SupplyPointCase.get(doc['form']['case']['@case_id'])
if case.type == 'supply-point':
print 'Updating XFormInstance:', doc['_id']
pps_name = locations_map[case.location_id]
instance = XFormInstance.get(doc['_id'])
# fix the XFormInstance
instance.form['PPS_name'] = pps_name
for instance_prod in instance.form['products']:
instance_prod['PPS_name'] = instance_prod['PPS_name'] or pps_name
# fix the actual form.xml
xml_object = etree.fromstring(instance.get_xml())
pps_name_node = xml_object.find(re.sub('}.*', '}PPS_name', xml_object.tag))
pps_name_node.text = pps_name
products_nodes = xml_object.findall(re.sub('}.*', '}products', xml_object.tag))
for product_node in products_nodes:
product_pps_name_node = product_node.find(re.sub('}.*', '}PPS_name', xml_object.tag))
product_pps_name_node.text = pps_name
updated_xml = etree.tostring(xml_object)
attachment_builder = CouchAttachmentsBuilder(instance._attachments)
attachment_builder.add(
name='form.xml',
content=updated_xml,
content_type=instance._attachments['form.xml']['content_type']
)
instance._attachments = attachment_builder.to_json()
to_save.append(instance)
except Exception:
print 'Failed to save XFormInstance:', doc['_id']
if len(to_save) > 500:
XFormInstance.get_db().bulk_save(to_save)
to_save = []
if to_save:
XFormInstance.get_db().bulk_save(to_save)
def handle(self, *args, **options):
ids = get_form_ids_by_type('ipm-senegal', 'XFormInstance')
to_save = []
locations = SQLLocation.objects.filter(
domain='ipm-senegal').values_list('location_id', 'name')
locations_map = {
location_id: name
for (location_id, name) in locations
}
for doc in iter_docs(XFormInstance.get_db(), ids):
try:
if 'PPS_name' in doc['form'] and not doc['form']['PPS_name']:
case = SupplyPointCase.get(doc['form']['case']['@case_id'])
if case.type == 'supply-point':
print 'Updating XFormInstance:', doc['_id']
pps_name = locations_map[case.location_id]
instance = XFormInstance.get(doc['_id'])
# fix the XFormInstance
instance.form['PPS_name'] = pps_name
for instance_prod in instance.form['products']:
instance_prod['PPS_name'] = instance_prod[
'PPS_name'] or pps_name
# fix the actual form.xml
xml_object = etree.fromstring(instance.get_xml())
pps_name_node = xml_object.find(
re.sub('}.*', '}PPS_name', xml_object.tag))
pps_name_node.text = pps_name
products_nodes = xml_object.findall(
re.sub('}.*', '}products', xml_object.tag))
for product_node in products_nodes:
product_pps_name_node = product_node.find(
re.sub('}.*', '}PPS_name', xml_object.tag))
product_pps_name_node.text = pps_name
updated_xml = etree.tostring(xml_object)
attachment_builder = CouchAttachmentsBuilder(
instance._attachments)
attachment_builder.add(
name='form.xml',
content=updated_xml,
content_type=instance._attachments['form.xml']
['content_type'])
instance._attachments = attachment_builder.to_json()
to_save.append(instance)
def handle(self, *args, **options):
startkey = ['ipm-senegal', 'by_type', 'XFormInstance']
endkey = startkey + [{}]
ids = [row['id'] for row in XFormInstance.get_db().view(
"couchforms/all_submissions_by_domain",
startkey=startkey,
endkey=endkey,
reduce=False
)]
to_save = []
for doc in iter_docs(XFormInstance.get_db(), ids):
try:
if 'location_id' in doc['form'] and not doc['form']['location_id']:
case = SupplyPointCase.get(doc['form']['case']['@case_id'])
if case.type == 'supply-point':
instance = XFormInstance.get(doc['_id'])
# fix the XFormInstance
instance.form['location_id'] = case.location_id
# fix the actual form.xml
xml_object = etree.fromstring(instance.get_xml())
location_id_node = xml_object.find(re.sub('}.*', '}location_id', xml_object.tag))
location_id_node.text = case.location_id
updated_xml = etree.tostring(xml_object)
attachment_builder = CouchAttachmentsBuilder(instance._attachments)
attachment_builder.add(
name='form.xml',
content=updated_xml,
content_type=instance._attachments['form.xml']['content_type']
)
instance._attachments = attachment_builder.to_json()
print 'Updating XFormInstance:', doc['_id']
to_save.append(instance)
except Exception:
print 'Failed to save XFormInstance:', doc['_id']
if len(to_save) > 500:
XFormInstance.get_db().bulk_save(to_save)
to_save = []
if to_save:
XFormInstance.get_db().bulk_save(to_save)
def _apply_attachments_action(self, attachment_action, xform=None):
"""
if xform is provided it will be used to fetch attachments
"""
# the actions and attachment must be added before the first saves can happen
# todo attach cached attachment info
def fetch_attachment(name):
if fetch_attachment.form is None:
fetch_attachment.form = XFormInstance.get(attachment_action.xform_id)
return fetch_attachment.form.fetch_attachment(name)
fetch_attachment.form = xform
attach_dict = {}
# cache all attachment streams from xform
for k, v in attachment_action.attachments.items():
if v.is_present:
# fetch attachment, update metadata, get the stream
attach_data = fetch_attachment(v.attachment_src)
attach_dict[k] = attach_data
v.attachment_size = len(attach_data)
if v.is_image:
img = Image.open(StringIO(attach_data))
img_size = img.size
props = dict(width=img_size[0], height=img_size[1])
v.attachment_properties = props
update_attachments = {}
for k, v in self.case.case_attachments.items():
if v.is_present:
update_attachments[k] = v
if self.case._attachments:
attachment_builder = CouchAttachmentsBuilder(
self.case['_attachments'])
else:
attachment_builder = CouchAttachmentsBuilder()
for k, v in attachment_action.attachments.items():
# grab xform_attachments
# copy over attachments from form onto case
update_attachments[k] = v
if v.is_present:
#fetch attachment from xform
identifier = v.identifier
attach = attach_dict[identifier]
attachment_builder.add(name=k, content=attach,
content_type=v.server_mime)
else:
try:
attachment_builder.remove(k)
except KeyError:
pass
del update_attachments[k]
self.case._attachments = attachment_builder.to_json()
self.case.case_attachments = update_attachments
def create_xform(xml_string, attachments=None, _id=None, process=None):
"""
create but do not save an XFormInstance from an xform payload (xml_string)
optionally set the doc _id to a predefined value (_id)
return doc _id of the created doc
`process` is transformation to apply to the form right before saving
This is to avoid having to save multiple times
If xml_string is bad xml
- raise couchforms.XMLSyntaxError
"""
from corehq.util.couch_helpers import CouchAttachmentsBuilder
assert attachments is not None
json_form = convert_xform_to_json(xml_string)
adjust_datetimes(json_form)
_id = (_id or _extract_meta_instance_id(json_form)
or XFormInstance.get_db().server.next_uuid())
assert _id
attachments_builder = CouchAttachmentsBuilder()
attachments_builder.add(
content=xml_string,
name='form.xml',
content_type='text/xml',
)
for key, value in attachments.items():
attachments_builder.add(
content=value,
name=key,
content_type=value.content_type,
)
xform = XFormInstance(
# form has to be wrapped
{'form': json_form},
# other properties can be set post-wrap
_id=_id,
xmlns=json_form.get('@xmlns'),
_attachments=attachments_builder.to_json(),
received_on=datetime.datetime.utcnow(),
)
# this had better not fail, don't think it ever has
# if it does, nothing's saved and we get a 500
if process:
process(xform)
lock = acquire_lock_for_xform(_id)
with ReleaseOnError(lock):
if _id in XFormInstance.get_db():
raise DuplicateError()
return LockManager(xform, lock)
def _apply_attachments_action(self, attachment_action, xform=None):
"""
if xform is provided it will be used to fetch attachments
"""
# the actions and attachment must be added before the first saves can happen
# todo attach cached attachment info
def fetch_attachment(name):
if fetch_attachment.form is None:
fetch_attachment.form = XFormInstance.get(attachment_action.xform_id)
return fetch_attachment.form.fetch_attachment(name)
fetch_attachment.form = xform
attach_dict = {}
# cache all attachment streams from xform
for k, v in attachment_action.attachments.items():
if v.is_present:
# fetch attachment, update metadata, get the stream
attach_data = fetch_attachment(v.attachment_src)
attach_dict[k] = attach_data
v.attachment_size = len(attach_data)
if v.is_image:
img = Image.open(StringIO(attach_data))
img_size = img.size
props = dict(width=img_size[0], height=img_size[1])
v.attachment_properties = props
update_attachments = {}
for k, v in self.case.case_attachments.items():
if v.is_present:
update_attachments[k] = v
if self.case._attachments:
attachment_builder = CouchAttachmentsBuilder(
self.case['_attachments'])
else:
attachment_builder = CouchAttachmentsBuilder()
for k, v in attachment_action.attachments.items():
# grab xform_attachments
# copy over attachments from form onto case
update_attachments[k] = v
if v.is_present:
#fetch attachment from xform
identifier = v.identifier
attach = attach_dict[identifier]
attachment_builder.add(name=k, content=attach,
content_type=v.server_mime)
else:
try:
attachment_builder.remove(k)
except KeyError:
pass
del update_attachments[k]
self.case._attachments = attachment_builder.to_json()
self.case.case_attachments = update_attachments
def create_xform(xml_string, attachments=None, _id=None, process=None):
"""
create but do not save an XFormInstance from an xform payload (xml_string)
optionally set the doc _id to a predefined value (_id)
return doc _id of the created doc
`process` is transformation to apply to the form right before saving
This is to avoid having to save multiple times
If xml_string is bad xml
- raise couchforms.XMLSyntaxError
"""
from corehq.util.couch_helpers import CouchAttachmentsBuilder
assert attachments is not None
json_form = convert_xform_to_json(xml_string)
adjust_datetimes(json_form)
_id = (_id or _extract_meta_instance_id(json_form)
or XFormInstance.get_db().server.next_uuid())
assert _id
attachments_builder = CouchAttachmentsBuilder()
attachments_builder.add(
content=xml_string,
name='form.xml',
content_type='text/xml',
)
for key, value in attachments.items():
attachments_builder.add(
content=value,
name=key,
content_type=value.content_type,
)
xform = XFormInstance(
# form has to be wrapped
{'form': json_form},
# other properties can be set post-wrap
_id=_id,
xmlns=json_form.get('@xmlns'),
_attachments=attachments_builder.to_json(),
received_on=datetime.datetime.utcnow(),
)
# this had better not fail, don't think it ever has
# if it does, nothing's saved and we get a 500
if process:
process(xform)
lock = acquire_lock_for_xform(_id)
with ReleaseOnError(lock):
if _id in XFormInstance.get_db():
raise DuplicateError()
return LockManager(xform, lock)
def _apply_attachments_action(self, attachment_action, xform=None):
"""
if xform is provided, attachments will be looked for
in the xform's _attachments.
They should be base64 encoded under _attachments[name]['data']
"""
# the actions and _attachment must be added before the first saves can happen
# todo attach cached attachment info
def fetch_attachment(name):
if xform and 'data' in xform._attachments[name]:
assert xform.form_id == attachment_action.xform_id
return base64.b64decode(xform._attachments[name]['data'])
else:
return XFormInstance.get_db().fetch_attachment(attachment_action.xform_id, name)
stream_dict = {}
# cache all attachment streams from xform
for k, v in attachment_action.attachments.items():
if v.is_present:
# fetch attachment, update metadata, get the stream
attach_data = fetch_attachment(v.attachment_src)
stream_dict[k] = attach_data
v.attachment_size = len(attach_data)
if v.is_image:
img = Image.open(StringIO(attach_data))
img_size = img.size
props = dict(width=img_size[0], height=img_size[1])
v.attachment_properties = props
update_attachments = {}
for k, v in self.case.case_attachments.items():
if v.is_present:
update_attachments[k] = v
if self.case._attachments:
attachment_builder = CouchAttachmentsBuilder(
self.case['_attachments'])
else:
attachment_builder = CouchAttachmentsBuilder()
for k, v in attachment_action.attachments.items():
# grab xform_attachments
# copy over attachments from form onto case
update_attachments[k] = v
if v.is_present:
#fetch attachment from xform
attachment_key = v.attachment_key
attach = stream_dict[attachment_key]
attachment_builder.add(name=k, content=attach,
content_type=v.server_mime)
else:
try:
attachment_builder.remove(k)
except KeyError:
pass
del update_attachments[k]
self.case._attachments = attachment_builder.to_json()
self.case.case_attachments = update_attachments
def apply_attachments(self, attachment_action, xform=None):
"""
if xform is provided, attachments will be looked for
in the xform's _attachments.
They should be base64 encoded under _attachments[name]['data']
"""
# the actions and _attachment must be added before the first saves can happen
# todo attach cached attachment info
def fetch_attachment(name):
if xform and 'data' in xform._attachments[name]:
assert xform._id == attachment_action.xform_id
return base64.b64decode(xform._attachments[name]['data'])
else:
return XFormInstance.get_db().fetch_attachment(
attachment_action.xform_id, name)
stream_dict = {}
# cache all attachment streams from xform
for k, v in attachment_action.attachments.items():
if v.is_present:
# fetch attachment, update metadata, get the stream
attach_data = fetch_attachment(v.attachment_src)
stream_dict[k] = attach_data
v.attachment_size = len(attach_data)
if v.is_image:
img = Image.open(StringIO(attach_data))
img_size = img.size
props = dict(width=img_size[0], height=img_size[1])
v.attachment_properties = props
update_attachments = {}
for k, v in self.case_attachments.items():
if v.is_present:
update_attachments[k] = v
if self._attachments:
attachment_builder = CouchAttachmentsBuilder(self['_attachments'])
else:
attachment_builder = CouchAttachmentsBuilder()
for k, v in attachment_action.attachments.items():
# grab xform_attachments
# copy over attachments from form onto case
update_attachments[k] = v
if v.is_present:
#fetch attachment from xform
attachment_key = v.attachment_key
attach = stream_dict[attachment_key]
attachment_builder.add(name=k,
content=attach,
content_type=v.server_mime)
else:
try:
attachment_builder.remove(k)
except KeyError:
pass
del update_attachments[k]
self._attachments = attachment_builder.to_json()
self.case_attachments = update_attachments
