def handle(self, *args, **options): ids = get_form_ids_by_type('ipm-senegal', 'XFormInstance') to_save = [] for doc in iter_docs(XFormInstance.get_db(), ids): try: if 'location_id' in doc[ 'form'] and not doc['form']['location_id']: case = SupplyPointCase.get(doc['form']['case']['@case_id']) if case.type == 'supply-point': instance = XFormInstance.get(doc['_id']) # fix the XFormInstance instance.form['location_id'] = case.location_id # fix the actual form.xml xml_object = etree.fromstring(instance.get_xml()) location_id_node = xml_object.find( re.sub('}.*', '}location_id', xml_object.tag)) location_id_node.text = case.location_id updated_xml = etree.tostring(xml_object) attachment_builder = CouchAttachmentsBuilder( instance._attachments) attachment_builder.add( name='form.xml', content=updated_xml, content_type=instance._attachments['form.xml'] ['content_type']) instance._attachments = attachment_builder.to_json() print 'Updating XFormInstance:', doc['_id'] to_save.append(instance)
def store_attachments(cls, xform, attachments): builder = CouchAttachmentsBuilder() for attachment in attachments: builder.add( content=attachment.content, name=attachment.name, content_type=attachment.content_type, ) xform._attachments = builder.to_json()
def from_instance(cls, instance, message): """ Create an instance of this record from a submission body """ attachments_builder = CouchAttachmentsBuilder() attachments_builder.add(content=instance, name=ATTACHMENT_NAME, content_type="text/xml") return SubmissionErrorLog( received_on=datetime.datetime.utcnow(), md5=hashlib.md5(instance).hexdigest(), problem=message, _attachments=attachments_builder.to_json(), )
def handle(self, *args, **options): ids = get_form_ids_by_type('ipm-senegal', 'XFormInstance') to_save = [] locations = SQLLocation.objects.filter(domain='ipm-senegal').values_list('location_id', 'name') locations_map = {location_id: name for (location_id, name) in locations} for doc in iter_docs(XFormInstance.get_db(), ids): try: if 'PPS_name' in doc['form'] and not doc['form']['PPS_name']: case = SupplyPointCase.get(doc['form']['case']['@case_id']) if case.type == 'supply-point': print 'Updating XFormInstance:', doc['_id'] pps_name = locations_map[case.location_id] instance = XFormInstance.get(doc['_id']) # fix the XFormInstance instance.form['PPS_name'] = pps_name for instance_prod in instance.form['products']: instance_prod['PPS_name'] = instance_prod['PPS_name'] or pps_name # fix the actual form.xml xml_object = etree.fromstring(instance.get_xml()) pps_name_node = xml_object.find(re.sub('}.*', '}PPS_name', xml_object.tag)) pps_name_node.text = pps_name products_nodes = xml_object.findall(re.sub('}.*', '}products', xml_object.tag)) for product_node in products_nodes: product_pps_name_node = product_node.find(re.sub('}.*', '}PPS_name', xml_object.tag)) product_pps_name_node.text = pps_name updated_xml = etree.tostring(xml_object) attachment_builder = CouchAttachmentsBuilder(instance._attachments) attachment_builder.add( name='form.xml', content=updated_xml, content_type=instance._attachments['form.xml']['content_type'] ) instance._attachments = attachment_builder.to_json() to_save.append(instance) except Exception: print 'Failed to save XFormInstance:', doc['_id'] if len(to_save) > 500: XFormInstance.get_db().bulk_save(to_save) to_save = [] if to_save: XFormInstance.get_db().bulk_save(to_save)
def handle(self, *args, **options): ids = get_form_ids_by_type('ipm-senegal', 'XFormInstance') to_save = [] locations = SQLLocation.objects.filter( domain='ipm-senegal').values_list('location_id', 'name') locations_map = { location_id: name for (location_id, name) in locations } for doc in iter_docs(XFormInstance.get_db(), ids): try: if 'PPS_name' in doc['form'] and not doc['form']['PPS_name']: case = SupplyPointCase.get(doc['form']['case']['@case_id']) if case.type == 'supply-point': print 'Updating XFormInstance:', doc['_id'] pps_name = locations_map[case.location_id] instance = XFormInstance.get(doc['_id']) # fix the XFormInstance instance.form['PPS_name'] = pps_name for instance_prod in instance.form['products']: instance_prod['PPS_name'] = instance_prod[ 'PPS_name'] or pps_name # fix the actual form.xml xml_object = etree.fromstring(instance.get_xml()) pps_name_node = xml_object.find( re.sub('}.*', '}PPS_name', xml_object.tag)) pps_name_node.text = pps_name products_nodes = xml_object.findall( re.sub('}.*', '}products', xml_object.tag)) for product_node in products_nodes: product_pps_name_node = product_node.find( re.sub('}.*', '}PPS_name', xml_object.tag)) product_pps_name_node.text = pps_name updated_xml = etree.tostring(xml_object) attachment_builder = CouchAttachmentsBuilder( instance._attachments) attachment_builder.add( name='form.xml', content=updated_xml, content_type=instance._attachments['form.xml'] ['content_type']) instance._attachments = attachment_builder.to_json() to_save.append(instance)
def handle(self, *args, **options): startkey = ['ipm-senegal', 'by_type', 'XFormInstance'] endkey = startkey + [{}] ids = [row['id'] for row in XFormInstance.get_db().view( "couchforms/all_submissions_by_domain", startkey=startkey, endkey=endkey, reduce=False )] to_save = [] for doc in iter_docs(XFormInstance.get_db(), ids): try: if 'location_id' in doc['form'] and not doc['form']['location_id']: case = SupplyPointCase.get(doc['form']['case']['@case_id']) if case.type == 'supply-point': instance = XFormInstance.get(doc['_id']) # fix the XFormInstance instance.form['location_id'] = case.location_id # fix the actual form.xml xml_object = etree.fromstring(instance.get_xml()) location_id_node = xml_object.find(re.sub('}.*', '}location_id', xml_object.tag)) location_id_node.text = case.location_id updated_xml = etree.tostring(xml_object) attachment_builder = CouchAttachmentsBuilder(instance._attachments) attachment_builder.add( name='form.xml', content=updated_xml, content_type=instance._attachments['form.xml']['content_type'] ) instance._attachments = attachment_builder.to_json() print 'Updating XFormInstance:', doc['_id'] to_save.append(instance) except Exception: print 'Failed to save XFormInstance:', doc['_id'] if len(to_save) > 500: XFormInstance.get_db().bulk_save(to_save) to_save = [] if to_save: XFormInstance.get_db().bulk_save(to_save)
def _apply_attachments_action(self, attachment_action, xform=None): """ if xform is provided it will be used to fetch attachments """ # the actions and attachment must be added before the first saves can happen # todo attach cached attachment info def fetch_attachment(name): if fetch_attachment.form is None: fetch_attachment.form = XFormInstance.get(attachment_action.xform_id) return fetch_attachment.form.fetch_attachment(name) fetch_attachment.form = xform attach_dict = {} # cache all attachment streams from xform for k, v in attachment_action.attachments.items(): if v.is_present: # fetch attachment, update metadata, get the stream attach_data = fetch_attachment(v.attachment_src) attach_dict[k] = attach_data v.attachment_size = len(attach_data) if v.is_image: img = Image.open(StringIO(attach_data)) img_size = img.size props = dict(width=img_size[0], height=img_size[1]) v.attachment_properties = props update_attachments = {} for k, v in self.case.case_attachments.items(): if v.is_present: update_attachments[k] = v if self.case._attachments: attachment_builder = CouchAttachmentsBuilder( self.case['_attachments']) else: attachment_builder = CouchAttachmentsBuilder() for k, v in attachment_action.attachments.items(): # grab xform_attachments # copy over attachments from form onto case update_attachments[k] = v if v.is_present: #fetch attachment from xform identifier = v.identifier attach = attach_dict[identifier] attachment_builder.add(name=k, content=attach, content_type=v.server_mime) else: try: attachment_builder.remove(k) except KeyError: pass del update_attachments[k] self.case._attachments = attachment_builder.to_json() self.case.case_attachments = update_attachments
def create_xform(xml_string, attachments=None, _id=None, process=None): """ create but do not save an XFormInstance from an xform payload (xml_string) optionally set the doc _id to a predefined value (_id) return doc _id of the created doc `process` is transformation to apply to the form right before saving This is to avoid having to save multiple times If xml_string is bad xml - raise couchforms.XMLSyntaxError """ from corehq.util.couch_helpers import CouchAttachmentsBuilder assert attachments is not None json_form = convert_xform_to_json(xml_string) adjust_datetimes(json_form) _id = (_id or _extract_meta_instance_id(json_form) or XFormInstance.get_db().server.next_uuid()) assert _id attachments_builder = CouchAttachmentsBuilder() attachments_builder.add( content=xml_string, name='form.xml', content_type='text/xml', ) for key, value in attachments.items(): attachments_builder.add( content=value, name=key, content_type=value.content_type, ) xform = XFormInstance( # form has to be wrapped {'form': json_form}, # other properties can be set post-wrap _id=_id, xmlns=json_form.get('@xmlns'), _attachments=attachments_builder.to_json(), received_on=datetime.datetime.utcnow(), ) # this had better not fail, don't think it ever has # if it does, nothing's saved and we get a 500 if process: process(xform) lock = acquire_lock_for_xform(_id) with ReleaseOnError(lock): if _id in XFormInstance.get_db(): raise DuplicateError() return LockManager(xform, lock)
def _apply_attachments_action(self, attachment_action, xform=None): """ if xform is provided it will be used to fetch attachments """ # the actions and attachment must be added before the first saves can happen # todo attach cached attachment info def fetch_attachment(name): if fetch_attachment.form is None: fetch_attachment.form = XFormInstance.get(attachment_action.xform_id) return fetch_attachment.form.fetch_attachment(name) fetch_attachment.form = xform attach_dict = {} # cache all attachment streams from xform for k, v in attachment_action.attachments.items(): if v.is_present: # fetch attachment, update metadata, get the stream attach_data = fetch_attachment(v.attachment_src) attach_dict[k] = attach_data v.attachment_size = len(attach_data) if v.is_image: img = Image.open(StringIO(attach_data)) img_size = img.size props = dict(width=img_size[0], height=img_size[1]) v.attachment_properties = props update_attachments = {} for k, v in self.case.case_attachments.items(): if v.is_present: update_attachments[k] = v if self.case._attachments: attachment_builder = CouchAttachmentsBuilder( self.case['_attachments']) else: attachment_builder = CouchAttachmentsBuilder() for k, v in attachment_action.attachments.items(): # grab xform_attachments # copy over attachments from form onto case update_attachments[k] = v if v.is_present: #fetch attachment from xform identifier = v.identifier attach = attach_dict[identifier] attachment_builder.add(name=k, content=attach, content_type=v.server_mime) else: try: attachment_builder.remove(k) except KeyError: pass del update_attachments[k] self.case._attachments = attachment_builder.to_json() self.case.case_attachments = update_attachments
def create_xform(xml_string, attachments=None, _id=None, process=None): """ create but do not save an XFormInstance from an xform payload (xml_string) optionally set the doc _id to a predefined value (_id) return doc _id of the created doc `process` is transformation to apply to the form right before saving This is to avoid having to save multiple times If xml_string is bad xml - raise couchforms.XMLSyntaxError """ from corehq.util.couch_helpers import CouchAttachmentsBuilder assert attachments is not None json_form = convert_xform_to_json(xml_string) adjust_datetimes(json_form) _id = (_id or _extract_meta_instance_id(json_form) or XFormInstance.get_db().server.next_uuid()) assert _id attachments_builder = CouchAttachmentsBuilder() attachments_builder.add( content=xml_string, name='form.xml', content_type='text/xml', ) for key, value in attachments.items(): attachments_builder.add( content=value, name=key, content_type=value.content_type, ) xform = XFormInstance( # form has to be wrapped {'form': json_form}, # other properties can be set post-wrap _id=_id, xmlns=json_form.get('@xmlns'), _attachments=attachments_builder.to_json(), received_on=datetime.datetime.utcnow(), ) # this had better not fail, don't think it ever has # if it does, nothing's saved and we get a 500 if process: process(xform) lock = acquire_lock_for_xform(_id) with ReleaseOnError(lock): if _id in XFormInstance.get_db(): raise DuplicateError() return LockManager(xform, lock)
def _apply_attachments_action(self, attachment_action, xform=None): """ if xform is provided, attachments will be looked for in the xform's _attachments. They should be base64 encoded under _attachments[name]['data'] """ # the actions and _attachment must be added before the first saves can happen # todo attach cached attachment info def fetch_attachment(name): if xform and 'data' in xform._attachments[name]: assert xform.form_id == attachment_action.xform_id return base64.b64decode(xform._attachments[name]['data']) else: return XFormInstance.get_db().fetch_attachment(attachment_action.xform_id, name) stream_dict = {} # cache all attachment streams from xform for k, v in attachment_action.attachments.items(): if v.is_present: # fetch attachment, update metadata, get the stream attach_data = fetch_attachment(v.attachment_src) stream_dict[k] = attach_data v.attachment_size = len(attach_data) if v.is_image: img = Image.open(StringIO(attach_data)) img_size = img.size props = dict(width=img_size[0], height=img_size[1]) v.attachment_properties = props update_attachments = {} for k, v in self.case.case_attachments.items(): if v.is_present: update_attachments[k] = v if self.case._attachments: attachment_builder = CouchAttachmentsBuilder( self.case['_attachments']) else: attachment_builder = CouchAttachmentsBuilder() for k, v in attachment_action.attachments.items(): # grab xform_attachments # copy over attachments from form onto case update_attachments[k] = v if v.is_present: #fetch attachment from xform attachment_key = v.attachment_key attach = stream_dict[attachment_key] attachment_builder.add(name=k, content=attach, content_type=v.server_mime) else: try: attachment_builder.remove(k) except KeyError: pass del update_attachments[k] self.case._attachments = attachment_builder.to_json() self.case.case_attachments = update_attachments
def apply_attachments(self, attachment_action, xform=None): """ if xform is provided, attachments will be looked for in the xform's _attachments. They should be base64 encoded under _attachments[name]['data'] """ # the actions and _attachment must be added before the first saves can happen # todo attach cached attachment info def fetch_attachment(name): if xform and 'data' in xform._attachments[name]: assert xform._id == attachment_action.xform_id return base64.b64decode(xform._attachments[name]['data']) else: return XFormInstance.get_db().fetch_attachment( attachment_action.xform_id, name) stream_dict = {} # cache all attachment streams from xform for k, v in attachment_action.attachments.items(): if v.is_present: # fetch attachment, update metadata, get the stream attach_data = fetch_attachment(v.attachment_src) stream_dict[k] = attach_data v.attachment_size = len(attach_data) if v.is_image: img = Image.open(StringIO(attach_data)) img_size = img.size props = dict(width=img_size[0], height=img_size[1]) v.attachment_properties = props update_attachments = {} for k, v in self.case_attachments.items(): if v.is_present: update_attachments[k] = v if self._attachments: attachment_builder = CouchAttachmentsBuilder(self['_attachments']) else: attachment_builder = CouchAttachmentsBuilder() for k, v in attachment_action.attachments.items(): # grab xform_attachments # copy over attachments from form onto case update_attachments[k] = v if v.is_present: #fetch attachment from xform attachment_key = v.attachment_key attach = stream_dict[attachment_key] attachment_builder.add(name=k, content=attach, content_type=v.server_mime) else: try: attachment_builder.remove(k) except KeyError: pass del update_attachments[k] self._attachments = attachment_builder.to_json() self.case_attachments = update_attachments