def do_import(self, meta_type, data, REQUEST=None): """ """ if REQUEST and not self.getParentNode().checkPermissionPublishObjects(): raise Unauthorized errors = [] schema = self.getSite().getSchemaTool().getSchemaForMetatype(meta_type) if schema is None: raise ValueError('Schema for meta-type not found: "%s"' % meta_type) content_type = self.getSite().get_pluggable_item(meta_type) add_object = content_type['add_method'] location_obj = self.getParentNode() # build a list of property names based on the object schema # TODO: extract this loop into a separate function prop_map = {} for widget in schema.listWidgets(): prop_name = widget.prop_name() if widget.multiple_form_values: for subname in widget.multiple_form_values: prop_subname = prop_name + '.' + subname prop_map[widget.title + ' - ' + subname] = { 'column': prop_subname, 'convert': widget.convert_from_user_string, } if isinstance(widget, GeoWidget): for subname in widget.multiple_form_values: self.geo_fields[subname] = prop_name + '.' + subname else: prop_map[widget.title] = { 'column': prop_name, 'convert': widget.convert_from_user_string, } # and now for dynamic properties dynprop_tool = self.getSite().getDynamicPropertiesTool() for dyn_prop in dynprop_tool.getDynamicProperties(meta_type): prop_map[dyn_prop.name] = { 'column': dyn_prop.id, 'convert': lambda x: x, } try: reader = UnicodeReader(data) try: header = reader.next() except StopIteration: msg = 'Invalid CSV file' if REQUEST is None: raise ValueError(msg) else: errors.append(msg) reader = [] record_number = 0 obj_ids = [] for row in reader: try: record_number += 1 # TODO: extract this block into a separate function properties = {} extra_properties = {} for column, value in zip(header, row): if value == '': continue if column not in prop_map: extra_properties[column] = value continue key = prop_map[column]['column'] convert = prop_map[column]['convert'] properties[key] = convert(value) properties = self.do_geocoding(properties) ob_id = add_object(location_obj, _send_notifications=False, **properties) ob = location_obj._getOb(ob_id) if extra_properties: adapter = ICSVImportExtraColumns(ob, None) if adapter is not None: adapter.handle_columns(extra_properties) obj_ids.append(ob.getId()) ob.submitThis() ob.approveThis() except UnicodeDecodeError, e: raise except Exception, e: self.log_current_error() msg = ('Error while importing from CSV, row ${record_number}: ${error}', {'record_number': record_number, 'error': str(e)}) if REQUEST is None: raise ValueError(msg) else: errors.append(msg)
class CSVImportTool(Implicit, Item): title = "Import from structured file" security = ClassSecurityInfo() geo_fields = {} def __init__(self, id): self.id = id security.declareProtected(PERMISSION_PUBLISH_OBJECTS, 'template') def template(self, meta_type, file_type, as_attachment=False, REQUEST=None): """ """ if REQUEST and not self.getParentNode().checkPermissionPublishObjects(): raise Unauthorized schema = self.getSite().getSchemaTool().getSchemaForMetatype(meta_type) if schema is None: raise ValueError('Schema for meta-type "%s" not found' % meta_type) columns = [] for widget in schema.listWidgets(): if widget.multiple_form_values: for subname in widget.multiple_form_values: columns.append(widget.title + ' - ' + subname) else: columns.append(widget.title) if file_type == 'CSV': ret = generate_csv(columns, [[]]) content_type = 'text/csv; charset=utf-8' filename = schema.title_or_id() + ' bulk upload.csv' elif file_type == 'Excel': assert excel_export_available ret = generate_excel(columns, [[]]) content_type = 'application/vnd.ms-excel' filename = schema.title_or_id() + ' bulk upload.xls' else: raise ValueError('unknown file format %r' % file_type) if REQUEST is not None: set_response_attachment(REQUEST.RESPONSE, filename, content_type, len(ret)) return ret security.declareProtected(PERMISSION_PUBLISH_OBJECTS, 'do_geocoding') def do_geocoding(self, properties): lat = properties.get(self.geo_fields['lat'], '') lon = properties.get(self.geo_fields['lon'], '') address = properties.get(self.geo_fields['address'], '') if lat.strip() == '' and lon.strip() == '' and address: coordinates = geocoding.geocode(self.portal_map, address) if coordinates != None: lat, lon = coordinates properties[self.geo_fields['lat']] = lat properties[self.geo_fields['lon']] = lon return properties security.declareProtected(PERMISSION_PUBLISH_OBJECTS, 'do_import') def do_import(self, meta_type, file_type, data, REQUEST=None): """ """ if REQUEST and not self.getParentNode().checkPermissionPublishObjects(): raise Unauthorized errors = [] schema = self.getSite().getSchemaTool().getSchemaForMetatype(meta_type) if schema is None: raise ValueError('Schema for meta-type not found: "%s"' % meta_type) content_type = self.getSite().get_pluggable_item(meta_type) add_object = content_type['add_method'] location_obj = self.getParentNode() # build a list of property names based on the object schema # TODO: extract this loop into a separate function prop_map = {} for widget in schema.listWidgets(): prop_name = widget.prop_name() if widget.multiple_form_values: for subname in widget.multiple_form_values: prop_subname = prop_name + '.' + subname prop_map[widget.title + ' - ' + subname] = { 'column': prop_subname, 'convert': widget.convert_from_user_string, } if isinstance(widget, GeoWidget): for subname in widget.multiple_form_values: self.geo_fields[subname] = prop_name + '.' + subname else: prop_map[widget.title] = { 'column': prop_name, 'convert': widget.convert_from_user_string, } if file_type == 'Excel': assert excel_export_available try: wb = xlrd.open_workbook(file_contents=data.read()) ws = wb.sheets()[0] header = ws.row_values(0) rows = [] for i in range(ws.nrows)[1:]: rows.append(ws.row_values(i)) except xlrd.XLRDError: msg = 'Invalid Excel file' if REQUEST is None: raise ValueError(msg) else: errors.append(msg) rows = [] elif file_type == 'CSV': rows = UnicodeReader(data) try: header = rows.next() except StopIteration: msg = 'Invalid CSV file' if REQUEST is None: raise ValueError(msg) else: errors.append(msg) rows = [] except UnicodeDecodeError, e: if REQUEST is None: raise else: errors.append('CSV file is not utf-8 encoded') else: raise ValueError('unknown file format %r' % file_type) record_number = 0 obj_ids = [] try: for row in rows: try: record_number += 1 #TODO: extract this block into a separate function properties = {} extra_properties = {} for column, value in zip(header, row): if value == '': continue if column not in prop_map: extra_properties[column] = value continue key = prop_map[column]['column'] convert = prop_map[column]['convert'] properties[key] = convert(value) properties = self.do_geocoding(properties) ob_id = add_object(location_obj, _send_notifications=False, **properties) ob = location_obj._getOb(ob_id) if extra_properties: adapter = ICSVImportExtraColumns(ob, None) if adapter is not None: extra_props_messages = adapter.handle_columns( extra_properties) if extra_props_messages: errors.append(extra_props_messages) obj_ids.append(ob.getId()) ob.submitThis() ob.approveThis(_send_notifications=False) except UnicodeDecodeError, e: raise except Exception, e: self.log_current_error() msg = ('Error while importing from file, row ${record_number}: ${error}', {'record_number': record_number, 'error': str(e)}) if REQUEST is None: raise ValueError(msg) else: errors.append(msg)
record_number + 1, # account for header 'error': str(e) }) warnings.append(msg) address = properties.pop(self.geo_fields['address']) ob_id = add_object(location_obj, _send_notifications=False, **properties) ob = location_obj._getOb(ob_id) if address: setattr(ob, self.geo_fields['address'].split('.')[0], Geo(address=address)) #user = self.REQUEST.AUTHENTICATED_USER.getUserName() #notify(NyContentObjectEditEvent(ob, user)) if extra_properties: adapter = ICSVImportExtraColumns(ob, None) if adapter is not None: extra_props_messages = adapter.handle_columns( extra_properties) if extra_props_messages: errors.append(extra_props_messages) obj_ids.append(ob.getId()) ob.submitThis() ob.approveThis(_send_notifications=False) except UnicodeDecodeError, e: raise except Exception, e: self.log_current_error() msg = ( 'Error while importing from file, row ${record_number}: ${error}', {
class CSVImporterTask(Persistent): """ An import task for rows from Excel/CSV files """ payload = None template = None error = None status = 'unfinished' def __init__(self, *args, **kwargs): super(CSVImporterTask, self).__init__(*args, **kwargs) self.warnings = PersistentList() self.errors = PersistentList() def on_failure(self, *args, **kwargs): self.status = 'failed' def on_success(self, *args, **kwargs): self.status = 'finished' def run(self, *args, **kwargs): self.payload = kwargs['payload'] self.template = kwargs['template'] self.rec_id = kwargs['rec_id'] site = getSite() user = getSecurityManager().getUser() acl_users = site.acl_users user = user.__of__(acl_users) user_id = kwargs['user_id'] request = BaseRequest() request.response = Response() request.AUTHENTICATED_USER = user request['PARENTS'] = [site] request['URL'] = kwargs['url'] request.steps = [] request.cookies = {} request.form = {} import_location = site.unrestrictedTraverse(kwargs['import_path']) import_location.REQUEST = request site.REQUEST = request geo_fields = kwargs['geo_fields'] self.prop_map = kwargs['properties'] meta_type = kwargs['meta_type'] header = self.template row = self.payload record_number = self.rec_id content_type = import_location.getSite().get_pluggable_item(meta_type) add_object = content_type['add_method'] properties = {} extra_properties = {} address = None for column, value in zip(header, row): if value == '': continue if column not in self.prop_map: extra_properties[column] = value continue key = self.prop_map[column]['column'] widget = self.prop_map[column]['widget'] widget = widget.__of__(import_location) convert = widget.convert_from_user_string properties[key] = convert(value) try: properties = do_geocoding(geo_fields, properties) except GeocoderServiceError, e: msg = ( 'Warnings: could not find a valid address ' 'for row ${record_number}: ${error}', { 'record_number': record_number + 1, # account for header 'error': str(e) }) self.warnings.append(msg) address = properties.pop(geo_fields['address']) ob_id = add_object(import_location, _send_notifications=False, **properties) ob = import_location._getOb(ob_id) if address: setattr(ob, geo_fields['address'].split('.')[0], Geo(address=address)) notify(NyContentObjectEditEvent(ob, user_id)) if extra_properties: adapter = ICSVImportExtraColumns(ob, None) if adapter is not None: extra_props_messages = adapter.handle_columns(extra_properties) if extra_props_messages: self.errors.append(extra_props_messages) #obj_ids.append(ob.getId()) ob.submitThis() ob.approveThis(_send_notifications=False) del import_location.REQUEST del site.REQUEST