def handleCancel(self, action): api.portal.show_message( message=_("import_message_cancel", # nopep8 default="Import canceled."), request=self.request, type="info") self.request.response.redirect(self.context.absolute_url())
class IImportSchema(form.Schema): """Define fields used on the form.""" #TODO: need to get rid of "Keep existing file" etc. It's confusing # suspect this is the wrong field type import_file = NamedFile( title=_( "import_field_import_file_title", # nopep8 default=u"CSV metadata to import"), description=_( "import_field_import_file_description", # nopep8 default= u"CSV file containing rows for each content to create or update"), required=False) # form.widget('header_mapping', NamedFileFieldWidget) header_mapping = schema.List( title=_(u'Header Mapping'), description=_( u"For Import any matching headers in your CSV will be mapped to " u"these fields. If Header is not found then the data will be ignored." u" For export, the internal fields will be exported to the " u" header names listed."), value_type=DictRow(title=u"tablerow", schema=IMappingRow), missing_value={}, required=False) primary_key = schema.Choice( title=_( "import_field_primary_key_title", # nopep8 default=u"Test if content exists using"), description=_( "import_field_primary_key_description", default= u"Field with unique id to use to check if content already exists. " u"Ignored for export." "Normally 'Short Name' or 'Path'."), source=fields_list, #TODO: should be index not fieldname required=True, default=u"id") object_type = schema.Choice( title=_( "import_field_object_type_title", # nopep8 default=u"If not found"), description=_( "import_field_object_type_description", default= u"If content can't be found then Create, Skip or Stop at that row. " u"For rich media such as Videos, upload first. Ignored for " u"export."), source=if_not_found_list, required=True)
def handleSaveImport(self, action): """Create and handle form button "Save and Import".""" # Extract form field values and errors from HTTP request data, errors = self.extractData() if errors: self.status = self.formErrorsMessage return False import_file = data["import_file"] if not import_file: raise WidgetActionExecutionError('import_file', Invalid(_(u"Please provide a csv file to import"))) return # File upload is not saved in settings file_resource = import_file.data file_name = import_file.filename if not (import_file.contentType.startswith("text/") or \ import_file.contentType.startswith("application/csv")): raise WidgetActionExecutionError('import_file', Invalid(_(u"Please provide a file of type CSV"))) return if import_file.contentType.startswith("application/vnd.ms-excel"): raise WidgetActionExecutionError('import_file', Invalid(_(u"Please convert your Excel file to CSV first"))) return if data["object_type"] in ['__ignore__', '__stop__']: create_new = False object_type = None else: create_new = True object_type = data["object_type"] # list all the dexterity types #dx_types = get_portal_types(self.request) #log.debug(dx_types) # based from the types, display all the fields # fields = get_schema_info(CREATION_TYPE) # log.debug(fields) # blank header or field means we don't want it header_mapping = [d for d in data['header_mapping'] if d['field'] and d['header']] matching_headers = dict([(d['field'],d['header']) for d in header_mapping]) if create_new and not(matching_headers.get('id') or matching_headers.get('title')): raise WidgetActionExecutionError('header_mapping', Invalid(_(u"If creating new content you need either 'Short Name" u" or 'Title' in your data."))) return if not matching_headers: raise WidgetActionExecutionError('header_mapping', Invalid(_(u"You must pick which fields should contain your data"))) return primary_key = data["primary_key"] if primary_key and not matching_headers.get(primary_key): raise WidgetActionExecutionError('primary_key', Invalid(_(u"Must be a field selected in Header Mapping"))) return # based from the matching fields, get all the values. matching_fields = dict([(d['header'],d['field']) for d in header_mapping]) import_metadata = dexterity_import( self.context, file_resource, matching_fields, object_type, create_new, primary_key ) existing_count = import_metadata["existing_count"] new_count = import_metadata["new_count"] ignore_count = import_metadata["ignore_count"] api.portal.show_message( message=_("import_message_csv_info", # nopep8 default=u"""${new_num} items added, ${existing_num} items updated and ${ignore_num} items skipped from ${filename}""", mapping={"new_num": new_count, "existing_num": existing_count, "ignore_num": ignore_count, "filename": file_name}), request=self.request, type="info") self.import_metadata = import_metadata # Save our sucessful settings to save time next import annotations = IAnnotations(self.context) settings = annotations.setdefault(KEY, {}) settings['header_list'] = [d['header'] for d in header_mapping] # we will keep making this bigger in case they switch between several CSVs settings.setdefault("matching_fields",{}).update(matching_fields) settings['primary_key'] = primary_key settings['object_type'] = object_type return True
def handleSaveImport(self, action): """Create and handle form button "Save and Import".""" # Extract form field values and errors from HTTP request data, errors = self.extractData() if errors: self.status = self.formErrorsMessage return False import_file = data["import_file"] if not import_file: raise WidgetActionExecutionError( 'import_file', Invalid(_(u"Please provide a csv file to import"))) return # File upload is not saved in settings file_resource = import_file.data file_name = import_file.filename if not (import_file.contentType.startswith("text/") or \ import_file.contentType.startswith("application/csv")): raise WidgetActionExecutionError( 'import_file', Invalid(_(u"Please provide a file of type CSV"))) return if import_file.contentType.startswith("application/vnd.ms-excel"): raise WidgetActionExecutionError( 'import_file', Invalid(_(u"Please convert your Excel file to CSV first"))) return if data["object_type"] in ['__ignore__', '__stop__']: create_new = False object_type = None else: create_new = True object_type = data["object_type"] # list all the dexterity types #dx_types = get_portal_types(self.request) #log.debug(dx_types) # based from the types, display all the fields # fields = get_schema_info(CREATION_TYPE) # log.debug(fields) # blank header or field means we don't want it header_mapping = [ d for d in data['header_mapping'] if d['field'] and d['header'] ] matching_headers = dict([(d['field'], d['header']) for d in header_mapping]) if create_new and not (matching_headers.get('id') or matching_headers.get('title')): raise WidgetActionExecutionError( 'header_mapping', Invalid( _(u"If creating new content you need either 'Short Name" u" or 'Title' in your data."))) return if not matching_headers: raise WidgetActionExecutionError( 'header_mapping', Invalid( _(u"You must pick which fields should contain your data"))) return primary_key = data["primary_key"] if primary_key and not matching_headers.get(primary_key): raise WidgetActionExecutionError( 'primary_key', Invalid(_(u"Must be a field selected in Header Mapping"))) return # based from the matching fields, get all the values. matching_fields = dict([(d['header'], d['field']) for d in header_mapping]) import_metadata = dexterity_import(self.context, file_resource, matching_fields, object_type, create_new, primary_key) existing_count = import_metadata["existing_count"] new_count = import_metadata["new_count"] ignore_count = import_metadata["ignore_count"] api.portal.show_message( message=_( "import_message_csv_info", # nopep8 default=u"""${new_num} items added, ${existing_num} items updated and ${ignore_num} items skipped from ${filename}""", mapping={ "new_num": new_count, "existing_num": existing_count, "ignore_num": ignore_count, "filename": file_name }), request=self.request, type="info") self.import_metadata = import_metadata # Save our sucessful settings to save time next import annotations = IAnnotations(self.context) settings = annotations.setdefault(KEY, {}) settings['header_list'] = [d['header'] for d in header_mapping] # we will keep making this bigger in case they switch between several CSVs settings.setdefault("matching_fields", {}).update(matching_fields) settings['primary_key'] = primary_key settings['object_type'] = object_type return True
class ImportForm(form.SchemaForm): """Import data to dexterity-types.""" # Which plone.directives.form.Schema subclass is used to define # fields for this form schema = IImportSchema ignoreContext = False # Form label label = _( "import_form_label", # nopep8 default=u"CSV Import/Export") description = _( "import_form_description", # nopep8 default=u"Create or Update content from a CSV. " u"For images, files, videos or html documents, use Upload first and use " u"CSV import to set the metadata of the uploaded files.") def getContent(self): """ """ # Create a temporary object holding the settings values out of the patient class TemporarySettingsContext(object): implements(IImportSchema) obj = TemporarySettingsContext() annotations = IAnnotations(self.context) settings = annotations.get(KEY) if settings: obj.primary_key = settings['primary_key'] obj.object_type = settings['object_type'] obj.header_mapping = self.headersFromRequest() return obj def headersFromRequest(self): rows = [] request = self.request context = self.context # try and load it from settings settings = IAnnotations(getContext(context)).get(KEY, {}) if not settings: #TODO: should look in the parent? pass header_list = settings.get('header_list', []) matching_fields = settings.get('matching_fields', {}) if request.get('csv_header'): reader = csv.DictReader(request.get('csv_header').splitlines(), delimiter=",", dialect="excel", quotechar='"') header_list = reader.fieldnames fields = fields_list(None) field_names = {} for field in fields: field_names[field.title.lower()] = field.value field_names[field.value.lower()] = field.value for col in header_list: col = unicode(col.strip()) if not col: continue if col in matching_fields: matched_field = matching_fields[col] elif col.lower() in field_names: matched_field = field_names[col.lower()] else: matched_field = "" rows.append(dict(header=col, field=matched_field)) return rows def updateWidgets(self): self.fields['header_mapping'].widgetFactory = DataGridFieldFactory # get around a bug. not sure whose fault it is. # seems likely is the datagrid field self.fields['header_mapping'].field.bind(self.context) super(ImportForm, self).updateWidgets() def getObjectsToExport(self): # Extract form field values and errors from HTTP request data, errors = self.extractData() if errors: return False container = self.context container_path = "/".join(container.getPhysicalPath()) #TODO: should we allow more criteria? or at least filter by type? query = dict(path={ "query": container_path, "depth": 1 }, # portal_type=object_type, ) # query[primary_key]=key_arg[primary_key] # blank header or field means we don't want it header_mapping = [ d for d in data['header_mapping'] if d['field'] and d['header'] ] catalog = api.portal.get_tool("portal_catalog") results = catalog(**query) return header_mapping, results @button.buttonAndHandler( _( "import_button_save_import", # nopep8 default=u"CSV Import")) def handleSaveImport(self, action): """Create and handle form button "Save and Import".""" # Extract form field values and errors from HTTP request data, errors = self.extractData() if errors: self.status = self.formErrorsMessage return False import_file = data["import_file"] if not import_file: raise WidgetActionExecutionError( 'import_file', Invalid(_(u"Please provide a csv file to import"))) return # File upload is not saved in settings file_resource = import_file.data file_name = import_file.filename if not (import_file.contentType.startswith("text/") or \ import_file.contentType.startswith("application/csv")): raise WidgetActionExecutionError( 'import_file', Invalid(_(u"Please provide a file of type CSV"))) return if import_file.contentType.startswith("application/vnd.ms-excel"): raise WidgetActionExecutionError( 'import_file', Invalid(_(u"Please convert your Excel file to CSV first"))) return if data["object_type"] in ['__ignore__', '__stop__']: create_new = False object_type = None else: create_new = True object_type = data["object_type"] # list all the dexterity types #dx_types = get_portal_types(self.request) #log.debug(dx_types) # based from the types, display all the fields # fields = get_schema_info(CREATION_TYPE) # log.debug(fields) # blank header or field means we don't want it header_mapping = [ d for d in data['header_mapping'] if d['field'] and d['header'] ] matching_headers = dict([(d['field'], d['header']) for d in header_mapping]) if create_new and not (matching_headers.get('id') or matching_headers.get('title')): raise WidgetActionExecutionError( 'header_mapping', Invalid( _(u"If creating new content you need either 'Short Name" u" or 'Title' in your data."))) return if not matching_headers: raise WidgetActionExecutionError( 'header_mapping', Invalid( _(u"You must pick which fields should contain your data"))) return primary_key = data["primary_key"] if primary_key and not matching_headers.get(primary_key): raise WidgetActionExecutionError( 'primary_key', Invalid(_(u"Must be a field selected in Header Mapping"))) return # based from the matching fields, get all the values. matching_fields = dict([(d['header'], d['field']) for d in header_mapping]) import_metadata = dexterity_import(self.context, file_resource, matching_fields, object_type, create_new, primary_key) existing_count = import_metadata["existing_count"] new_count = import_metadata["new_count"] ignore_count = import_metadata["ignore_count"] api.portal.show_message( message=_( "import_message_csv_info", # nopep8 default=u"""${new_num} items added, ${existing_num} items updated and ${ignore_num} items skipped from ${filename}""", mapping={ "new_num": new_count, "existing_num": existing_count, "ignore_num": ignore_count, "filename": file_name }), request=self.request, type="info") self.import_metadata = import_metadata # Save our sucessful settings to save time next import annotations = IAnnotations(self.context) settings = annotations.setdefault(KEY, {}) settings['header_list'] = [d['header'] for d in header_mapping] # we will keep making this bigger in case they switch between several CSVs settings.setdefault("matching_fields", {}).update(matching_fields) settings['primary_key'] = primary_key settings['object_type'] = object_type return True #TODO: replace with a report element on import that gives an extra button # to "Download CSV of changes". Requires we have hidden field of all changed # UIDs and then relookup all of those. # Report element can also display list of top 20 creations, top 20 updates etc # replaces info window @button.buttonAndHandler( _( "import___button_import_export", # nopep8 default=u"Import and Export Changes")) def handleImportExport(self, action): if not self.handleSaveImport(self, action): return False data, errors = self.extractData() if errors: return False # blank header or field means we don't want it header_mapping = [ d for d in data['header_mapping'] if d['field'] and d['header'] ] # export to csv file # import pdb; pdb.set_trace() if self.import_metadata["report"]: normalizer = getUtility(IIDNormalizer) random_id = normalizer.normalize(time.time()) filename = "export_{0}.{1}".format(random_id, 'csv') dataset = export_file(self.import_metadata["report"], header_mapping, self.request) file = StringIO.StringIO() file.write(dataset.csv) attachment = file.getvalue() self.request.response.setHeader('content-type', 'text/csv') self.request.response.setHeader( 'Content-Disposition', 'attachment; filename="%s"' % filename) self.request.response.setBody(attachment, lock=True) file.close() #self.request.response.redirect(self.context.absolute_url()) @button.buttonAndHandler( _( "import___button_export_csv", # nopep8 default=u"CSV Export")) def handleExportCSV(self, action): [header_mapping, results] = self.getObjectsToExport() normalizer = getUtility(IIDNormalizer) random_id = normalizer.normalize(time.time()) filename = "export_{0}.{1}".format(random_id, 'csv') dataset = export_file(results, header_mapping, self.request) #log.debug(filename) #log.debug(attachment) self.request.response.setHeader('content-type', 'text/csv') self.request.response.setHeader('Content-Disposition', 'attachment; filename="%s"' % filename) self.request.response.setBody(dataset.csv, lock=True) return True @button.buttonAndHandler( _( "import___button_export_xlsx", # nopep8 default=u"XLSX Export")) def handleExportXLSX(self, action): [header_mapping, results] = self.getObjectsToExport() normalizer = getUtility(IIDNormalizer) random_id = normalizer.normalize(time.time()) filename = "export_{0}.{1}".format(random_id, 'xlsx') dataset = export_file(results, header_mapping, self.request) #log.debug(filename) #log.debug(attachment) self.request.response.setHeader( 'content-type', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' ) self.request.response.setHeader('Content-Disposition', 'attachment; filename="%s"' % filename) self.request.response.setBody(dataset.xlsx, lock=True) return True @button.buttonAndHandler(u"Cancel") def handleCancel(self, action): api.portal.show_message( message=_( "import_message_cancel", # nopep8 default="Import canceled."), request=self.request, type="info") self.request.response.redirect(self.context.absolute_url())