def do_import(self, meta_type, data, REQUEST=None):
""" """
if REQUEST and not self.getParentNode().checkPermissionPublishObjects():
raise Unauthorized
errors = []
schema = self.getSite().getSchemaTool().getSchemaForMetatype(meta_type)
if schema is None:
raise ValueError('Schema for meta-type not found: "%s"' % meta_type)
content_type = self.getSite().get_pluggable_item(meta_type)
add_object = content_type['add_method']
location_obj = self.getParentNode()
# build a list of property names based on the object schema
# TODO: extract this loop into a separate function
prop_map = {}
for widget in schema.listWidgets():
prop_name = widget.prop_name()
if widget.multiple_form_values:
for subname in widget.multiple_form_values:
prop_subname = prop_name + '.' + subname
prop_map[widget.title + ' - ' + subname] = {
'column': prop_subname,
'convert': widget.convert_from_user_string,
}
if isinstance(widget, GeoWidget):
for subname in widget.multiple_form_values:
self.geo_fields[subname] = prop_name + '.' + subname
else:
prop_map[widget.title] = {
'column': prop_name,
'convert': widget.convert_from_user_string,
}
# and now for dynamic properties
dynprop_tool = self.getSite().getDynamicPropertiesTool()
for dyn_prop in dynprop_tool.getDynamicProperties(meta_type):
prop_map[dyn_prop.name] = {
'column': dyn_prop.id,
'convert': lambda x: x,
}
try:
reader = UnicodeReader(data)
try:
header = reader.next()
except StopIteration:
msg = 'Invalid CSV file'
if REQUEST is None:
raise ValueError(msg)
else:
errors.append(msg)
reader = []
record_number = 0
obj_ids = []
for row in reader:
try:
record_number += 1
# TODO: extract this block into a separate function
properties = {}
extra_properties = {}
for column, value in zip(header, row):
if value == '':
continue
if column not in prop_map:
extra_properties[column] = value
continue
key = prop_map[column]['column']
convert = prop_map[column]['convert']
properties[key] = convert(value)
properties = self.do_geocoding(properties)
ob_id = add_object(location_obj, _send_notifications=False, **properties)
ob = location_obj._getOb(ob_id)
if extra_properties:
adapter = ICSVImportExtraColumns(ob, None)
if adapter is not None:
adapter.handle_columns(extra_properties)
obj_ids.append(ob.getId())
ob.submitThis()
ob.approveThis()
except UnicodeDecodeError, e:
raise
except Exception, e:
self.log_current_error()
msg = ('Error while importing from CSV, row ${record_number}: ${error}',
{'record_number': record_number, 'error': str(e)})
if REQUEST is None:
raise ValueError(msg)
else:
errors.append(msg)
class CSVImportTool(Implicit, Item):
title = "Import from structured file"
security = ClassSecurityInfo()
geo_fields = {}
def __init__(self, id):
self.id = id
security.declareProtected(PERMISSION_PUBLISH_OBJECTS, 'template')
def template(self, meta_type, file_type, as_attachment=False, REQUEST=None):
""" """
if REQUEST and not self.getParentNode().checkPermissionPublishObjects():
raise Unauthorized
schema = self.getSite().getSchemaTool().getSchemaForMetatype(meta_type)
if schema is None:
raise ValueError('Schema for meta-type "%s" not found' % meta_type)
columns = []
for widget in schema.listWidgets():
if widget.multiple_form_values:
for subname in widget.multiple_form_values:
columns.append(widget.title + ' - ' + subname)
else:
columns.append(widget.title)
if file_type == 'CSV':
ret = generate_csv(columns, [[]])
content_type = 'text/csv; charset=utf-8'
filename = schema.title_or_id() + ' bulk upload.csv'
elif file_type == 'Excel':
assert excel_export_available
ret = generate_excel(columns, [[]])
content_type = 'application/vnd.ms-excel'
filename = schema.title_or_id() + ' bulk upload.xls'
else: raise ValueError('unknown file format %r' % file_type)
if REQUEST is not None:
set_response_attachment(REQUEST.RESPONSE, filename,
content_type, len(ret))
return ret
security.declareProtected(PERMISSION_PUBLISH_OBJECTS, 'do_geocoding')
def do_geocoding(self, properties):
lat = properties.get(self.geo_fields['lat'], '')
lon = properties.get(self.geo_fields['lon'], '')
address = properties.get(self.geo_fields['address'], '')
if lat.strip() == '' and lon.strip() == '' and address:
coordinates = geocoding.geocode(self.portal_map, address)
if coordinates != None:
lat, lon = coordinates
properties[self.geo_fields['lat']] = lat
properties[self.geo_fields['lon']] = lon
return properties
security.declareProtected(PERMISSION_PUBLISH_OBJECTS, 'do_import')
def do_import(self, meta_type, file_type, data, REQUEST=None):
""" """
if REQUEST and not self.getParentNode().checkPermissionPublishObjects():
raise Unauthorized
errors = []
schema = self.getSite().getSchemaTool().getSchemaForMetatype(meta_type)
if schema is None:
raise ValueError('Schema for meta-type not found: "%s"' % meta_type)
content_type = self.getSite().get_pluggable_item(meta_type)
add_object = content_type['add_method']
location_obj = self.getParentNode()
# build a list of property names based on the object schema
# TODO: extract this loop into a separate function
prop_map = {}
for widget in schema.listWidgets():
prop_name = widget.prop_name()
if widget.multiple_form_values:
for subname in widget.multiple_form_values:
prop_subname = prop_name + '.' + subname
prop_map[widget.title + ' - ' + subname] = {
'column': prop_subname,
'convert': widget.convert_from_user_string,
}
if isinstance(widget, GeoWidget):
for subname in widget.multiple_form_values:
self.geo_fields[subname] = prop_name + '.' + subname
else:
prop_map[widget.title] = {
'column': prop_name,
'convert': widget.convert_from_user_string,
}
if file_type == 'Excel':
assert excel_export_available
try:
wb = xlrd.open_workbook(file_contents=data.read())
ws = wb.sheets()[0]
header = ws.row_values(0)
rows = []
for i in range(ws.nrows)[1:]:
rows.append(ws.row_values(i))
except xlrd.XLRDError:
msg = 'Invalid Excel file'
if REQUEST is None:
raise ValueError(msg)
else:
errors.append(msg)
rows = []
elif file_type == 'CSV':
rows = UnicodeReader(data)
try:
header = rows.next()
except StopIteration:
msg = 'Invalid CSV file'
if REQUEST is None:
raise ValueError(msg)
else:
errors.append(msg)
rows = []
except UnicodeDecodeError, e:
if REQUEST is None:
raise
else:
errors.append('CSV file is not utf-8 encoded')
else: raise ValueError('unknown file format %r' % file_type)
record_number = 0
obj_ids = []
try:
for row in rows:
try:
record_number += 1
#TODO: extract this block into a separate function
properties = {}
extra_properties = {}
for column, value in zip(header, row):
if value == '':
continue
if column not in prop_map:
extra_properties[column] = value
continue
key = prop_map[column]['column']
convert = prop_map[column]['convert']
properties[key] = convert(value)
properties = self.do_geocoding(properties)
ob_id = add_object(location_obj, _send_notifications=False,
**properties)
ob = location_obj._getOb(ob_id)
if extra_properties:
adapter = ICSVImportExtraColumns(ob, None)
if adapter is not None:
extra_props_messages = adapter.handle_columns(
extra_properties)
if extra_props_messages:
errors.append(extra_props_messages)
obj_ids.append(ob.getId())
ob.submitThis()
ob.approveThis(_send_notifications=False)
except UnicodeDecodeError, e:
raise
except Exception, e:
self.log_current_error()
msg = ('Error while importing from file, row ${record_number}: ${error}',
{'record_number': record_number, 'error': str(e)})
if REQUEST is None:
raise ValueError(msg)
else:
errors.append(msg)
record_number + 1, # account for header
'error': str(e)
})
warnings.append(msg)
address = properties.pop(self.geo_fields['address'])
ob_id = add_object(location_obj,
_send_notifications=False,
**properties)
ob = location_obj._getOb(ob_id)
if address:
setattr(ob, self.geo_fields['address'].split('.')[0],
Geo(address=address))
#user = self.REQUEST.AUTHENTICATED_USER.getUserName()
#notify(NyContentObjectEditEvent(ob, user))
if extra_properties:
adapter = ICSVImportExtraColumns(ob, None)
if adapter is not None:
extra_props_messages = adapter.handle_columns(
extra_properties)
if extra_props_messages:
errors.append(extra_props_messages)
obj_ids.append(ob.getId())
ob.submitThis()
ob.approveThis(_send_notifications=False)
except UnicodeDecodeError, e:
raise
except Exception, e:
self.log_current_error()
msg = (
'Error while importing from file, row ${record_number}: ${error}',
{
class CSVImporterTask(Persistent):
""" An import task for rows from Excel/CSV files
"""
payload = None
template = None
error = None
status = 'unfinished'
def __init__(self, *args, **kwargs):
super(CSVImporterTask, self).__init__(*args, **kwargs)
self.warnings = PersistentList()
self.errors = PersistentList()
def on_failure(self, *args, **kwargs):
self.status = 'failed'
def on_success(self, *args, **kwargs):
self.status = 'finished'
def run(self, *args, **kwargs):
self.payload = kwargs['payload']
self.template = kwargs['template']
self.rec_id = kwargs['rec_id']
site = getSite()
user = getSecurityManager().getUser()
acl_users = site.acl_users
user = user.__of__(acl_users)
user_id = kwargs['user_id']
request = BaseRequest()
request.response = Response()
request.AUTHENTICATED_USER = user
request['PARENTS'] = [site]
request['URL'] = kwargs['url']
request.steps = []
request.cookies = {}
request.form = {}
import_location = site.unrestrictedTraverse(kwargs['import_path'])
import_location.REQUEST = request
site.REQUEST = request
geo_fields = kwargs['geo_fields']
self.prop_map = kwargs['properties']
meta_type = kwargs['meta_type']
header = self.template
row = self.payload
record_number = self.rec_id
content_type = import_location.getSite().get_pluggable_item(meta_type)
add_object = content_type['add_method']
properties = {}
extra_properties = {}
address = None
for column, value in zip(header, row):
if value == '':
continue
if column not in self.prop_map:
extra_properties[column] = value
continue
key = self.prop_map[column]['column']
widget = self.prop_map[column]['widget']
widget = widget.__of__(import_location)
convert = widget.convert_from_user_string
properties[key] = convert(value)
try:
properties = do_geocoding(geo_fields, properties)
except GeocoderServiceError, e:
msg = (
'Warnings: could not find a valid address '
'for row ${record_number}: ${error}',
{
'record_number': record_number + 1, # account for header
'error': str(e)
})
self.warnings.append(msg)
address = properties.pop(geo_fields['address'])
ob_id = add_object(import_location,
_send_notifications=False,
**properties)
ob = import_location._getOb(ob_id)
if address:
setattr(ob, geo_fields['address'].split('.')[0],
Geo(address=address))
notify(NyContentObjectEditEvent(ob, user_id))
if extra_properties:
adapter = ICSVImportExtraColumns(ob, None)
if adapter is not None:
extra_props_messages = adapter.handle_columns(extra_properties)
if extra_props_messages:
self.errors.append(extra_props_messages)
#obj_ids.append(ob.getId())
ob.submitThis()
ob.approveThis(_send_notifications=False)
del import_location.REQUEST
del site.REQUEST
