def __iter__(self):
for item in self.previous:
# organization parent ?
item_type = item['_type']
if item_type in ('organization', 'held_position') and item['_oid']:
if item['_oid'] not in self.ids['organization'][item['_set']]:
log_error(item,
u"SKIPPING: invalid parent organization id '{}'".
format(item['_oid']),
level='critical')
if self.roe:
raise Exception(u'Cannot find parent ! See log...')
continue
item['_parent'] = self.ids['organization'][item['_set']][
item['_oid']]['path']
item['_related_title'] = self.portal.unrestrictedTraverse(
item['_parent']).get_full_title()
# person parent ?
if item_type == 'held_position':
if item['_pid'] not in self.ids['person'][item['_set']]:
log_error(
item,
u"SKIPPING: invalid related person id '{}'".format(
item['_pid']),
level='critical')
if self.roe:
raise Exception(
u'Cannot find related person ! See log...')
continue
item['_parent'] = self.ids['person'][item['_set']][
item['_pid']]['path']
yield item
def rows(self, typ, sett):
o_logger.info(u"Reading '{}' '{}' ({})".format(sett, typ, self.storage['csv_files'][typ].name))
reader = csv.DictReader(self.storage['csv_files'][typ], dialect=self.dialect,
fieldnames=self.storage['fieldnames'][typ], restkey='_rest',
restval='__NO_CO_LU_MN__', **self.fmtparam)
for item in reader:
item['_type'] = typ
item['_set'] = sett
item['_ln'] = reader.line_num
# check fieldnames length on first line
if reader.line_num == 1:
reader.restval = u''
if '_rest' in item:
log_error(item, u'STOPPING: some columns are not defined in fieldnames: {}'.format(item['_rest']),
level='critical')
if self.roe:
raise Exception(u'Some columns for {} are not defined in fieldnames: {}'.format(typ,
item['_rest']))
break
extra_cols = [key for (key, val) in item.items() if val == '__NO_CO_LU_MN__']
if extra_cols:
log_error(item, u'STOPPING: to much columns defined in fieldnames: {}'.format(extra_cols),
level='critical')
if self.roe:
raise Exception(u'To much columns for {} defined in fieldnames: {}'.format(typ, extra_cols))
break
# pass headers if any
if self.csv_headers:
continue
yield item
self.storage['csv_files'][typ].close()
def __iter__(self):
idnormalizer = getUtility(IIDNormalizer)
for item in self.previous:
if '_path' in item: # _path has already be set
yield item
continue
item_type = item['_type']
title = u'-'.join(
[item[key] for key in self.id_keys[item_type] if item[key]])
if item_type == 'held_position' and '_related_title' in item:
title = u'-'.join([title, item.pop('_related_title')])
if not title:
log_error(item,
u'cannot get an id from id keys {}'.format(
self.id_keys[item_type]),
level='critical')
if self.roe:
raise Exception(u'No title ! See log...')
continue
new_id = idnormalizer.normalize(title)
item['_path'] = '/'.join([item['_parent'], new_id])
# we rename id if it already exists
item['_path'] = correct_path(self.portal, item['_path'])
item['_act'] = 'new'
# we store _path for each _id
self.ids[item_type][item['_set']][
item['_id']]['path'] = item['_path']
yield item
def __iter__(self):
for item in self.previous:
if item['_type'] == 'person' and item['_ic'] and item['internal_number']:
# for internal person, internal_number contains plone username
if not api.user.get(username=item['internal_number']):
log_error(item, "username '{}' not found".format(item['internal_number']), level='critical')
if self.roe:
raise Exception(u'User not found ! See log...')
else:
# we define the specific field
item['userid'] = item['internal_number']
item['internal_number'] = None
yield item
def __iter__(self):
for item in self.previous:
if '_path' not in item:
yield item
continue
if '_inactive' in item:
obj = self.portal.unrestrictedTraverse(item['_path'],
default=None)
state = api.content.get_state(obj=obj)
if item['_inactive'] and state == 'active':
item['_transitions'] = 'deactivate'
elif not item['_inactive'] and state == 'deactivated':
log_error(
item,
u'_inactive is False and current state is deactivated: we do not activate'
)
yield item
def __iter__(self):
intids = getUtility(IIntIds)
for item in self.previous:
item_type = item['_type']
if item_type == 'held_position':
if item['_oid'] and item['_oid'] not in self.ids[
'organization'][item['_set']]:
log_error(
item,
u"SKIPPING: invalid related organization id '{}'".
format(item['_oid']),
level='critical')
if self.roe:
raise Exception(u'Cannot find _oid ! See log...')
continue
# not using _pid yet
org = self.portal.unrestrictedTraverse(
self.ids['organization'][item['_set']][
item['_oid']]['path'])
item['position'] = RelationValue(intids.getId(org))
yield item
def replace_relation(item,
portal,
catalog,
rel,
path='from_path',
field='',
repl_iid=None):
obj = portal.unrestrictedTraverse(getattr(rel, path), default=None)
if obj is None:
log_error(item, u"cannot find linked object: {}".format(rel.from_path))
catalog.unindex(rel) # remove bad relation...
return
value = getattr(obj, field)
if isinstance(value, RelationValue):
value = RelationValue(repl_iid)
else:
if isinstance(value, tuple):
value = list(value)
value.remove(rel)
value.append(RelationValue(repl_iid))
setattr(obj, field, value)
modified(obj) # will update relations catalog too
def __iter__(self):
for item in self.previous:
if item['_type'] != 'directory' and item.get('_act', 'no') == 'update' and \
item['_parent'] != os.path.dirname(item['_path']):
obj = self.portal.unrestrictedTraverse(item['_path'],
default=None)
if obj is None:
log_error(
item,
u"SKIPPING: cannot find existing object '{}'".format(
item['_path']),
level='critical')
if self.roe:
raise Exception(
u'Cannot find existing object ! See log...')
continue
target = self.portal.unrestrictedTraverse(item['_parent'],
default=None)
if target is None:
log_error(item,
u"SKIPPING: cannot find new parent '{}'".format(
item['_parent']),
level='critical')
if self.roe:
raise Exception(u'Cannot find new parent ! See log...')
continue
# we move the object and update path, so all the next sections will work on this path
# constructor NO, update YES
moved_obj = api.content.move(obj, target)
# TODO manage organization_type see dir_org_config
# print("'{}' moved to '{}'".format(item['_path'], item['_parent']))
item['_path'] = relative_path(
self.portal, '/'.join(moved_obj.getPhysicalPath()))
self.ids[item['_type']][item['_set']][
item['_id']]['path'] = item['_path']
# indexes and relations are well updated
yield item
def __iter__(self):
for item in self.previous:
if '_path' in item: # _path has already be set
yield item
continue
item_type = item['_type']
# we will do a search for each index
for field, idx, condition, must_exist in self.uniques[item_type]:
if item[field] and condition(item):
if field == 'internal_number' and idx == 'internal_number' and not self.cbin_beh[
item_type]:
log_error(
item,
u"the internalnumber behavior is not defined on type {}"
.format(item_type),
level='critical')
if self.roe:
raise Exception(
u'The internalnumber behavior is not defined on type {}'
.format(item_type))
continue
brains = self.catalog.unrestrictedSearchResults({
'portal_type':
item_type,
idx:
item[field]
})
if len(brains) > 1:
log_error(
item,
u"the search with '{}'='{}' gets multiple objs: {}"
.format(idx, item[field],
u', '.join([b.getPath() for b in brains])),
level='critical')
if self.roe:
raise Exception(u'Too more results ! See log...')
continue
elif len(brains):
item['_path'] = relative_path(self.portal,
brains[0].getPath())
item['_act'] = 'update'
# we store _path for each _id
self.ids[item_type][item['_set']][
item['_id']]['path'] = item['_path']
break
elif must_exist(item):
log_error(
item,
u"the search with '{}'='{}' doesn't get any result"
.format(idx, item[field]),
level='critical')
if self.roe:
raise Exception(
u'Must find something ! See log...')
yield item
def __iter__(self): # noqa
for item in self.previous:
if item['_merger']:
# checking current contact
current_obj = self.portal.unrestrictedTraverse(item['_path'], default=None)
if current_obj is None:
log_error(item, "Cannot find main object with path '{}' and act '{}'".format(item['_path'],
item['_act']), level='critical')
current_iid = self.intids.queryId(current_obj)
if current_iid is None:
log_error(item, u"cannot find current object intid: {}".format(current_obj), level='critical')
if self.roe:
raise Exception("Cannot find current object intid '{}'".format(current_obj))
# searching replacement object
brains = self.catalog.unrestrictedSearchResults({'portal_type': item['_type'],
'internal_number': item['_merger']})
if len(brains) > 1:
log_error(item, u"the search with 'internal_number'='{}' gets multiple objs: {}".format(
item['_merger'], u', '.join([b.getPath() for b in brains])), level='critical')
if self.roe:
raise Exception("Find multiple objects with internal number '{}'".format(item['_merger']))
continue
elif not brains:
log_error(item, u"the search with 'internal_number'='{}' doesn't "
u"get any result".format(item['_merger']), level='critical')
if self.roe:
raise Exception("Cannot find object with internal number '{}'".format(item['_merger']))
continue
repl_obj = brains[0].getObject()
repl_iid = self.intids.getId(repl_obj)
if repl_iid is None:
log_error(item, u"cannot find replacement object intid: {}".format(repl_obj), level='critical')
if self.roe:
raise Exception("Cannot find replacement object intid '{}'".format(repl_obj))
continue
# getting relations pointing to current object
rels = list(self.rel_catalog.findRelations({'to_id': current_iid}))
for rel in rels:
if rel.from_object.portal_type in ('dmsincomingmail', 'dmsincoming_email') and \
rel.from_attribute == 'sender':
replace_relation(item, self.portal, self.catalog, rel,
path='from_path', field='sender', repl_iid=repl_iid)
elif rel.from_object.portal_type == 'dmsoutgoingmail' and rel.from_attribute == 'recipients':
replace_relation(item, self.portal, self.catalog, rel,
path='from_path', field='recipients', repl_iid=repl_iid)
elif rel.from_object.portal_type == 'held_position' and rel.from_attribute == 'position':
replace_relation(item, self.portal, self.catalog, rel,
path='from_path', field='position', repl_iid=repl_iid)
elif rel.from_object.portal_type == 'contact_list' and rel.from_attribute == 'contacts':
replace_relation(item, self.portal, self.catalog, rel,
path='from_path', field='contacts', repl_iid=repl_iid)
else:
raise Exception("Relation type not handled! pt '{}', field '{}'".format(
rel.from_object.portal_type, rel.from_attribute))
# getting relations pointing to current object
rels = list(self.rel_catalog.findRelations({'from_id': current_iid}))
if rels:
raise Exception(u"relation from_id not handled! to paths '{}'".format(
', '.join([rel.to_path for rel in rels])))
# deleting current obj
api.content.delete(obj=current_obj)
item['_act'] = 'delete'
# cleaning item
item['_del_path'] = item.pop('_path') # we remove the path so all others sections are skipped
yield item
def __iter__(self):
idnormalizer = getUtility(IIDNormalizer)
for item in self.previous:
item_type = item['_type']
# set correct values
for fld in self.fieldnames[item_type]:
item[fld] = safe_unicode(item[fld].strip(' '),
encoding=self.csv_encoding)
if item_type == 'held_position':
item['_fid'] = None # we don't yet manage position
# set directory as default parent
item['_parent'] = self.directory_path
# duplicated _id ?
if not item['_id']:
log_error(item,
u"SKIPPING: missing id '_id'",
level='critical')
if self.roe:
raise Exception(u'Missing id ! See log...')
continue
if item['_id'] in self.ids[item_type][item['_set']]:
log_error(
item,
u"SKIPPING: duplicated id '{}', already present line {}".
format(
item['_id'],
self.ids[item_type][item['_set']][item['_id']]['ln']),
level='critical')
if self.roe:
raise Exception(u'Duplicated id ! See log...')
self.ids[item_type][item['_set']][item['_id']] = {
'path': '',
'ln': item['_ln']
}
# uniqueness
for key in self.uniques[item_type]:
if not item[key]:
continue
uniques = self.uniques[item_type][key].setdefault(
item['_set'], {})
if item[key] in uniques:
log_error(
item, u"duplicated {} '{}', already present line {:d}".
format(key, item[key], uniques[item[key]]))
else:
uniques[item[key]] = item['_ln']
# to bool from int
for key in self.booleans[item_type]:
item[key] = to_bool(item, key)
if 'country' in item:
country_code = get_country_code(item, 'country',
self.phone_country,
self.languages)
# check zip
if 'zip_code' in item:
item['zip_code'] = valid_zip(item, 'zip_code', country_code)
# check phones
for key in ('phone', 'cell_phone', 'fax'):
if key in item:
item[key] = valid_phone(item, key, country_code,
self.phone_country)
# check email
if 'email' in item:
item['email'] = valid_email(item, 'email')
# organization checks
if item_type == 'organization':
if item['_id'] == item['_oid']:
log_error(item,
u'SKIPPING: _oid is equal to _id {}'.format(
item['_id']),
level='critical')
if self.roe:
raise Exception(u'Inconsistent _oid ! See log...')
continue
# keep only alphanum chars
if 'enterprise_number' in item and item['enterprise_number']:
item['enterprise_number'] = alphanum(
item['enterprise_number']).strip()
# manage org type if 'organization_type' column is defined
# (some clients use this column to put something else)
if 'organization_type' in item:
type_type = item['_oid'] and 'levels' or 'types'
if item['organization_type']:
if item['organization_type'] not in self.dir_org_config[
type_type]:
self.dir_org_config[type_type][item['organization_type']] = \
safe_unicode(idnormalizer.normalize(item['organization_type']))
item['organization_type'] = self.dir_org_config[
type_type][item['organization_type']]
else: # we take the first value
item['organization_type'] = self.dir_org_config[
type_type].values()[0]
elif item_type == 'person':
item['gender'] = valid_value_in_list(item, item['gender'],
('', 'F', 'M'))
item['birthday'] = valid_date(item, item['birthday'])
elif item_type == 'held_position':
item['start_date'] = valid_date(item, item['start_date'])
item['end_date'] = valid_date(item, item['end_date'])
if not item['_pid']:
log_error(item,
u"SKIPPING: missing related person id",
level='critical')
if self.roe:
raise Exception(u'Missing _pid ! See log...')
continue
if not item['_oid'] and not item['_fid']:
log_error(item,
u"SKIPPING: missing organization/position id",
level='critical')
if self.roe:
raise Exception(u'Missing _oid/_fid ! See log...')
continue
yield item