def _check_meeting_data(self, csv_meeting):
if not csv_meeting.items:
message = "Meeting id {id} has no item. Skipping...".format(
id=csv_meeting.external_id)
logger.info(message)
self.errors["meeting"].append(message)
return False
return True
def set_up_meeting_config_used_items_attributes(meeting_config):
logger.info(
"Activating proposingGroupWithGroupInCharge and disabling groupsInCharge"
)
used_item_attributes = list(meeting_config.usedItemAttributes)
if u"proposingGroupWithGroupInCharge" not in meeting_config.usedItemAttributes:
used_item_attributes.append(u"proposingGroupWithGroupInCharge")
if u"groupsInCharge" in used_item_attributes:
used_item_attributes.remove(u"groupsInCharge")
meeting_config.usedItemAttributes = tuple(used_item_attributes)
meeting_config.at_post_edit_script()
def _annex_file_content(_path):
if not os.path.isfile(_path):
logger.info("Le fichier %s n'a pas ete trouve." % _path)
return None
with open(_path, "r") as annex_file:
name = safe_unicode(os.path.basename(_path))
annex_read = annex_file.read()
annex_blob = namedfile.NamedBlobFile(annex_read, filename=name)
return annex_blob
def import_data_from_csv(
self,
f_group_mapping,
f_items,
f_meetings,
default_group,
meeting_annex_dir_path,
item_annex_dir_path,
default_category_college=None,
default_category_council=None,
):
start_date = datetime.now()
import_csv = ImportCSV(
self,
f_group_mapping,
f_items,
f_meetings,
meeting_annex_dir_path,
item_annex_dir_path,
default_group,
default_category_college,
default_category_council,
)
meeting_counter, item_counter, errors = import_csv.run()
logger.info(
u"Inserted {meeting} meetings and {item} meeting items.".format(
meeting=meeting_counter, item=item_counter))
logger.warning(
u"{malforemed} meeting items were not created due to missing data in csv :\n\t{list}"
.format(malforemed=len(errors["item"]),
list=u"\n\t ".join(errors["item"])))
logger.warning(
u"{ioerr} errors occured while adding annexes :\n{list}".format(
ioerr=len(errors["io"]), list=u"\n\t ".join(errors["io"])))
logger.warning(
u"{meeting} meetings where skipped because they have no annex or no items :\n\t{list}"
.format(meeting=len(errors["meeting"]),
list=u"\n\t ".join(errors["meeting"])))
without_annex = u"\n\t ".join(safe_unicode(errors["item_without_annex"]))
logger.warning(u"{items} meeting items where skipped :\n\t{list}".format(
items=len(errors["item_without_annex"]), list=without_annex))
end_date = datetime.now()
seconds = end_date - start_date
seconds = seconds.seconds
hours = seconds / 3600
left_sec = seconds - hours * 3600
minutes = left_sec / 60
left_sec = left_sec - minutes * 60
logger.info(u"Import finished in {0} seconds ({1} h {2} m {3} s).".format(
seconds, hours, minutes, left_sec))
def fix_language(self, reindex_meetings=False, commit_threshold=1000, target_lang="fr"):
start_date = datetime.now()
logger.info("Start fixing languages converting to {}".format(target_lang))
fixer = FixLanguage(self)
fixer.run(reindex_meetings, commit_threshold, target_lang)
end_date = datetime.now()
seconds = end_date - start_date
seconds = seconds.seconds
hours = seconds / 3600
minutes = (seconds - hours * 3600) / 60
logger.info(
"Fixing finished in {0} seconds ({1} h {2} m).".format(seconds, hours, minutes)
)
def addUsers(self, org_descriptors):
'''Creates Plone users and add it to linked Plone groups.'''
plone_utils = self.portal.plone_utils
# if we are in dev, we use DEFAULT_USER_PASSWORD, else we will generate a
# password that is compliant with the current password policy...
if is_develop_environment():
password = DEFAULT_USER_PASSWORD
else:
password = generate_password()
msg = "The password used for added users is %s" % (
password or DEFAULT_USER_PASSWORD)
logger.info(msg)
# add a portal_message so admin adding the Plone site knows password
plone_utils.addPortalMessage(msg, 'warning')
member_tool = api.portal.get_tool('portal_membership')
own_org = get_own_organization()
plonegroup_org_uids = get_registry_organizations()
for org_descr in org_descriptors:
if org_descr.parent_path:
# find parent organization following parent path from container
container = own_org.restrictedTraverse(org_descr.parent_path)
else:
container = own_org
org = container.get(org_descr.id)
# Create users
for userDescr in org_descr.getUsers():
# if we defined a generated password here above, we use it
# either we use the password provided in the applied profile
if password:
userDescr.password = password
self.addUser(userDescr)
# Add users in the correct Plone groups.
org_uid = org.UID()
if org_uid in plonegroup_org_uids:
for suffix in get_all_suffixes(org_uid):
plone_group = get_plone_group(org_uid, suffix)
group_members = plone_group.getMemberIds()
# protect in case we have suffixes only for some groups
for userDescr in getattr(org_descr, suffix, []):
if userDescr.id not in group_members:
api.group.add_user(group=plone_group,
username=userDescr.id)
if userDescr.create_member_area:
member_tool.createMemberArea(userDescr.id)
def object_already_exists(self, obj_id, portal_type):
catalog_query = [
{
"i": "portal_type",
"o": "plone.app.querystring.operation.selection.is",
"v": portal_type,
},
{
"i": "id",
"o": "plone.app.querystring.operation.selection.is",
"v": obj_id,
},
]
query = queryparser.parseFormquery(self.portal, catalog_query)
res = self.portal.portal_catalog(**query)
if res:
logger.info("Already created {object}".format(object=obj_id))
return res
def insert_and_close_meeting(self, member_folder, csv_meeting):
if not self._check_meeting_data(csv_meeting):
return
_id = "meetingimport.{external_id}".format(
external_id=csv_meeting.external_id)
meeting = self.object_already_exists(_id, csv_meeting.portal_type)
if meeting and meeting[0]:
message = "Skipping meeting {id} and it items because it already exists".format(
id=_id)
logger.info(message)
self.errors["meeting"].append(message)
return
meeting_date = csv_meeting.date
meetingid = member_folder.invokeFactory(
type_name=csv_meeting.portal_type, id=_id, date=meeting_date)
meeting = getattr(member_folder, meetingid)
meeting.signatures = None
if csv_meeting.assembly:
meeting.assembly = richtextval(csv_meeting.assembly)
meeting.date = meeting_date
meeting.start_date = csv_meeting.started_on
meeting.end_date = csv_meeting.ended_on
meeting.creation_date = DateTime(csv_meeting.created_on)
logger.info(u"Created {type} {id} {date}".format(
type=csv_meeting.portal_type, id=_id, date=meeting.title))
if csv_meeting.annexes:
self.add_all_annexes_to_object(csv_meeting.annexes,
meeting,
confidential=True)
else:
meeting.observations = u"<p><strong>Cette séance n'a aucune annexe</strong></p>"
logger.info(u"Adding {items} items to {type} of {date}".format(
items=len(csv_meeting.items),
type=csv_meeting.portal_type,
date=meeting.title))
self.portal.REQUEST["PUBLISHED"] = meeting
for csv_item in csv_meeting.items:
self.insert_and_present_item(member_folder, csv_item)
if meeting.get_items():
meeting.portal_workflow.doActionFor(meeting, "freeze")
meeting.portal_workflow.doActionFor(meeting, "decide")
meeting.portal_workflow.doActionFor(meeting, "close")
for item in meeting.get_items():
item.setModificationDate(meeting_date)
item.reindexObject(idxs=["modified"])
meeting.setModificationDate(DateTime(meeting_date))
meeting.reindexObject(idxs=["modified"])
self.meeting_counter += 1
transaction.commit()
def set_default_in_charge_if_misssing_and_fix_certified_sign(
default_in_charge_uid,
remove_certified_signatures=[],
remove_certified_signatures_ignore_pattern=None):
cfg_groups = get_organizations(only_selected=False)
for group in cfg_groups:
if not group.groups_in_charge:
group.groups_in_charge = [default_in_charge_uid]
logger.info(u"Added default group in charge to {}".format(
group.title))
if not remove_certified_signatures_ignore_pattern or not re.match(
remove_certified_signatures_ignore_pattern, group.title):
# the organisation members create items
certified_signatures = []
for signature in group.certified_signatures:
if signature.get(
'signature_number') not in remove_certified_signatures:
certified_signatures.append(signature)
group.certified_signatures = certified_signatures
group.reindexObject()
def run(self, reindex_meetings, commit_threshold, target_lang):
counter = 0
member = self.portal.portal_membership.getAuthenticatedMember()
if not member.has_role("Manager"):
return "You must be a Manager to access this script !"
logger.info("Querying objects to fix")
brains = self.get_objects_to_correct()
logger.info("Treating {} Meeting or MeetingItems".format(len(brains)))
for brain in brains:
if brain.__class__.__name__ == "PloneFlare":
lang = brain['Language']
else:
lang = brain.Language()
if lang not in (target_lang, 'any'):
if brain.meta_type == "Meeting":
self.meeting_counter += 1
elif brain.meta_type == "MeetingItem":
self.item_counter += 1
if lang not in self.odds:
self.odds.append("'{}'".format(lang))
obj = brain.getObject()
obj.setLanguage("fr")
if brain.meta_type != "Meeting" or reindex_meetings:
obj.reindexObject(idxs=['modified', 'ModificationDate', 'Date'])
if brain.meta_type not in self.treated_count:
self.treated_count[brain.meta_type] = 1
else:
self.treated_count[brain.meta_type] += 1
counter += 1
if counter % commit_threshold == 0:
self.commit_and_print_state()
self.commit_and_print_state()
def load_items(self, delib_file, meetings):
logger.info("Load {0}".format(delib_file))
csv.field_size_limit(100000000)
self.item_meeting_ids = []
with io.open(delib_file, "r") as csvfile:
reader = csv.reader(csvfile)
for row in reader:
if reader.line_num == 1:
# skip header line
continue
try:
meeting_external_id = int(row[8].strip())
if meeting_external_id not in self.item_meeting_ids:
self.item_meeting_ids.append(meeting_external_id)
if meeting_external_id not in meetings:
logger.info(
"Unknown meeting for item : {row}".format(row=row))
else:
item = self.parse_and_clean_raw_csv_item(row)
meeting = meetings[meeting_external_id]
if meeting.portal_type == self.college_cfg.getMeetingTypeName(
):
portal_type = self.college_cfg.getItemTypeName()
elif meeting.portal_type == self.council_cfg.getMeetingTypeName(
):
portal_type = self.council_cfg.getItemTypeName()
else:
raise NotImplementedError(
"Unknown meeting type {}".format(type))
item.portal_type = portal_type
if not item.created_on:
item.created_on = meeting.date
meeting.items.append(item)
except ValueError as e:
self.errors["item"].append(e.message)
logger.info(e.message)
def createMeetingConfig(self, configData, source):
'''Creates a new meeting configuration from p_configData which is a
MeetingConfigDescriptor instance. p_source is a string that
corresponds to the absolute path of a profile; additional (binary)
data like images or templates that need to be attached to some
sub-objects of the meeting config will be searched there.'''
cData = configData.getData()
# turn org ids into org uids
for field_name in [
'orderedAssociatedOrganizations', 'orderedContacts',
'orderedGroupsInCharge', 'orderedItemInitiators',
'powerAdvisersGroups', 'selectableCopyGroups',
'selectableAdvisers', 'usingGroups'
]:
data = cData.get(field_name)
try:
data = [
org_id_to_uid(suffixed_group_id)
for suffixed_group_id in data
if org_id_to_uid(suffixed_group_id, raise_on_error=False)
]
except KeyError:
logger.warning(
'Error while turning org ids to org uids for field_name {0}'
.format(field_name))
data = []
cData[field_name] = data
# manage customAdvisers, turn 'org' value from org id to uid
ca = cData.get('customAdvisers')
adapted_ca = []
for v in ca:
new_value = v.copy()
new_value['org'] = org_id_to_uid(new_value['org'])
adapted_ca.append(new_value)
cData['customAdvisers'] = adapted_ca
# return if already exists
if cData['id'] in self.tool.objectIds():
logger.info(
'A MeetingConfig with id {0} already exists, passing...'.
format(cData['id']))
return
self.tool.invokeFactory('MeetingConfig', **cData)
cfgId = configData.id
cfg = getattr(self.tool, cfgId)
cfg._at_creation_flag = True
# for tests where config id is shuffled, save the real id
if "__real_id__" in cData:
cfg.__real_id__ = cData["__real_id__"]
# TextArea fields are not set properly.
for field in cfg.Schema().fields():
fieldName = field.getName()
widgetName = field.widget.getName()
if (widgetName == 'TextAreaWidget') and fieldName in cData:
field.set(cfg, cData[fieldName], mimetype='text/html')
# call processForm passing dummy values so existing values are not touched
cfg.processForm(values={'dummy': None})
# Validates meeting config (validation seems not to be triggered
# automatically when an object is created from code).
errors = []
for field in cfg.Schema().fields():
error = field.validate(cfg.getField(field.getName()).get(cfg), cfg)
if error:
errors.append("'%s': %s" % (field.getName(), error))
if errors:
raise PloneMeetingError(
MEETING_CONFIG_ERROR %
(safe_unicode(cfg.Title()), cfg.getId(), u'\n'.join(errors)))
if not configData.active:
self.portal.portal_wokflow.doActionFor(cfg, 'deactivate')
# Adds the sub-objects within the config: categories, classifiers, items in config, ...
for descr in configData.categories:
self.addCategory(cfg, descr)
for descr in configData.classifiers:
self.addCategory(cfg, descr, classifier=True)
for descr in configData.recurringItems:
self.addItemToConfig(cfg, descr)
for descr in configData.itemTemplates:
self.addItemToConfig(cfg, descr, isRecurring=False)
# configure ContentCategoryGroups before adding annex_types
# this will enable confidentiality, to_sign/signed, ... if necessary
for category_group_id, attrs in configData.category_group_activated_attrs.items(
):
category_group = cfg.annexes_types.get(category_group_id)
for attr in attrs:
if not hasattr(category_group, attr):
raise PloneMeetingError(
'Attribute {0} does not exist on {1} of {2}'.format(
attr, category_group_id, cfgId))
setattr(category_group, attr, True)
for descr in configData.annexTypes:
self.addAnnexType(cfg, descr, source)
# first create style templates so it exist before being used by a pod template
for descr in configData.styleTemplates:
self.addPodTemplate(cfg, descr, source)
for descr in configData.podTemplates:
self.addPodTemplate(cfg, descr, source)
# manage MeetingManagers
groupsTool = self.portal.portal_groups
for userId in configData.meetingManagers:
groupsTool.addPrincipalToGroup(
userId, '{0}_{1}'.format(cfg.getId(),
MEETINGMANAGERS_GROUP_SUFFIX))
# manage annex confidentiality, enable it on relevant CategoryGroup
if configData.itemAnnexConfidentialVisibleFor:
cfg.annexes_types.item_annexes.confidentiality_activated = True
cfg.annexes_types.item_decision_annexes.confidentiality_activated = True
if configData.adviceAnnexConfidentialVisibleFor:
cfg.annexes_types.advice_annexes.confidentiality_activated = True
if configData.meetingAnnexConfidentialVisibleFor:
cfg.annexes_types.meeting_annexes.confidentiality_activated = True
return cfg
def _finishConfigFor(self, cfg, data):
"""When the MeetingConfig has been created, some parameters still need to be applied
because they need the MeetingConfig to exist."""
# apply the meetingTopicStates to the 'searchnotdecidedmeetings' DashboardCollection
updateCollectionCriterion(
cfg.searches.searches_meetings.searchnotdecidedmeetings,
'review_state', list(data.meetingTopicStates))
# apply the maxDaysDecisions to the 'searchlastdecisions' DashboardCollection
updateCollectionCriterion(
cfg.searches.searches_decisions.searchlastdecisions,
'meeting_date', unicode(data.maxDaysDecisions))
# apply the decisionTopicStates to the 'searchlastdecisions'
# and 'searchalldecision' DashboardCollections
updateCollectionCriterion(
cfg.searches.searches_decisions.searchlastdecisions,
'review_state', list(data.decisionTopicStates))
# select correct default view
meetingAppDefaultView = data.meetingAppDefaultView
if meetingAppDefaultView in cfg.searches.searches_items.objectIds():
cfg._set_default_faceted_search(meetingAppDefaultView)
else:
error = 'meetingAppDefaultView : No DashboardCollection with id %s' % meetingAppDefaultView
raise PloneMeetingError(MEETING_CONFIG_ERROR %
(cfg.Title(), cfg.getId(), error))
# now we can set values for dashboard...Filters fields as the 'searches' folder has been created
for fieldName in ('dashboardItemsListingsFilters',
'dashboardMeetingAvailableItemsFilters',
'dashboardMeetingLinkedItemsFilters'):
field = cfg.getField(fieldName)
# we want to validate the vocabulay, as if enforceVocabulary was True
error = field.validate_vocabulary(
cfg,
cfg.getField(field.getName()).get(cfg), {})
if error:
raise PloneMeetingError(MEETING_CONFIG_ERROR %
(cfg.Title(), cfg.getId(), error))
if data.addContactsCSV:
output = import_contacts(self.portal, path=self.profilePath)
logger.info(output)
selectableOrderedContacts = cfg.getField(
'orderedContacts').Vocabulary(cfg).keys()
cfg.setOrderedContacts(selectableOrderedContacts)
# turn contact path to uid
for org_storing_field in ('orderedContacts', ):
org_storing_data = getattr(data, org_storing_field, [])
if org_storing_data:
contact_uids = []
for contact_path in org_storing_data:
try:
contact_uid = self.portal.contacts.restrictedTraverse(
contact_path).UID()
contact_uids.append(contact_uid)
except KeyError:
logger.warning(
'While computing "{0}", could not get contact at "{1}"'
.format(org_storing_field, contact_path))
cfg.getField(org_storing_field).set(cfg, contact_uids)
# set default labels
if data.defaultLabels:
jar = ILabelJar(cfg)
update_labels_jar(jar, values=data.defaultLabels)
# disable relevant dashboard collections
for collection_path in data.disabled_collections:
try:
collection = cfg.searches.restrictedTraverse(collection_path)
except AttributeError:
logger.warning(
'While disabling collections, no collection found at {0}, ...'
.format(collection_path))
collection.enabled = False
collection.reindexObject(idxs=['enabled'])
def initialize_proposingGroupWithGroupInCharge(
self,
default_in_charge_uid,
config_ids=[],
ignore_if_others=[],
remove_certified_signatures=[],
remove_certified_signatures_ignore_pattern=None):
if not isinstance(remove_certified_signatures, list):
remove_certified_signatures = [remove_certified_signatures]
start_date = datetime.now()
count_patched = 0
count_global = 0
set_default_in_charge_if_misssing_and_fix_certified_sign(
default_in_charge_uid, remove_certified_signatures,
remove_certified_signatures_ignore_pattern)
item_type_names = []
if not config_ids:
meeting_configs = self.portal_plonemeeting.listFolderContents()
else:
meeting_configs = []
for config_id in config_ids:
meeting_configs.append(self.portal_plonemeeting.get(config_id))
for meeting_config in meeting_configs:
set_up_meeting_config_used_items_attributes(meeting_config)
item_type_names.append(meeting_config.getItemTypeName())
items = self.portal_catalog(portal_type=item_type_names)
logger.info("Checking {} {}".format(len(items), item_type_names))
for brain in items:
if not brain.getGroupsInCharge:
formatted_gp = None
item = brain.getObject()
proposing_group = item.getProposingGroup(theObject=True)
if proposing_group:
groups_in_charge = deepcopy(proposing_group.groups_in_charge)
for in_charge in groups_in_charge:
if in_charge not in ignore_if_others:
formatted_gp = special_format.format(
item.getProposingGroup(), in_charge)
item.setProposingGroupWithGroupInCharge(formatted_gp)
break
if not formatted_gp:
formatted_gp = special_format.format(
item.getProposingGroup(),
item.getGroupsInCharge(fromOrgIfEmpty=True,
first=True),
)
item.setProposingGroupWithGroupInCharge(formatted_gp)
item.reindexObject(idxs=["getGroupsInCharge"])
item.updateLocalRoles()
count_patched += 1
count_global += 1
if count_global % 200 == 0:
logger.info("Checked {} / {} items. Patched {} of them".format(
count_global,
len(items),
count_patched,
))
# save what's already done
if count_patched % 10000 == 0:
transaction.commit()
end_date = datetime.now()
seconds = end_date - start_date
seconds = seconds.seconds
hours = seconds / 3600
minutes = (seconds - hours * 3600) / 60
logger.info(u"Completed in {0} seconds (about {1} h {2} m).".format(
seconds, hours, minutes))
if count_patched > 0:
ratio = count_patched / seconds
logger.info(u"That's %2.2f items patched per second" % ratio)
def run(self):
member = self.portal.portal_membership.getAuthenticatedMember()
if not member.has_role("Manager"):
raise ValueError("You must be a Manager to access this script !")
# Load all csv into memory
cfg_groups = get_organizations(only_selected=False)
for group in cfg_groups:
self.grp_id_mapping[group.UID()] = group
logger.info("Load {0}".format(self.f_group_mapping))
with io.open(self.f_group_mapping, "r") as csvfile:
reader = csv.reader(csvfile)
for row in reader:
grp_id = row[1].strip()
if grp_id in self.grp_id_mapping:
self.groups[
row[0].strip()] = self.grp_id_mapping[grp_id].UID()
else:
self.groups[row[0].strip()] = self.default_group
meetings = {}
logger.info("Load {0}".format(self.f_meetings))
with io.open(self.f_meetings, "r") as csvfile:
reader = csv.reader(csvfile)
for row in reader:
if reader.line_num == 1:
# skip header line
continue
# Because numbers are not numbers but unicode chars...
external_id = int(row[0].strip())
csv_type = safe_unicode(row[6].strip()).lower()
if 'col' in csv_type:
portal_type = self.college_cfg.getMeetingTypeName()
elif 'cons' in csv_type:
portal_type = self.council_cfg.getMeetingTypeName()
else:
continue
meeting = CSVMeeting(external_id=external_id,
date=row[1].strip(),
created_on=row[2].strip(),
started_on=row[3].strip(),
ended_on=row[4].strip(),
assembly=safe_unicode(row[5].strip()),
portal_type=portal_type,
annexes_dir=self.meeting_annex_dir_path)
self.add_meeting_to_dict(meetings, meeting)
self.load_items(self.f_items, meetings)
# insert All
self.disable_recurring_items()
logger.info("Inserting Objects")
try:
for csv_meeting in meetings.values():
if csv_meeting.portal_type == self.college_cfg.getMeetingTypeName(
):
self.insert_and_close_meeting(self.college_member_folder,
csv_meeting)
elif csv_meeting.portal_type == self.council_cfg.getMeetingTypeName(
):
self.insert_and_close_meeting(self.council_member_folder,
csv_meeting)
else:
raise NotImplementedError(
u"Not managed meeting type '{}' for meeting id {}".
format(csv_meeting.type, meeting.external_id))
finally:
tool = api.portal.get_tool('portal_plonemeeting')
self.re_enable_recurring_items()
tool.invalidateAllCache()
return self.meeting_counter, self.item_counter, self.errors
def commit_and_print_state(self):
transaction.commit()
message = "Transaction commited. {}".format(self.treated_count)
logger.info(message)