def _check_meeting_data(self, csv_meeting): if not csv_meeting.items: message = "Meeting id {id} has no item. Skipping...".format( id=csv_meeting.external_id) logger.info(message) self.errors["meeting"].append(message) return False return True
def set_up_meeting_config_used_items_attributes(meeting_config): logger.info( "Activating proposingGroupWithGroupInCharge and disabling groupsInCharge" ) used_item_attributes = list(meeting_config.usedItemAttributes) if u"proposingGroupWithGroupInCharge" not in meeting_config.usedItemAttributes: used_item_attributes.append(u"proposingGroupWithGroupInCharge") if u"groupsInCharge" in used_item_attributes: used_item_attributes.remove(u"groupsInCharge") meeting_config.usedItemAttributes = tuple(used_item_attributes) meeting_config.at_post_edit_script()
def _annex_file_content(_path): if not os.path.isfile(_path): logger.info("Le fichier %s n'a pas ete trouve." % _path) return None with open(_path, "r") as annex_file: name = safe_unicode(os.path.basename(_path)) annex_read = annex_file.read() annex_blob = namedfile.NamedBlobFile(annex_read, filename=name) return annex_blob
def import_data_from_csv( self, f_group_mapping, f_items, f_meetings, default_group, meeting_annex_dir_path, item_annex_dir_path, default_category_college=None, default_category_council=None, ): start_date = datetime.now() import_csv = ImportCSV( self, f_group_mapping, f_items, f_meetings, meeting_annex_dir_path, item_annex_dir_path, default_group, default_category_college, default_category_council, ) meeting_counter, item_counter, errors = import_csv.run() logger.info( u"Inserted {meeting} meetings and {item} meeting items.".format( meeting=meeting_counter, item=item_counter)) logger.warning( u"{malforemed} meeting items were not created due to missing data in csv :\n\t{list}" .format(malforemed=len(errors["item"]), list=u"\n\t ".join(errors["item"]))) logger.warning( u"{ioerr} errors occured while adding annexes :\n{list}".format( ioerr=len(errors["io"]), list=u"\n\t ".join(errors["io"]))) logger.warning( u"{meeting} meetings where skipped because they have no annex or no items :\n\t{list}" .format(meeting=len(errors["meeting"]), list=u"\n\t ".join(errors["meeting"]))) without_annex = u"\n\t ".join(safe_unicode(errors["item_without_annex"])) logger.warning(u"{items} meeting items where skipped :\n\t{list}".format( items=len(errors["item_without_annex"]), list=without_annex)) end_date = datetime.now() seconds = end_date - start_date seconds = seconds.seconds hours = seconds / 3600 left_sec = seconds - hours * 3600 minutes = left_sec / 60 left_sec = left_sec - minutes * 60 logger.info(u"Import finished in {0} seconds ({1} h {2} m {3} s).".format( seconds, hours, minutes, left_sec))
def fix_language(self, reindex_meetings=False, commit_threshold=1000, target_lang="fr"): start_date = datetime.now() logger.info("Start fixing languages converting to {}".format(target_lang)) fixer = FixLanguage(self) fixer.run(reindex_meetings, commit_threshold, target_lang) end_date = datetime.now() seconds = end_date - start_date seconds = seconds.seconds hours = seconds / 3600 minutes = (seconds - hours * 3600) / 60 logger.info( "Fixing finished in {0} seconds ({1} h {2} m).".format(seconds, hours, minutes) )
def addUsers(self, org_descriptors): '''Creates Plone users and add it to linked Plone groups.''' plone_utils = self.portal.plone_utils # if we are in dev, we use DEFAULT_USER_PASSWORD, else we will generate a # password that is compliant with the current password policy... if is_develop_environment(): password = DEFAULT_USER_PASSWORD else: password = generate_password() msg = "The password used for added users is %s" % ( password or DEFAULT_USER_PASSWORD) logger.info(msg) # add a portal_message so admin adding the Plone site knows password plone_utils.addPortalMessage(msg, 'warning') member_tool = api.portal.get_tool('portal_membership') own_org = get_own_organization() plonegroup_org_uids = get_registry_organizations() for org_descr in org_descriptors: if org_descr.parent_path: # find parent organization following parent path from container container = own_org.restrictedTraverse(org_descr.parent_path) else: container = own_org org = container.get(org_descr.id) # Create users for userDescr in org_descr.getUsers(): # if we defined a generated password here above, we use it # either we use the password provided in the applied profile if password: userDescr.password = password self.addUser(userDescr) # Add users in the correct Plone groups. org_uid = org.UID() if org_uid in plonegroup_org_uids: for suffix in get_all_suffixes(org_uid): plone_group = get_plone_group(org_uid, suffix) group_members = plone_group.getMemberIds() # protect in case we have suffixes only for some groups for userDescr in getattr(org_descr, suffix, []): if userDescr.id not in group_members: api.group.add_user(group=plone_group, username=userDescr.id) if userDescr.create_member_area: member_tool.createMemberArea(userDescr.id)
def object_already_exists(self, obj_id, portal_type): catalog_query = [ { "i": "portal_type", "o": "plone.app.querystring.operation.selection.is", "v": portal_type, }, { "i": "id", "o": "plone.app.querystring.operation.selection.is", "v": obj_id, }, ] query = queryparser.parseFormquery(self.portal, catalog_query) res = self.portal.portal_catalog(**query) if res: logger.info("Already created {object}".format(object=obj_id)) return res
def insert_and_close_meeting(self, member_folder, csv_meeting): if not self._check_meeting_data(csv_meeting): return _id = "meetingimport.{external_id}".format( external_id=csv_meeting.external_id) meeting = self.object_already_exists(_id, csv_meeting.portal_type) if meeting and meeting[0]: message = "Skipping meeting {id} and it items because it already exists".format( id=_id) logger.info(message) self.errors["meeting"].append(message) return meeting_date = csv_meeting.date meetingid = member_folder.invokeFactory( type_name=csv_meeting.portal_type, id=_id, date=meeting_date) meeting = getattr(member_folder, meetingid) meeting.signatures = None if csv_meeting.assembly: meeting.assembly = richtextval(csv_meeting.assembly) meeting.date = meeting_date meeting.start_date = csv_meeting.started_on meeting.end_date = csv_meeting.ended_on meeting.creation_date = DateTime(csv_meeting.created_on) logger.info(u"Created {type} {id} {date}".format( type=csv_meeting.portal_type, id=_id, date=meeting.title)) if csv_meeting.annexes: self.add_all_annexes_to_object(csv_meeting.annexes, meeting, confidential=True) else: meeting.observations = u"<p><strong>Cette séance n'a aucune annexe</strong></p>" logger.info(u"Adding {items} items to {type} of {date}".format( items=len(csv_meeting.items), type=csv_meeting.portal_type, date=meeting.title)) self.portal.REQUEST["PUBLISHED"] = meeting for csv_item in csv_meeting.items: self.insert_and_present_item(member_folder, csv_item) if meeting.get_items(): meeting.portal_workflow.doActionFor(meeting, "freeze") meeting.portal_workflow.doActionFor(meeting, "decide") meeting.portal_workflow.doActionFor(meeting, "close") for item in meeting.get_items(): item.setModificationDate(meeting_date) item.reindexObject(idxs=["modified"]) meeting.setModificationDate(DateTime(meeting_date)) meeting.reindexObject(idxs=["modified"]) self.meeting_counter += 1 transaction.commit()
def set_default_in_charge_if_misssing_and_fix_certified_sign( default_in_charge_uid, remove_certified_signatures=[], remove_certified_signatures_ignore_pattern=None): cfg_groups = get_organizations(only_selected=False) for group in cfg_groups: if not group.groups_in_charge: group.groups_in_charge = [default_in_charge_uid] logger.info(u"Added default group in charge to {}".format( group.title)) if not remove_certified_signatures_ignore_pattern or not re.match( remove_certified_signatures_ignore_pattern, group.title): # the organisation members create items certified_signatures = [] for signature in group.certified_signatures: if signature.get( 'signature_number') not in remove_certified_signatures: certified_signatures.append(signature) group.certified_signatures = certified_signatures group.reindexObject()
def run(self, reindex_meetings, commit_threshold, target_lang): counter = 0 member = self.portal.portal_membership.getAuthenticatedMember() if not member.has_role("Manager"): return "You must be a Manager to access this script !" logger.info("Querying objects to fix") brains = self.get_objects_to_correct() logger.info("Treating {} Meeting or MeetingItems".format(len(brains))) for brain in brains: if brain.__class__.__name__ == "PloneFlare": lang = brain['Language'] else: lang = brain.Language() if lang not in (target_lang, 'any'): if brain.meta_type == "Meeting": self.meeting_counter += 1 elif brain.meta_type == "MeetingItem": self.item_counter += 1 if lang not in self.odds: self.odds.append("'{}'".format(lang)) obj = brain.getObject() obj.setLanguage("fr") if brain.meta_type != "Meeting" or reindex_meetings: obj.reindexObject(idxs=['modified', 'ModificationDate', 'Date']) if brain.meta_type not in self.treated_count: self.treated_count[brain.meta_type] = 1 else: self.treated_count[brain.meta_type] += 1 counter += 1 if counter % commit_threshold == 0: self.commit_and_print_state() self.commit_and_print_state()
def load_items(self, delib_file, meetings): logger.info("Load {0}".format(delib_file)) csv.field_size_limit(100000000) self.item_meeting_ids = [] with io.open(delib_file, "r") as csvfile: reader = csv.reader(csvfile) for row in reader: if reader.line_num == 1: # skip header line continue try: meeting_external_id = int(row[8].strip()) if meeting_external_id not in self.item_meeting_ids: self.item_meeting_ids.append(meeting_external_id) if meeting_external_id not in meetings: logger.info( "Unknown meeting for item : {row}".format(row=row)) else: item = self.parse_and_clean_raw_csv_item(row) meeting = meetings[meeting_external_id] if meeting.portal_type == self.college_cfg.getMeetingTypeName( ): portal_type = self.college_cfg.getItemTypeName() elif meeting.portal_type == self.council_cfg.getMeetingTypeName( ): portal_type = self.council_cfg.getItemTypeName() else: raise NotImplementedError( "Unknown meeting type {}".format(type)) item.portal_type = portal_type if not item.created_on: item.created_on = meeting.date meeting.items.append(item) except ValueError as e: self.errors["item"].append(e.message) logger.info(e.message)
def createMeetingConfig(self, configData, source): '''Creates a new meeting configuration from p_configData which is a MeetingConfigDescriptor instance. p_source is a string that corresponds to the absolute path of a profile; additional (binary) data like images or templates that need to be attached to some sub-objects of the meeting config will be searched there.''' cData = configData.getData() # turn org ids into org uids for field_name in [ 'orderedAssociatedOrganizations', 'orderedContacts', 'orderedGroupsInCharge', 'orderedItemInitiators', 'powerAdvisersGroups', 'selectableCopyGroups', 'selectableAdvisers', 'usingGroups' ]: data = cData.get(field_name) try: data = [ org_id_to_uid(suffixed_group_id) for suffixed_group_id in data if org_id_to_uid(suffixed_group_id, raise_on_error=False) ] except KeyError: logger.warning( 'Error while turning org ids to org uids for field_name {0}' .format(field_name)) data = [] cData[field_name] = data # manage customAdvisers, turn 'org' value from org id to uid ca = cData.get('customAdvisers') adapted_ca = [] for v in ca: new_value = v.copy() new_value['org'] = org_id_to_uid(new_value['org']) adapted_ca.append(new_value) cData['customAdvisers'] = adapted_ca # return if already exists if cData['id'] in self.tool.objectIds(): logger.info( 'A MeetingConfig with id {0} already exists, passing...'. format(cData['id'])) return self.tool.invokeFactory('MeetingConfig', **cData) cfgId = configData.id cfg = getattr(self.tool, cfgId) cfg._at_creation_flag = True # for tests where config id is shuffled, save the real id if "__real_id__" in cData: cfg.__real_id__ = cData["__real_id__"] # TextArea fields are not set properly. for field in cfg.Schema().fields(): fieldName = field.getName() widgetName = field.widget.getName() if (widgetName == 'TextAreaWidget') and fieldName in cData: field.set(cfg, cData[fieldName], mimetype='text/html') # call processForm passing dummy values so existing values are not touched cfg.processForm(values={'dummy': None}) # Validates meeting config (validation seems not to be triggered # automatically when an object is created from code). errors = [] for field in cfg.Schema().fields(): error = field.validate(cfg.getField(field.getName()).get(cfg), cfg) if error: errors.append("'%s': %s" % (field.getName(), error)) if errors: raise PloneMeetingError( MEETING_CONFIG_ERROR % (safe_unicode(cfg.Title()), cfg.getId(), u'\n'.join(errors))) if not configData.active: self.portal.portal_wokflow.doActionFor(cfg, 'deactivate') # Adds the sub-objects within the config: categories, classifiers, items in config, ... for descr in configData.categories: self.addCategory(cfg, descr) for descr in configData.classifiers: self.addCategory(cfg, descr, classifier=True) for descr in configData.recurringItems: self.addItemToConfig(cfg, descr) for descr in configData.itemTemplates: self.addItemToConfig(cfg, descr, isRecurring=False) # configure ContentCategoryGroups before adding annex_types # this will enable confidentiality, to_sign/signed, ... if necessary for category_group_id, attrs in configData.category_group_activated_attrs.items( ): category_group = cfg.annexes_types.get(category_group_id) for attr in attrs: if not hasattr(category_group, attr): raise PloneMeetingError( 'Attribute {0} does not exist on {1} of {2}'.format( attr, category_group_id, cfgId)) setattr(category_group, attr, True) for descr in configData.annexTypes: self.addAnnexType(cfg, descr, source) # first create style templates so it exist before being used by a pod template for descr in configData.styleTemplates: self.addPodTemplate(cfg, descr, source) for descr in configData.podTemplates: self.addPodTemplate(cfg, descr, source) # manage MeetingManagers groupsTool = self.portal.portal_groups for userId in configData.meetingManagers: groupsTool.addPrincipalToGroup( userId, '{0}_{1}'.format(cfg.getId(), MEETINGMANAGERS_GROUP_SUFFIX)) # manage annex confidentiality, enable it on relevant CategoryGroup if configData.itemAnnexConfidentialVisibleFor: cfg.annexes_types.item_annexes.confidentiality_activated = True cfg.annexes_types.item_decision_annexes.confidentiality_activated = True if configData.adviceAnnexConfidentialVisibleFor: cfg.annexes_types.advice_annexes.confidentiality_activated = True if configData.meetingAnnexConfidentialVisibleFor: cfg.annexes_types.meeting_annexes.confidentiality_activated = True return cfg
def _finishConfigFor(self, cfg, data): """When the MeetingConfig has been created, some parameters still need to be applied because they need the MeetingConfig to exist.""" # apply the meetingTopicStates to the 'searchnotdecidedmeetings' DashboardCollection updateCollectionCriterion( cfg.searches.searches_meetings.searchnotdecidedmeetings, 'review_state', list(data.meetingTopicStates)) # apply the maxDaysDecisions to the 'searchlastdecisions' DashboardCollection updateCollectionCriterion( cfg.searches.searches_decisions.searchlastdecisions, 'meeting_date', unicode(data.maxDaysDecisions)) # apply the decisionTopicStates to the 'searchlastdecisions' # and 'searchalldecision' DashboardCollections updateCollectionCriterion( cfg.searches.searches_decisions.searchlastdecisions, 'review_state', list(data.decisionTopicStates)) # select correct default view meetingAppDefaultView = data.meetingAppDefaultView if meetingAppDefaultView in cfg.searches.searches_items.objectIds(): cfg._set_default_faceted_search(meetingAppDefaultView) else: error = 'meetingAppDefaultView : No DashboardCollection with id %s' % meetingAppDefaultView raise PloneMeetingError(MEETING_CONFIG_ERROR % (cfg.Title(), cfg.getId(), error)) # now we can set values for dashboard...Filters fields as the 'searches' folder has been created for fieldName in ('dashboardItemsListingsFilters', 'dashboardMeetingAvailableItemsFilters', 'dashboardMeetingLinkedItemsFilters'): field = cfg.getField(fieldName) # we want to validate the vocabulay, as if enforceVocabulary was True error = field.validate_vocabulary( cfg, cfg.getField(field.getName()).get(cfg), {}) if error: raise PloneMeetingError(MEETING_CONFIG_ERROR % (cfg.Title(), cfg.getId(), error)) if data.addContactsCSV: output = import_contacts(self.portal, path=self.profilePath) logger.info(output) selectableOrderedContacts = cfg.getField( 'orderedContacts').Vocabulary(cfg).keys() cfg.setOrderedContacts(selectableOrderedContacts) # turn contact path to uid for org_storing_field in ('orderedContacts', ): org_storing_data = getattr(data, org_storing_field, []) if org_storing_data: contact_uids = [] for contact_path in org_storing_data: try: contact_uid = self.portal.contacts.restrictedTraverse( contact_path).UID() contact_uids.append(contact_uid) except KeyError: logger.warning( 'While computing "{0}", could not get contact at "{1}"' .format(org_storing_field, contact_path)) cfg.getField(org_storing_field).set(cfg, contact_uids) # set default labels if data.defaultLabels: jar = ILabelJar(cfg) update_labels_jar(jar, values=data.defaultLabels) # disable relevant dashboard collections for collection_path in data.disabled_collections: try: collection = cfg.searches.restrictedTraverse(collection_path) except AttributeError: logger.warning( 'While disabling collections, no collection found at {0}, ...' .format(collection_path)) collection.enabled = False collection.reindexObject(idxs=['enabled'])
def initialize_proposingGroupWithGroupInCharge( self, default_in_charge_uid, config_ids=[], ignore_if_others=[], remove_certified_signatures=[], remove_certified_signatures_ignore_pattern=None): if not isinstance(remove_certified_signatures, list): remove_certified_signatures = [remove_certified_signatures] start_date = datetime.now() count_patched = 0 count_global = 0 set_default_in_charge_if_misssing_and_fix_certified_sign( default_in_charge_uid, remove_certified_signatures, remove_certified_signatures_ignore_pattern) item_type_names = [] if not config_ids: meeting_configs = self.portal_plonemeeting.listFolderContents() else: meeting_configs = [] for config_id in config_ids: meeting_configs.append(self.portal_plonemeeting.get(config_id)) for meeting_config in meeting_configs: set_up_meeting_config_used_items_attributes(meeting_config) item_type_names.append(meeting_config.getItemTypeName()) items = self.portal_catalog(portal_type=item_type_names) logger.info("Checking {} {}".format(len(items), item_type_names)) for brain in items: if not brain.getGroupsInCharge: formatted_gp = None item = brain.getObject() proposing_group = item.getProposingGroup(theObject=True) if proposing_group: groups_in_charge = deepcopy(proposing_group.groups_in_charge) for in_charge in groups_in_charge: if in_charge not in ignore_if_others: formatted_gp = special_format.format( item.getProposingGroup(), in_charge) item.setProposingGroupWithGroupInCharge(formatted_gp) break if not formatted_gp: formatted_gp = special_format.format( item.getProposingGroup(), item.getGroupsInCharge(fromOrgIfEmpty=True, first=True), ) item.setProposingGroupWithGroupInCharge(formatted_gp) item.reindexObject(idxs=["getGroupsInCharge"]) item.updateLocalRoles() count_patched += 1 count_global += 1 if count_global % 200 == 0: logger.info("Checked {} / {} items. Patched {} of them".format( count_global, len(items), count_patched, )) # save what's already done if count_patched % 10000 == 0: transaction.commit() end_date = datetime.now() seconds = end_date - start_date seconds = seconds.seconds hours = seconds / 3600 minutes = (seconds - hours * 3600) / 60 logger.info(u"Completed in {0} seconds (about {1} h {2} m).".format( seconds, hours, minutes)) if count_patched > 0: ratio = count_patched / seconds logger.info(u"That's %2.2f items patched per second" % ratio)
def run(self): member = self.portal.portal_membership.getAuthenticatedMember() if not member.has_role("Manager"): raise ValueError("You must be a Manager to access this script !") # Load all csv into memory cfg_groups = get_organizations(only_selected=False) for group in cfg_groups: self.grp_id_mapping[group.UID()] = group logger.info("Load {0}".format(self.f_group_mapping)) with io.open(self.f_group_mapping, "r") as csvfile: reader = csv.reader(csvfile) for row in reader: grp_id = row[1].strip() if grp_id in self.grp_id_mapping: self.groups[ row[0].strip()] = self.grp_id_mapping[grp_id].UID() else: self.groups[row[0].strip()] = self.default_group meetings = {} logger.info("Load {0}".format(self.f_meetings)) with io.open(self.f_meetings, "r") as csvfile: reader = csv.reader(csvfile) for row in reader: if reader.line_num == 1: # skip header line continue # Because numbers are not numbers but unicode chars... external_id = int(row[0].strip()) csv_type = safe_unicode(row[6].strip()).lower() if 'col' in csv_type: portal_type = self.college_cfg.getMeetingTypeName() elif 'cons' in csv_type: portal_type = self.council_cfg.getMeetingTypeName() else: continue meeting = CSVMeeting(external_id=external_id, date=row[1].strip(), created_on=row[2].strip(), started_on=row[3].strip(), ended_on=row[4].strip(), assembly=safe_unicode(row[5].strip()), portal_type=portal_type, annexes_dir=self.meeting_annex_dir_path) self.add_meeting_to_dict(meetings, meeting) self.load_items(self.f_items, meetings) # insert All self.disable_recurring_items() logger.info("Inserting Objects") try: for csv_meeting in meetings.values(): if csv_meeting.portal_type == self.college_cfg.getMeetingTypeName( ): self.insert_and_close_meeting(self.college_member_folder, csv_meeting) elif csv_meeting.portal_type == self.council_cfg.getMeetingTypeName( ): self.insert_and_close_meeting(self.council_member_folder, csv_meeting) else: raise NotImplementedError( u"Not managed meeting type '{}' for meeting id {}". format(csv_meeting.type, meeting.external_id)) finally: tool = api.portal.get_tool('portal_plonemeeting') self.re_enable_recurring_items() tool.invalidateAllCache() return self.meeting_counter, self.item_counter, self.errors
def commit_and_print_state(self): transaction.commit() message = "Transaction commited. {}".format(self.treated_count) logger.info(message)