def _update(self, portal): portal_properties = portal.getPropertiesTool() global_subobjects = set(portal.adt_meta_types) all_nyfolders = ofs_walk(portal, [INyFolder], [IFolder]) patched = 0 customized = 0 for nyfolder in all_nyfolders: if global_subobjects != set(nyfolder.folder_meta_types): # subobjects are different, they were prev. modified # keep the folder_meta_types list as it is customized += 1 else: # same with global, mark it with `Uses default` (None) fmt = FolderMetaTypes(nyfolder) fmt.set_values(None) patched += 1 self.log.info(("%d folders now use default subobjects setting, " "%d folders have subobject customizations") % (patched, customized)) if not add_admin_entry(self, portal, ("""<li tal:condition="canPublish"><a """ """tal:attributes="href string:${site_url}/""" """admin_folder_subobjects_html" title""" """="Portal comments" i18n:attributes="title" """ """i18n:translate="">Folder subobjects</a></li>"""), '${site_url}/admin_maintopics_html"'): self.log.error("MANUAL action: Insert link html in admin portlet") return False else: return True return True
def test_walk_basic(self): from naaya.core.zope2util import ofs_walk fol = Folder('fol').__of__(Folder('root')) self.assertEqual(list(ofs_walk(fol)), []) item = SimpleItem('item') fol._setObject('item', item) self.assertEqual(list(ofs_walk(fol)), [item]) fol2 = Folder('fol2') fol._setObject('fol2', fol2) self.assertEqual(list(ofs_walk(fol)), [item, fol2]) item2 = SimpleItem('item2') fol2._setObject('item2', item2) self.assertEqual(list(ofs_walk(fol)), [item, fol2, item2])
def test_always_traverse_root(self): from naaya.core.zope2util import ofs_walk fol = Folder('fol').__of__(Folder('root')) fol2 = Folder('fol2') item = SimpleItem('item') item2 = SimpleItem('item2') fol._setObject('item', item) fol._setObject('fol2', fol2) fol2._setObject('item2', item2) self.assertEqual(list(ofs_walk(fol, containers=[])), [item, fol2])
def _update(self, portal): all_objects = itertools.chain([portal], ofs_walk(portal)) # this class is not used anymore: no_trans = 0 for obj in all_objects: # Part 0: if broken, report it if isinstance(obj, BrokenClass): self.log.error(("Object %r is broken! Unable to clean local" " properties, if any ") % obj) continue # Part 1: normalize representation of local properties _local_properties = getattr(obj, '_local_properties', None) if _local_properties is not None: for (property, trans) in _local_properties.items(): delete_keys = set() for (lang, translation) in trans.items(): if not lang and translation[0]: delete_keys.add(lang) en_trans = obj.getLocalProperty(property, 'en') if not en_trans: obj.set_localpropvalue(property, 'en', translation[0]) self.log.info( "Moved to 'En' translation for empty lang %s - %s: '%s'" % (obj.absolute_url(), property, translation[0])) else: self.log.info( "Removed translation for empty lang %s - %s: '%s', Keeping: '%s' for En" % (obj.absolute_url(), property, translation[0], en_trans)) if not translation[0]: delete_keys.add(lang) if len(delete_keys): no_trans += len(delete_keys) obj._p_changed = 1 for key in delete_keys: del trans[key] self.log.debug('%d empty translations removed' % no_trans) return True
def test_restrict_traversal(self): from naaya.core.zope2util import ofs_walk fol = Folder('fol').__of__(Folder('root')) fol2 = Folder('fol2') item = SimpleItem('item') item2 = SimpleItem('item2') fol._setObject('item', item) fol._setObject('fol2', fol2) fol2._setObject('item2', item2) class IMyContainer(interface.Interface): pass class IMyCont2(interface.Interface): pass # we call with 2 interfaces to make sure "OR" logic is applied walk = lambda: list(ofs_walk(fol, containers=[IMyContainer, IMyCont2])) self.assertEqual(walk(), [item, fol2]) interface.alsoProvides(fol2, IMyContainer) self.assertEqual(walk(), [item, fol2, item2])
def _update(self, portal): portal_properties = portal.getPropertiesTool() global_subobjects = set(portal.adt_meta_types) all_nyfolders = ofs_walk(portal, [INyFolder], [IFolder]) patched = 0 customized = 0 for nyfolder in all_nyfolders: if global_subobjects != set(nyfolder.folder_meta_types): # subobjects are different, they were prev. modified # keep the folder_meta_types list as it is customized += 1 else: # same with global, mark it with `Uses default` (None) fmt = FolderMetaTypes(nyfolder) fmt.set_values(None) patched += 1 self.log.info( ("%d folders now use default subobjects setting, " "%d folders have subobject customizations") % (patched, customized) ) if not add_admin_entry( self, portal, ( """<li tal:condition="canPublish"><a """ """tal:attributes="href string:${site_url}/""" """admin_folder_subobjects_html" title""" """="Portal comments" i18n:attributes="title" """ """i18n:translate="">Folder subobjects</a></li>""" ), '${site_url}/admin_maintopics_html"', ): self.log.error("MANUAL action: Insert link html in admin portlet") return False else: return True return True
def _update(self, portal): all_objects = itertools.chain([portal], ofs_walk(portal)) # this class is not used anymore: no_trans = 0 for obj in all_objects: # Part 0: if broken, report it if isinstance(obj, BrokenClass): self.log.error(("Object %r is broken! Unable to clean local" " properties, if any ") % obj) continue # Part 1: normalize representation of local properties _local_properties = getattr(obj, '_local_properties', None) if _local_properties is not None: for (property, trans) in _local_properties.items(): delete_keys = set() for (lang, translation) in trans.items(): if not lang and translation[0]: delete_keys.add(lang) en_trans = obj.getLocalProperty(property, 'en') if not en_trans: obj.set_localpropvalue(property, 'en', translation[0]) self.log.info("Moved to 'En' translation for empty lang %s - %s: '%s'" % (obj.absolute_url(), property, translation[0])) else: self.log.info("Removed translation for empty lang %s - %s: '%s', Keeping: '%s' for En" % (obj.absolute_url(), property, translation[0], en_trans)) if not translation[0]: delete_keys.add(lang) if len(delete_keys): no_trans += len(delete_keys) obj._p_changed = 1 for key in delete_keys: del trans[key] self.log.debug('%d empty translations removed' % no_trans) return True
def _update(self, portal): """ Summary of update: * test for portal_i18n existance, if true, skip portal * get languages and default language * create portal_i18n, place it in portal * copy message translations * fix localized properties * delete Localizer and portal_translations """ #if Localizer is None: # self.log.error('Migration unavailable when edw-localizer' # ' not installed') # return False if isinstance(portal.getPortalI18n(), NaayaI18n): self.log.debug( "Portal already uses naaya.i18n, skipping i18n init") localizer = None else: self.log.debug( "Creating portal_i18n and copying message catalog data") localizer = portal._getOb('Localizer', None) portal_trans = portal._getOb('portal_translations') if localizer is None: self.log.error("Localizer not found") return False if portal_trans is None: self.log.error("Portal Translations not found") return False languages = [(x, localizer.get_language_name(x)) for x in localizer.get_languages()] def_lang = localizer.get_default_language() self.log.debug('Found languages: %r, default: %s', languages, def_lang) manage_addNaayaI18n(portal, languages) portal.getPortalI18n().manage_changeDefaultLang(def_lang) message_cat = portal.getPortalI18n().get_message_catalog() (msg_cnt, trans_cnt) = (0, 0) for (msgid, trans) in portal_trans._messages.items(): if isinstance(msgid, str): msgid = force_to_unicode(msgid) # one call to gettext, to add 'en' identical translation, if missing message_cat.gettext(msgid, def_lang) msg_cnt += 1 for lang in trans: found = message_cat.gettext(msgid, lang, '') if lang != 'note': trans_cnt += 1 if isinstance(trans[lang], unicode): translation = trans[lang] elif isinstance(trans[lang], str): translation = force_to_unicode(trans[lang]) else: self.log.error(("Unacceptable type '%s' found for " "translation") % type(trans[lang])) self.log.error("Migration cancelled") return False if translation != found: message_cat.edit_message(msgid, lang, translation) self.log.debug('%d iterated, a total of %d translation mappings.' % (msg_cnt, trans_cnt)) self.log.debug('Message Catalog now counts %d entries (msgid-s).' % len(message_cat._messages.keys())) # Clean up and delete localizer localizer.manage_beforeDelete(localizer, portal) portal._delObject('Localizer') portal._delObject('portal_translations') self.log.debug('Localizer and Portal translations removed') # Fix local properties: # * remove translations with None-index (instead of language code) # * add existent properties in _local_properties_metadata # * remove blank/emptystring translations # Clean up any LocalAttribute-s on instances, if attribute not # present in class and superclasses or present but is LocalAttribute # key: class, value: attrs on class that are not LocalAttribute # their overrides need to be kept lookup_cache = {} localprops_del_cnt = 0 localprops_keep_cnt = 0 total_cnt = 0 all_objects = itertools.chain([portal], ofs_walk(portal)) # this class is not used anymore: for obj in all_objects: # Part 0.0: remove unused contenttype classes on sites: if INySite.providedBy(obj): if '_contenttypes_tool__contenttype_dictionary' in obj.__dict__: del obj.__dict__[ '_contenttypes_tool__contenttype_dictionary'] obj._p_changed = 1 # Part 0.1: if broken, report it # if other localizer in NySite child, skip it if isinstance(obj, BrokenClass): self.log.error(("Object %r is broken! Unable to fix local" " properties, if any ") % obj) continue if isinstance(obj, Localizer) and obj != localizer: continue # Part 1: delete unnecessary LocalAttributes on instances if obj.__dict__.get('_languages') is not None: del obj._languages if obj.__dict__.get('_default_language') is not None: del obj._default_language for (key, value) in obj.__dict__.items(): if isinstance(value, LocalAttribute): if not requires_localproperty(obj, key): self.log.debug("Deleting LocalAttribute: %r.%s", obj, key) delattr(obj, key) localprops_del_cnt += 1 else: self.log.debug("Keeping LocalAttribute: %r.%s", obj, key) setattr(obj, key, NewLocalAttribute(key)) localprops_keep_cnt += 1 # Part 2: normalize representation of local properties _local_properties = getattr(obj, '_local_properties', None) if _local_properties is not None: for (property, trans) in _local_properties.items(): if property not in obj._local_properties_metadata: obj.set_localproperty(property, 'string') delete_keys = set() for (lang, translation) in trans.items(): if not translation[0]: delete_keys.add(lang) if len(delete_keys): for key in delete_keys: del trans[key] obj._p_changed = 1 self.log.debug("%d LocalAttribute-s deleted from OFS" % localprops_del_cnt) self.log.debug("%d LocalAttribute-s kept in OFS" % localprops_keep_cnt) self.log.debug('Migration is complete!') return True
def walk_backup(index_file, open_backup_file, get_date, actor): folders_info = {'root_path': '', 'known_folders': {'': None}} gen_existing = ofs_walk(actor.context, [INyFolder], [INyFolder]) for folder in gen_existing: rel_path = relative_object_path(folder, actor.context) folders_info['known_folders'][rel_path] = None def remove_root_path(path): assert path.startswith(folders_info['root_path']) result = path[len(folders_info['root_path']):] if result.startswith('/'): result = result[1:] return result def handle_folder(line): title = line['TITLE'] description = line['ABSTRACT'] userid = parse_userid(line['OWNER']) folder_zip_path = line['FILENAME'][:-1].encode('utf-8') # for zope replace starting underscores folder_zope_path = sanitize_folder_path(folder_zip_path) # use get_date as a backup try: date = parse_date(line['CREATED']) except ValueError: date = get_date(folder_zip_path) if '/' in folder_zope_path: parent_path, folder_id = folder_zope_path.rsplit('/', 1) else: parent_path = '' folder_id = folder_zope_path if len(folders_info['known_folders']) == 1: assert folders_info['root_path'] == '' if parent_path: folders_info['root_path'] = parent_path else: folders_info['root_path'] = '' parent_path = remove_root_path(parent_path) folder_zope_path = remove_root_path(folder_zope_path) assert parent_path in folders_info['known_folders'] if folder_zope_path in folders_info['known_folders']: assert line['CREATED'] == folders_info['known_folders'][folder_zope_path]['CREATED'] assert line['OWNER'] == folders_info['known_folders'][folder_zope_path]['OWNER'] return folders_info['known_folders'][folder_zope_path] = line actor.folder_entry(parent_path, folder_id, title, description, date, userid) def handle_file(line): title = line['TITLE'] description = line['ABSTRACT'] userid = parse_userid(line['OWNER']) keywords = line['KEYWORDS'] reference = line.get('REFERENCE', '') status = line['STATUS'] doc_zip_path = line['FILENAME'] # for zope replace starting underscores doc_zope_path = sanitize_folder_path(doc_zip_path) # use get_date as a backup try: date = parse_date(line['UPLOADDATE']) except ValueError: date = get_date(doc_zip_path) doc_split_path = doc_zope_path.split('/') doc_filename = doc_split_path[-1].encode('utf-8') doc_langver = doc_split_path[-2] doc_dpl_name = str(doc_split_path[-3]) parent_path = '/'.join(doc_split_path[:-3]).encode('utf-8') parent_path = remove_root_path(parent_path) assert parent_path in folders_info['known_folders'] doc_id = doc_dpl_name[:-len('.dpl')] doc_id = sanitize_id(doc_id) full_path = parent_path+'/'+doc_id if not doc_langver.startswith('EN_'): actor.warn('non-english content: %r at %r' % (doc_langver, full_path)) if line['RANKING'] != 'Public': actor.warn('ranking is %r for %r' % (str(line['RANKING']), full_path)) if description.lower() == 'n/a': description = '' if status.lower() == 'n/a': status = '' if reference.lower() == 'n/a': reference = '' if status not in ('Draft', ''): description = ( ("<p>Status: %s</p>\n" % status) + description) if reference: description = ( ("<p>Reference: %s</p>\n" % reference) + description) doc_data_file = open_backup_file(doc_zip_path.encode('latin-1')) if doc_filename.endswith('.url'): url = doc_data_file.read().strip() if '\n' in url: matched = re.search(r'URL=(.*)', url) if matched: url = matched.groups()[0].strip() assert url.startswith('http://') or url.startswith('https://') or url.startswith('ftp://'), "bad url: %r" % url actor.url_entry(parent_path, doc_id, doc_filename, url, title, description, keywords, date, userid) else: actor.document_entry(parent_path, doc_id, doc_filename, doc_data_file, title, description, keywords, date, userid) for line in read_index(index_file, actor.warn): filename = line['FILENAME'] if filename.endswith('/'): handle_folder(line) else: handle_file(line)
def walk_backup(index_file, open_backup_file, get_date, actor): folders_info = {'root_path': '', 'known_folders': {'': None}} gen_existing = ofs_walk(actor.context, [INyFolder], [INyFolder]) for folder in gen_existing: rel_path = relative_object_path(folder, actor.context) folders_info['known_folders'][rel_path] = None def remove_root_path(path): assert path.startswith(folders_info['root_path']) result = path[len(folders_info['root_path']):] if result.startswith('/'): result = result[1:] return result def handle_folder(line): title = line.get('TITLE', line['Title']) description = line.get('ABSTRACT', line['Abstract']) userid = parse_userid(line.get('OWNER', line['Owner'])) filename = line.get('FILENAME', line['Filename']) folder_zip_path = filename[:-1].encode('utf-8') # for zope replace starting underscores folder_zope_path = sanitize_folder_path(folder_zip_path) # use get_date as a backup try: date = parse_date(line.get('CREATED', line['Created'])) except ValueError: date = get_date(folder_zip_path) if '/' in folder_zope_path: parent_path, folder_id = folder_zope_path.rsplit('/', 1) else: parent_path = '' folder_id = folder_zope_path if len(folders_info['known_folders']) == 1: assert folders_info['root_path'] == '' if parent_path: folders_info['root_path'] = parent_path else: folders_info['root_path'] = '' parent_path = remove_root_path(parent_path) folder_zope_path = remove_root_path(folder_zope_path) assert parent_path in folders_info['known_folders'] if folder_zope_path in folders_info['known_folders']: created = line.get('CREATED', line['Created']) owner = line.get('OWNER', line['Owner']) folder_info = folders_info['known_folders'][folder_zope_path] assert created == folder_info.get('CREATED', folder_info['Created']) assert owner == folder_info.get('OWNER', folder_info['Owner']) return folders_info['known_folders'][folder_zope_path] = line actor.folder_entry(parent_path, folder_id, title, description, date, userid) def handle_file(line): title = line.get('TITLE', line['Title']) description = line.get('ABSTRACT', line['Abstract']) userid = parse_userid(line.get('OWNER', line['Owner'])) keywords = line.get('KEYWORDS', line['Keywords']) reference = line.get('REFERENCE', line.get('Reference', '')) status = line.get('STATUS', line['Status']) doc_zip_path = line.get('FILENAME', line['Filename']) # for zope replace starting underscores doc_zope_path = sanitize_folder_path(doc_zip_path) # use get_date as a backup try: date = parse_date(line.get('UPLOADDATE', line['Uploaddate'])) except ValueError: date = get_date(doc_zip_path) doc_split_path = doc_zope_path.split('/') doc_filename = doc_split_path[-1].encode('utf-8') doc_langver = doc_split_path[-2] doc_dpl_name = str(doc_split_path[-3]) parent_path = '/'.join(doc_split_path[:-3]).encode('utf-8') parent_path = remove_root_path(parent_path) assert parent_path in folders_info['known_folders'] doc_id = doc_dpl_name[:-len('.dpl')] doc_id = sanitize_id(doc_id) full_path = parent_path+'/'+doc_id if not doc_langver.startswith('EN_'): actor.warn('non-english content: %r at %r' % (doc_langver, full_path)) ranking = line.get('RANKING', line['Ranking']) if ranking != 'Public': actor.warn('ranking is %r for %r' % (str(ranking), full_path)) if description.lower() == 'n/a': description = '' if status.lower() == 'n/a': status = '' if reference.lower() == 'n/a': reference = '' if status not in ('Draft', ''): description = (("<p>Status: %s</p>\n" % status) + description) if reference: description = (("<p>Reference: %s</p>\n" % reference) + description) doc_data_file = open_backup_file(doc_zip_path.encode('latin-1')) if doc_filename.endswith('.url'): url = doc_data_file.read().strip() if '\n' in url: matched = re.search(r'URL=(.*)', url) if matched: url = matched.groups()[0].strip() assert (url.startswith('http://') or url.startswith('https://') or url.startswith('ftp://'), "bad url: %r" % url) actor.url_entry(parent_path, doc_id, doc_filename, url, title, description, keywords, date, userid) else: actor.document_entry(parent_path, doc_id, doc_filename, doc_data_file, title, description, keywords, date, userid) for line in read_index(index_file, actor.warn): filename = line.get('FILENAME', line['Filename']) if filename.endswith('/'): handle_folder(line) else: handle_file(line)
def _update(self, portal): """ Summary of update: * test for portal_i18n existance, if true, skip portal * get languages and default language * create portal_i18n, place it in portal * copy message translations * fix localized properties * delete Localizer and portal_translations """ #if Localizer is None: # self.log.error('Migration unavailable when edw-localizer' # ' not installed') # return False if isinstance(portal.getPortalI18n(), NaayaI18n): self.log.debug("Portal already uses naaya.i18n, skipping i18n init") localizer = None else: self.log.debug("Creating portal_i18n and copying message catalog data") localizer = portal._getOb('Localizer', None) portal_trans = portal._getOb('portal_translations') if localizer is None: self.log.error("Localizer not found") return False if portal_trans is None: self.log.error("Portal Translations not found") return False languages = [ (x, localizer.get_language_name(x)) for x in localizer.get_languages() ] def_lang = localizer.get_default_language() self.log.debug('Found languages: %r, default: %s', languages, def_lang) manage_addNaayaI18n(portal, languages) portal.getPortalI18n().manage_changeDefaultLang(def_lang) message_cat = portal.getPortalI18n().get_message_catalog() (msg_cnt, trans_cnt) = (0, 0) for (msgid, trans) in portal_trans._messages.items(): if isinstance(msgid, str): msgid = force_to_unicode(msgid) # one call to gettext, to add 'en' identical translation, if missing message_cat.gettext(msgid, def_lang) msg_cnt += 1 for lang in trans: found = message_cat.gettext(msgid, lang, '') if lang != 'note': trans_cnt += 1 if isinstance(trans[lang], unicode): translation = trans[lang] elif isinstance(trans[lang], str): translation = force_to_unicode(trans[lang]) else: self.log.error(("Unacceptable type '%s' found for " "translation") % type(trans[lang])) self.log.error("Migration cancelled") return False if translation != found: message_cat.edit_message(msgid, lang, translation) self.log.debug('%d iterated, a total of %d translation mappings.' % (msg_cnt, trans_cnt)) self.log.debug('Message Catalog now counts %d entries (msgid-s).' % len(message_cat._messages.keys())) # Clean up and delete localizer localizer.manage_beforeDelete(localizer, portal) portal._delObject('Localizer') portal._delObject('portal_translations') self.log.debug('Localizer and Portal translations removed') # Fix local properties: # * remove translations with None-index (instead of language code) # * add existent properties in _local_properties_metadata # * remove blank/emptystring translations # Clean up any LocalAttribute-s on instances, if attribute not # present in class and superclasses or present but is LocalAttribute # key: class, value: attrs on class that are not LocalAttribute # their overrides need to be kept lookup_cache = {} localprops_del_cnt = 0 localprops_keep_cnt = 0 total_cnt = 0 all_objects = itertools.chain([portal], ofs_walk(portal)) # this class is not used anymore: for obj in all_objects: # Part 0.0: remove unused contenttype classes on sites: if INySite.providedBy(obj): if '_contenttypes_tool__contenttype_dictionary' in obj.__dict__: del obj.__dict__['_contenttypes_tool__contenttype_dictionary'] obj._p_changed = 1 # Part 0.1: if broken, report it # if other localizer in NySite child, skip it if isinstance(obj, BrokenClass): self.log.error(("Object %r is broken! Unable to fix local" " properties, if any ") % obj) continue if isinstance(obj, Localizer) and obj != localizer: continue # Part 1: delete unnecessary LocalAttributes on instances if obj.__dict__.get('_languages') is not None: del obj._languages if obj.__dict__.get('_default_language') is not None: del obj._default_language for (key, value) in obj.__dict__.items(): if isinstance(value, LocalAttribute): if not requires_localproperty(obj, key): self.log.debug("Deleting LocalAttribute: %r.%s", obj, key) delattr(obj, key) localprops_del_cnt += 1 else: self.log.debug("Keeping LocalAttribute: %r.%s", obj, key) setattr(obj, key, NewLocalAttribute(key)) localprops_keep_cnt += 1 # Part 2: normalize representation of local properties _local_properties = getattr(obj, '_local_properties', None) if _local_properties is not None: for (property, trans) in _local_properties.items(): if property not in obj._local_properties_metadata: obj.set_localproperty(property, 'string') delete_keys = set() for (lang, translation) in trans.items(): if not translation[0]: delete_keys.add(lang) if len(delete_keys): for key in delete_keys: del trans[key] obj._p_changed = 1 self.log.debug("%d LocalAttribute-s deleted from OFS" % localprops_del_cnt) self.log.debug("%d LocalAttribute-s kept in OFS" % localprops_keep_cnt) self.log.debug('Migration is complete!') return True