def evolve(root): logger.info( 'Running substanced evolve step 7: reset all blob mimetypes ' 'to nominal USE_MAGIC value' ) if magic: objectmap = find_objectmap(root) if objectmap is not None: oids = objectmap.get_extent(get_dotted_name(File)) if oids is not None: for oid in oids: f = objectmap.object_for(oid) try: if f.get_size(): blob = f.blob fp = blob.open('r') for chunk in chunks(fp): m = magic.Magic(mime=True) mimetype = m.from_buffer(chunk) f.mimetype = mimetype break except POSKeyError: logger.error( 'Missing blob for file %s, overwriting with ' 'empty blob' % resource_path(f) ) f.blob = Blob() f.mimetype = 'application/octet-stream'
def evolve(root, registry): logger.info('Running substanced evolve step 7: reset all blob mimetypes ' 'to nominal USE_MAGIC value') if magic: objectmap = find_objectmap(root) if objectmap is not None: oids = objectmap.get_extent(get_dotted_name(File)) if oids is not None: for oid in oids: f = objectmap.object_for(oid) try: if f.get_size(): blob = f.blob fp = blob.open('r') for chunk in chunks(fp): m = magic.Magic(mime=True) mimetype = m.from_buffer(chunk) f.mimetype = mimetype break except POSKeyError: logger.error( 'Missing blob for file %s, overwriting with ' 'empty blob' % resource_path(f)) f.blob = Blob() f.mimetype = 'application/octet-stream'
def main(): parser = OptionParser(description=__doc__) parser.add_option('-d', '--dry-run', dest='dry_run', action="store_true", default=False, help="Don't commit the transactions") parser.add_option('-i', '--interval', dest='commit_interval', action="store", default=3000, help="Commit every N transactions") parser.add_option('-p', '--path', dest='path', action="store", default=None, metavar='EXPR', help="Reindex only objects whose path matches a regular expression") parser.add_option('-n', '--index', dest='indexes', action="append", help="Reindex only the given index (can be repeated)") parser.add_option('-c', '--catalog', dest='catalog_specs', action="append", help=("Reindex only the catalog provided (may be a path or a name " "and may be specified multiple times)")) options, args = parser.parse_args() if args: config_uri = args[0] else: parser.error("Requires a config_uri as an argument") commit_interval = int(options.commit_interval) if options.path: path_re = re.compile(options.path) else: path_re = None kw = {} if options.indexes: kw['indexes'] = options.indexes setup_logging(config_uri) env = bootstrap(config_uri) site = env['root'] registry = env['registry'] kw['registry'] = registry objectmap = find_objectmap(site) catalog_oids = objectmap.get_extent(get_dotted_name(Catalog)) for oid in catalog_oids: catalog = objectmap.object_for(oid) path = resource_path(catalog) if options.catalog_specs: if ( (not path in options.catalog_specs) and (not catalog.__name__ in options.catalog_specs) ): continue catalog.reindex(path_re=path_re, commit_interval=commit_interval, dry_run=options.dry_run, **kw)
def treesetify_catalog_pathindexes(root): # pragma: no cover # to avoid having huge pickles objectmap = find_objectmap(root) index_oids = objectmap.get_extent(get_dotted_name(PathIndex)) for oid in index_oids: pathindex = objectmap.object_for(oid) pathindex._not_indexed = objectmap.family.IF.TreeSet(pathindex._not_indexed)
def treesetify_catalog_pathindexes(root): # pragma: no cover # to avoid having huge pickles objectmap = find_objectmap(root) index_oids = objectmap.get_extent(get_dotted_name(PathIndex)) for oid in index_oids: pathindex = objectmap.object_for(oid) pathindex._not_indexed = objectmap.family.IF.TreeSet( pathindex._not_indexed)
def evolve(root, registry): logger.info( 'Running substanced evolve step 6: files should not have USE_MAGIC ' 'as a mimetype or any other non-string value') objectmap = find_objectmap(root) if objectmap is not None: oids = objectmap.get_extent(get_dotted_name(File)) if oids is not None: for oid in oids: f = objectmap.object_for(oid) if not type(f.mimetype) in string_types: f.mimetype = 'application/octet-stream'
def to_dotted_name(context) -> str: """Get the dotted name of `context`. :returns: The dotted name of `context`, if it's a type. If `context` is a string it is returned as is (since we suppose that it already represents a type name). """ if isinstance(context, str): return context else: return get_dotted_name(context)
def evolve(root): logger.info( 'Running substanced evolve step 6: files should not have USE_MAGIC ' 'as a mimetype or any other non-string value' ) objectmap = find_objectmap(root) if objectmap is not None: oids = objectmap.get_extent(get_dotted_name(File)) if oids is not None: for oid in oids: f = objectmap.object_for(oid) if not type(f.mimetype) in string_types: f.mimetype = 'application/octet-stream'
def evolve(root): logger.info( 'Running substanced evolve step 5: remove None as default for index ' 'action mode (MODE_ATCOMMIT should be implicit default)') objectmap = find_objectmap(root) if objectmap is not None: oids = objectmap.get_extent(get_dotted_name(Catalog)) for oid in oids: catalog = objectmap.object_for(oid) if catalog is not None: for index in catalog.values(): # wake up index via getattr first if (index.action_mode is None and 'action_mode' in index.__dict__): del index.action_mode
def evolve(root): logger.info( 'Running substanced evolve step 9: reindex all allowed indices ' 'due to change in discriminator principal repr') site = root objectmap = find_objectmap(site) index_oids = objectmap.get_extent(get_dotted_name(AllowedIndex)) for oid in index_oids: index = objectmap.object_for(oid) catalog = find_interface(index, Catalog) catalog.reindex(indexes=(index.__name__, ))
def evolve(root): logger.info( 'Running substanced evolve step 5: remove None as default for index ' 'action mode (MODE_ATCOMMIT should be implicit default)' ) objectmap = find_objectmap(root) if objectmap is not None: oids = objectmap.get_extent(get_dotted_name(Catalog)) for oid in oids: catalog = objectmap.object_for(oid) if catalog is not None: for index in catalog.values(): # wake up index via getattr first if (index.action_mode is None and 'action_mode' in index.__dict__): del index.action_mode
def evolve(root): logger.info( 'Running substanced evolve step 9: reindex all allowed indices ' 'due to change in discriminator principal repr' ) site = root objectmap = find_objectmap(site) index_oids = objectmap.get_extent(get_dotted_name(AllowedIndex)) for oid in index_oids: index = objectmap.object_for(oid) catalog = find_interface(index, Catalog) catalog.reindex(indexes=(index.__name__,))
def serialize(self, node, value): """Serialize interface to dotted name.""" if value in (colander.null, ''): return value return get_dotted_name(value)
def get_dotted_name(self, obj): return get_dotted_name(obj)
def iface_representer(dumper, data): return dumper.represent_scalar(u('!interface'), get_dotted_name(data))
def serialize(self, node, value): """Serialize interface to dotted name.""" if value in (colander.null, ""): return value return get_dotted_name(value)