def __init__(self, root, small_max=200 * MB, large_max=1 * GB): def make_writer(d): name = dr.get_simple_name(d) if is_large(name): return LargeWriter(root, name, large_max) return Writer(root, name, small_max) def create_writers(): def partition(): small, large = set(), set() for f in os.listdir(root): name = f.split(".", 1)[0] (large if is_large(name) else small).add(name) return small, large small, large = partition() results = {s: Writer(root, s, small_max) for s in small} results.update({l: LargeWriter(root, l, large_max) for l in large}) return results ensure_dir(root) self.root = root self.small_max = small_max self.large_max = large_max self.writers = KeyPassingDefaultDict(make_writer) self.writers.update(create_writers())
class Accumulator(object): def __init__(self, root, small_max=200 * MB, large_max=1 * GB): def make_writer(d): name = dr.get_simple_name(d) if is_large(name): return LargeWriter(root, name, large_max) return Writer(root, name, small_max) def create_writers(): def partition(): small, large = set(), set() for f in os.listdir(root): name = f.split(".", 1)[0] (large if is_large(name) else small).add(name) return small, large small, large = partition() results = {s: Writer(root, s, small_max) for s in small} results.update({l: LargeWriter(root, l, large_max) for l in large}) return results ensure_dir(root) self.root = root self.small_max = small_max self.large_max = large_max self.writers = KeyPassingDefaultDict(make_writer) self.writers.update(create_writers()) def _handle_stream(self, name, stream): writer = self.writers[name] writer.write_stream(stream) writer.close() return writer def process(self, ctx, archive): for name, path, transform in ctx.process(archive): with open(path) as f: stream = transform(f) self._handle_stream(name, stream) def __iadd__(self, other): for k, them in other.writers.items(): self.writers[k] += them return self
def _import_component(name): """ Returns a class, function, or class method specified by the fully qualified name. """ for f in (_get_from_module, _get_from_class): try: return f(name) except Exception as e: log.debug("Couldn't load %s" % name) log.debug(e, exc_info=True) pass COMPONENT_IMPORT_CACHE = KeyPassingDefaultDict(_import_component) def get_component(name): """ Returns the class or function specified, importing it if necessary. """ return COMPONENT_IMPORT_CACHE[name] def _find_component(name): for d in DELEGATES: if get_name(d) == name: return d COMPONENTS_BY_NAME = KeyPassingDefaultDict(_find_component)
def _get_component(name): """ Returns a class, function, or class method specified by the fully qualified name. """ for f in (_get_from_module, _get_from_class): try: return f(name) except: pass log.debug("Couldn't load %s" % name) COMPONENT_NAME_CACHE = KeyPassingDefaultDict(_get_component) def get_component(name): """ Returns the class or function specified, importing it if necessary. """ return COMPONENT_NAME_CACHE[name] @defaults(None) def get_component_type(component): return get_delegate(component).type def get_components_of_type(_type): return COMPONENTS_BY_TYPE.get(_type)