def __init__(self, root, small_max=200 * MB, large_max=1 * GB):
def make_writer(d):
name = dr.get_simple_name(d)
if is_large(name):
return LargeWriter(root, name, large_max)
return Writer(root, name, small_max)
def create_writers():
def partition():
small, large = set(), set()
for f in os.listdir(root):
name = f.split(".", 1)[0]
(large if is_large(name) else small).add(name)
return small, large
small, large = partition()
results = {s: Writer(root, s, small_max) for s in small}
results.update({l: LargeWriter(root, l, large_max) for l in large})
return results
ensure_dir(root)
self.root = root
self.small_max = small_max
self.large_max = large_max
self.writers = KeyPassingDefaultDict(make_writer)
self.writers.update(create_writers())
class Accumulator(object):
def __init__(self, root, small_max=200 * MB, large_max=1 * GB):
def make_writer(d):
name = dr.get_simple_name(d)
if is_large(name):
return LargeWriter(root, name, large_max)
return Writer(root, name, small_max)
def create_writers():
def partition():
small, large = set(), set()
for f in os.listdir(root):
name = f.split(".", 1)[0]
(large if is_large(name) else small).add(name)
return small, large
small, large = partition()
results = {s: Writer(root, s, small_max) for s in small}
results.update({l: LargeWriter(root, l, large_max) for l in large})
return results
ensure_dir(root)
self.root = root
self.small_max = small_max
self.large_max = large_max
self.writers = KeyPassingDefaultDict(make_writer)
self.writers.update(create_writers())
def _handle_stream(self, name, stream):
writer = self.writers[name]
writer.write_stream(stream)
writer.close()
return writer
def process(self, ctx, archive):
for name, path, transform in ctx.process(archive):
with open(path) as f:
stream = transform(f)
self._handle_stream(name, stream)
def __iadd__(self, other):
for k, them in other.writers.items():
self.writers[k] += them
return self
def _import_component(name):
"""
Returns a class, function, or class method specified by the fully qualified
name.
"""
for f in (_get_from_module, _get_from_class):
try:
return f(name)
except Exception as e:
log.debug("Couldn't load %s" % name)
log.debug(e, exc_info=True)
pass
COMPONENT_IMPORT_CACHE = KeyPassingDefaultDict(_import_component)
def get_component(name):
""" Returns the class or function specified, importing it if necessary. """
return COMPONENT_IMPORT_CACHE[name]
def _find_component(name):
for d in DELEGATES:
if get_name(d) == name:
return d
COMPONENTS_BY_NAME = KeyPassingDefaultDict(_find_component)
def _get_component(name):
"""
Returns a class, function, or class method specified by the fully qualified
name.
"""
for f in (_get_from_module, _get_from_class):
try:
return f(name)
except:
pass
log.debug("Couldn't load %s" % name)
COMPONENT_NAME_CACHE = KeyPassingDefaultDict(_get_component)
def get_component(name):
""" Returns the class or function specified, importing it if necessary. """
return COMPONENT_NAME_CACHE[name]
@defaults(None)
def get_component_type(component):
return get_delegate(component).type
def get_components_of_type(_type):
return COMPONENTS_BY_TYPE.get(_type)