def __init__(self, working_dir: str, zone_id: str, transform_id: str):
super(BaseTransform, self).__init__()
self.working_dir = working_dir
self.zone_id = zone_id
self.transform_id = transform_id
self.kernel = None
self.warehouse_endpoint = config.get_setting(
working_dir)['warehouse_endpoint']
setting = config.get_setting(working_dir)
warehouse_dir = Path(working_dir, setting['warehouse_dir']).as_posix()
zone_storage_dir = setting['zone_storage_dir'].format(
warehouse_dir=warehouse_dir,
zone_id=zone_id,
)
self.transform_storage_dir = setting['transform_storage_dir'].format(
zone_storage_dir=zone_storage_dir, transform_id=transform_id)
self.transform_cache_dir = setting['transform_cache_dir'].format(
zone_storage_dir=zone_storage_dir, transform_id=transform_id)
Path(self.transform_cache_dir).parent.mkdir(parents=1, exist_ok=1)
self.transform_cache_db = plyvel.DB(self.transform_cache_dir,
create_if_missing=True)
self.logger = get_child_logger('aggregator.transform')
def __init__(self, working_dir: str, zone_id: str):
super(BaseStorage, self).__init__()
self.working_dir = working_dir
self.zone_id = zone_id
setting = config.get_setting(working_dir)
self.warehouse_dir = Path(working_dir,
setting['warehouse_dir']).as_posix()
self.zone_storage_dir = setting['zone_storage_dir'].format(
warehouse_dir=self.warehouse_dir,
zone_id=zone_id,
)
transforms = zone_manager.load_zone(
zone_id, working_dir)['aggregator']['transform_registry']
self.transform_storage_dirs = {
tid: setting['transform_storage_dir'].format(
zone_storage_dir=self.zone_storage_dir, transform_id=tid)
for tid in transforms
}
# Setup storage DB for all transforms
for p in self.transform_storage_dirs.values():
Path(p).parent.mkdir(parents=1, exist_ok=1)
self.transform_storage_dbs = {
tid: plyvel.DB(self.transform_storage_dirs[tid],
create_if_missing=True)
for tid in transforms
}
self.logger = get_child_logger('warehouse.storage')
def __init__(self, working_dir: str, zone_id: str):
super(BaseApiBundle, self).__init__()
self.working_dir = working_dir
self.zone_id = zone_id
self.collator = None
self.logger = get_child_logger('provider.api_bundle')
def __init__(self, working_dir: str, zone_id: str):
super(BaseCollator, self).__init__()
self.working_dir = working_dir
self.zone_id = zone_id
self.warehouse_endpoint = config.get_setting(
working_dir)['warehouse_endpoint']
self.logger = get_child_logger('provider.collator')
def __init__(self, working_dir: str, zone_id: str):
super(BaseDataFeeder, self).__init__()
self.working_dir = working_dir
self.zone_id = zone_id
self.zone = zone_manager.get_zone(working_dir, zone_id)
self.direct_db_access = self.zone['direct_db_access']
self.logger = get_child_logger('upstream.data_feeder')
def __init__(self, working_dir: str, zone_id: str):
super(BaseKernel, self).__init__()
self.working_dir = working_dir
self.zone_id = zone_id
self.transforms = {}
self.chain_registry = config.get_chain_registry(working_dir)
self.warehouse_endpoint = config.get_setting(working_dir)['warehouse_endpoint']
self.logger = get_child_logger('aggregator.kernel')