def __init__(self, channel_dict):
"""
:type channel_dict: :obj:`dict`
:arg channel_dict: channel configuration
"""
logging.getLogger("decision_engine").debug('Creating channel source')
self.sources = _make_workers_for(channel_dict['sources'], Source)
logging.getLogger("decision_engine").debug(
'Creating channel transform')
self.transforms = _make_workers_for(channel_dict['transforms'],
Transform)
logging.getLogger("decision_engine").debug(
'Creating channel logicengine')
self.le_s = _make_workers_for(channel_dict['logicengines'],
LogicEngine)
logging.getLogger("decision_engine").debug(
'Creating channel publisher')
self.publishers = _make_workers_for(channel_dict['publishers'],
Publisher)
self.task_manager = channel_dict.get('task_manager', {})
ensure_no_circularities(self.sources, self.transforms, self.publishers)
def test_ensure_no_circularities_correct_order():
sources = {"source": {"produces": ["a", "b"]}}
transforms = {
"b_do_first": {
"consumes": ["a"],
"produces": ["c"]
},
"a_do_second": {
"consumes": ["b", "c"],
"produces": ["d"]
},
}
publishers = {"pub": {"consumes": ["d"]}}
sorted_transforms = ensure_no_circularities(sources, transforms,
publishers)
assert list(sorted_transforms.keys()) == ["b_do_first", "a_do_second"]
def __init__(self, channel_dict, channel_name):
"""
:type channel_dict: :obj:`dict`
:arg channel_dict: channel configuration
"""
delogger.debug("Creating channel source")
self.sources = _make_workers_for(channel_dict["sources"], Source,
channel_name)
delogger.debug("Creating channel logicengine")
self.le_s = _make_workers_for(channel_dict["logicengines"],
LogicEngine, channel_name)
delogger.debug("Creating channel publisher")
self.publishers = _make_workers_for(channel_dict["publishers"],
Publisher, channel_name)
delogger.debug("Creating channel transform")
transforms = _make_workers_for(channel_dict["transforms"], Transform,
channel_name)
self.transforms = ensure_no_circularities(self.sources, transforms,
self.publishers)
self.task_manager = channel_dict.get("task_manager", {})
def __init__(self,
name,
channel_dict,
global_config,
de_source_workers=None):
"""
:type name: :obj:`str`
:arg name: Name of channel corresponding to this task manager
:type generation_id: :obj:`int`
:arg generation_id: Task manager generation id provided by caller
:type channel_dict: :obj:`dict`
:arg channel_dict: channel configuration
:type global_config: :obj:`dict`
:arg global_config: global configuration
"""
super().__init__(channel_dict.get("channel_name", name))
self.id = str(uuid.uuid4()).upper()
self.dataspace = dataspace.DataSpace(global_config)
self.data_block_t0 = datablock.DataBlock(self.dataspace, name, self.id,
1) # my current data block
self.logger = structlog.getLogger(CHANNELLOGGERNAME)
self.logger = self.logger.bind(module=__name__.split(".")[-1],
channel=self.name)
self.broker_url = global_config.get("broker_url",
"redis://localhost:6379/0")
self.logger.debug(f"Using data-broker URL: {self.broker_url}")
self.logger.debug("Creating channel sources")
self.source_workers = _make_workers_for(channel_dict["sources"],
Source, self.name)
if de_source_workers is not None:
# Decision engine owns the sources
de_source_workers[self.name] = self.source_workers
self.logger.debug("Creating channel publishers")
self.publisher_workers = _make_workers_for(channel_dict["publishers"],
Publisher, self.name)
self.logger.debug("Creating channel logic engines")
configured_le_s = channel_dict.get("logicengines")
if configured_le_s is None:
self.logger.debug(
"No 'logicengines' configuration detected; will use default configuration, which unconditionally executes all configured publishers."
)
configured_le_s = passthrough_configuration(
channel_dict["publishers"].keys())
if len(configured_le_s) > 1:
raise RuntimeError(
"Cannot support more than one logic engine per channel.")
self.logic_engine = None
if configured_le_s:
key, config = configured_le_s.popitem()
self.logic_engine = Worker(key, config, LogicEngine, self.name)
self.logger.debug("Creating channel transforms")
transform_workers = _make_workers_for(channel_dict["transforms"],
Transform, self.name)
self.transform_workers = ensure_no_circularities(
self.source_workers, transform_workers, self.publisher_workers)
exchange_name = global_config.get("exchange_name",
"hepcloud_topic_exchange")
self.logger.debug(
f"Creating topic exchange {exchange_name} for channel {self.name}")
self.exchange = Exchange(exchange_name, "topic")
self.connection = Connection(self.broker_url)
expected_source_products = set()
queues = {}
for worker in self.source_workers.values():
# FIXME: Just keeping track of instance names will not
# work whenever we have multiple source instances
# of the same source type.
expected_source_products.update(
worker.module_instance._produces.keys())
self.logger.debug(
f"Creating queue {worker.full_key} with routing key {worker.full_key}"
)
queues[worker.full_key] = Queue(
worker.full_key,
exchange=self.exchange,
routing_key=worker.full_key,
auto_delete=True,
)
self.expected_source_products = expected_source_products
self.queues = queues
# Caching to determine if all sources have run at least once.
self.sources_have_run_once = False
self.source_product_cache = {}
# The rest of this function will go away once the source-proxy
# has been reimplemented.
for src_worker in self.source_workers.values():
src_worker.module_instance.post_create(global_config)