def create_workflow(self, workflow_id, dataflow, is_persist):
workflow_obj = Workflow(workflow_id, dataflow)
# Picks out unique component names from all components mentioned in dataflow object
component_names = list(
set(
list(dataflow.keys()) + [
component_name for sublist in dataflow.values()
for component_name in sublist
]))
# TODO: Verify dataflow - components, converters
name_to_component = {}
for component_name in component_names:
# Skip for dataflow keywords
if component_name in ["root"]:
continue
component = None
# Reuse component if persist
if is_persist and component_name in self.persist_name_to_component:
component = self.persist_name_to_component[component_name]
# Components that are pre-deployed as persistant
if component_name in ["mongodb"]:
component = self.persist_name_to_component[component_name]
# Deploy component for non-persist case or first time persist
if component is None:
component = Component(component_name, is_persist,
self.http_session)
component.deploy(self.swarm_client)
if is_persist:
self.persist_name_to_component[component_name] = component
# Add component to map
name_to_component[component_name] = component
workflow_obj.name_to_component = name_to_component
self.workflow_id_to_obj[workflow_id] = workflow_obj
def __init__(self):
self.swarm_client = docker.from_env()
self.http_session = self._create_http_session()
self.persist_name_to_component = {}
self.workflow_id_to_obj = {}
self.request_id_to_component_data = {}
# Starting mongodb on workflow manager startup
component = Component("mongodb", True, None)
component.deploy(self.swarm_client,
mounts=[
"mongodb_mongo-data-1:/data/db",
"mongodb_mongo-config-1:/data/configdb"
])
self.persist_name_to_component["mongodb"] = component
mongo_url = "mongodb://10.176.67.87:{port}".format(
port=component.target_port)
component.client = MongoClient(mongo_url)
