from flow.configuration.settings.injector import setting
from flow.util.exit import exit_process
from flow import exit_codes
from injector import inject
from twisted.internet import defer
import flow.interfaces
import logging
import time
LOG = logging.getLogger(__name__)
@inject(broker=flow.interfaces.IBroker,
binding_config=setting('bindings'))
class ConfigureRabbitMQCommand(CommandBase):
injector_modules = [
BrokerConfiguration,
]
@staticmethod
def annotate_parser(parser):
pass
def _execute(self, parsed_arguments):
exchanges, queues, bindings = self._parse_config()
LOG.debug("Parsed config")
from flow.shell_command.lsf.resource_manager import LSFResourceManager
from pythonlsf import lsf
from twisted.python.procutils import which
import injector
import logging
import os
LOG = logging.getLogger(__name__)
@injector.inject(
resource_manager=LSFResourceManager,
option_manager=LSFOptionManager,
pre_exec=setting('shell_command.lsf.pre_exec'),
post_exec=setting('shell_command.lsf.post_exec'))
class LSFRequestBuilder(object):
def __init__(self):
if self.pre_exec:
self.pre_exec_command = _localize_cmd(self.pre_exec)
else:
self.pre_exec_command = None
if self.post_exec:
self.post_exec_command = _localize_cmd(self.post_exec)
else:
self.post_exec_command = None
def construct_request(self, executor_data):
request = create_empty_request()
import logging
LOG = logging.getLogger(__name__)
@inject(broker=flow.interfaces.IBroker)
class ShellCommandServiceInterface(IShellCommand):
def submit(self, **kwargs):
message = ShellCommandSubmitMessage(**kwargs)
return self.broker.publish(self.exchange,
self.submit_routing_key, message)
@inject(exchange=setting('shell_command.fork.exchange'),
submit_routing_key=setting('shell_command.fork.submit_routing_key'))
class ForkShellCommandServiceInterface(ShellCommandServiceInterface):
pass
@inject(exchange=setting('shell_command.lsf.exchange'),
submit_routing_key=setting('shell_command.lsf.submit_routing_key'))
class LSFShellCommandServiceInterface(ShellCommandServiceInterface):
def submit(self, **kwargs):
# XXX Additional message validation:
# executor_data must contain response places, net key, etc.
# executor_data must not contain working_directory (easy mistake)
return ShellCommandServiceInterface.submit(self, **kwargs)
else:
self.flag = None
self.type = TYPE_MAP[type]
self.options_name = 'options%s' % option_set
def set_option(self, request, value):
cast_value = self.type(value)
setattr(request, self.name, cast_value)
if self.flag is not None:
options = getattr(request, self.options_name)
setattr(request, self.options_name, options | int(self.flag))
@injector.inject(
option_definitions=setting('shell_command.lsf.available_options'),
default_options=setting('shell_command.lsf.default_options'))
class LSFOptionManager(object):
def __init__(self):
self.available_options = factory.build_objects(
self.option_definitions, MODULE, 'LSFOption')
def set_default_options(self, request):
for option, value in self.default_options.iteritems():
self.available_options[option].set_option(request, value)
def set_options(self, request, executor_data):
self.set_default_options(request)
lsf_options = executor_data.get('lsf_options', {})
from flow.shell_command.messages import ShellCommandSubmitMessage
from flow.util import environment as env_util
from injector import inject
from twisted.internet import defer, reactor
from flow.shell_command.monitor import ExecutorMonitor
import logging
import os
import sys
import twisted.python.procutils
LOG = logging.getLogger(__name__)
@inject(default_environment=setting('shell_command.default_environment', {}),
mandatory_environment=setting(
'shell_command.mandatory_environment', {}))
class ShellCommandSubmitMessageHandler(Handler):
message_class = ShellCommandSubmitMessage
@property
def executable(self):
if is_executable(os.path.abspath(self.executable_name)):
return os.path.abspath(self.executable_name)
elif is_executable(os.path.join(os.path.dirname(sys.executable),
self.executable_name)):
return os.path.join(os.path.dirname(sys.executable),
self.executable_name)
else:
return twisted.python.procutils.which(self.executable_name)[0]
@inject(storage=WorkflowHistorianStorage)
class HistorianHandlerBase(Handler):
def _handle_message(self, message):
try:
self._handle_historian_message(message)
return defer.succeed(None)
except EXIT_ON:
LOG.exception("This historian cannot handle messages anymore, "
"because it lost access to Oracle... exiting.")
exit_process(exit_codes.EXECUTE_FAILURE)
@inject(queue_name=setting('workflow.historian.update_queue'))
class HistorianUpdateHandler(HistorianHandlerBase):
message_class = messages.UpdateMessage
def _handle_historian_message(self, message):
message_dict = self._get_message_dict(message)
LOG.debug("Updating [net_key='%s', operation_id='%s']: %r",
message.operation_data['net_key'],
message.operation_data['operation_id'], message_dict)
self.storage.update(message_dict)
def _get_message_dict(self, message):
message_dict = message.to_dict()
message_dict['status'] = Status(message_dict['status'])
from flow.configuration.settings.injector import setting
from flow.interfaces import IServiceLocator
from flow.shell_command.handler_base import ShellCommandSubmitMessageHandler
from injector import inject
from twisted.internet import defer
import os
import socket
@inject(queue_name=setting('shell_command.fork.queue'),
exchange=setting('shell_command.fork.exchange'),
response_routing_key=setting('shell_command.fork.response_routing_key'),
service_interfaces=IServiceLocator)
class ForkShellCommandMessageHandler(ShellCommandSubmitMessageHandler):
executable_name = 'flow-fork-shell-command-executor'
def on_job_id_success(self, job_id, callback_data=None,
job_id_handled=None):
deferreds = []
dispatch_data = {'job_id': job_id}
deferreds.append(self.send_message('msg: dispatch_success',
callback_data, token_data=dispatch_data))
execute_data = {'hostname': socket.gethostname()}
deferreds.append(self.send_message('msg: execute_begin',
callback_data, token_data=execute_data))
dlist = defer.gatherResults(deferreds, consumeErrors=True)
dlist.addCallbacks(job_id_handled.callback, job_id_handled.errback)
LOG = logging.getLogger(__name__)
def _build_command_cmdline(method, action_id, inputs_file, outputs_file):
return ["command", method, action_id, inputs_file.name, outputs_file.name]
def _build_event_cmdline(method, action_id, inputs_file, outputs_file):
return ["event", method, action_id, outputs_file.name]
CMDLINE_BUILDERS = {"command": _build_command_cmdline, "event": _build_event_cmdline}
@inject(perl_wrapper=setting("workflow.perl_wrapper"), storage=flow.interfaces.IStorage)
class WorkflowWrapperCommand(CommandBase):
injector_modules = [RedisConfiguration]
@staticmethod
def annotate_parser(parser):
parser.add_argument("--method", required=True, help="shortcut or execute")
parser.add_argument("--action-type", required=True, help="event or command")
parser.add_argument("--action-id", required=True, help="event_id or perl_class")
parser.add_argument("--net-key", required=True, help="used to look up inputs")
parser.add_argument("--operation-id", type=int, required=True, help="used to look up inputs")
parser.add_argument("--parallel-id", default="[]", help="used to look up inputs")
def _execute(self, parsed_arguments):
from injector import inject
from pika.adapters import twisted_connection
from twisted.internet import reactor, defer, protocol
from flow.exit_codes import (EXECUTE_SYSTEM_FAILURE,
EXECUTE_ERROR,
EXECUTE_SERVICE_UNAVAILABLE)
from flow.util.defer import add_callback_and_default_errback
import logging
import pika
LOG = logging.getLogger(__name__)
@inject(
hostname=setting('amqp.hostname'),
port=setting('amqp.port'),
virtual_host=setting('amqp.vhost'),
retry_delay=setting('amqp.retry_delay'),
connection_attempts=setting('amqp.connection_attempts'),
prefetch_count=setting('amqp.prefetch_count'),
heartbeat_interval=setting('amqp.heartbeat_interval'),
)
class ConnectionParams(object):
@property
def pika_params(self):
return pika.ConnectionParameters(host=self.hostname, port=self.port,
virtual_host=self.virtual_host,
heartbeat_interval=self.heartbeat_interval)
DISCONNECTED = 'disconnected'
"""
TABLES = namedtuple('Tables', ['historian', 'instance', 'execution'])
SEQUENCES = namedtuple('Sequences', ['instance', 'execution'])
STATEMENTS = namedtuple('Statements', STATEMENTS_DICT.keys())
def on_oracle_connect(connection, record):
cursor = connection.cursor()
cursor.execute("alter session set NLS_DATE_FORMAT = "
"'YYYY-MM-DD HH24:MI:SS'")
cursor.execute("alter session set NLS_TIMESTAMP_FORMAT = "
"'YYYY-MM-DD HH24:MI:SSXFF'")
cursor.close()
@inject(connection_string=setting('workflow.historian.connection_string'),
owner=setting('workflow.historian.owner'))
class WorkflowHistorianStorage(object):
def __init__(self):
self.statements = STATEMENTS(**{k:v % self.owner
for k, v in STATEMENTS_DICT.items()})
self.tables = TABLES(historian='%s.workflow_historian' % self.owner,
instance='%s.workflow_instance' % self.owner,
execution='%s.workflow_instance_execution' % self.owner)
self.sequences = SEQUENCES(
instance='%s.workflow_instance_seq' % self.owner,
execution='%s.workflow_execution_seq' % self.owner)
self.engine = create_engine(self.connection_string,
case_sensitive=False, poolclass=StaticPool)
from flow.configuration.settings.injector import setting
from flow.orchestrator.messages import CreateTokenMessage, NotifyPlaceMessage
from flow.orchestrator.messages import NotifyTransitionMessage
from injector import inject
import flow.interfaces
import logging
LOG = logging.getLogger(__name__)
@inject(broker=flow.interfaces.IBroker,
create_token_exchange=
setting('orchestrator.create_token_exchange'),
create_token_routing_key=
setting('orchestrator.create_token_routing_key'),
notify_place_exchange=
setting('orchestrator.notify_place_exchange'),
notify_place_routing_key=
setting('orchestrator.notify_place_routing_key'),
notify_transition_exchange=
setting('orchestrator.notify_transition_exchange'),
notify_transition_routing_key=
setting('orchestrator.notify_transition_routing_key'))
class OrchestratorServiceInterface(flow.interfaces.IOrchestrator):
def create_token(self, net_key, place_idx,
color, color_group_idx, data=None):
message = CreateTokenMessage(net_key=net_key, place_idx=place_idx,
color=color, color_group_idx=color_group_idx, data=data)
return self.broker.publish(self.create_token_exchange,
from collections import deque, defaultdict
from flow.configuration.settings.injector import setting
from injector import inject
from twisted.internet import defer, reactor
import flow.interfaces
import logging
LOG = logging.getLogger(__name__)
@inject(bindings=setting('bindings'))
class LocalBroker(flow.interfaces.IBroker):
def __init__(self):
self.bindings = _transform_bindings(self.bindings)
self.queue = deque()
self.handlers = {}
def publish(self, exchange_name, routing_key, message):
encoded_message = message.encode()
return self.raw_publish(exchange_name, routing_key, encoded_message)
def register_handler(self, handler):
LOG.debug('Registering handler on %s', handler.queue_name)
self.handlers[handler.queue_name] = handler
def raw_publish(self, exchange, routing_key, encoded_message):
LOG.debug('Putting message for exchange (%s), '
'routing_key (%s) in queue: %s',
exchange, routing_key, encoded_message)
from flow.configuration.settings.injector import setting
from flow.shell_command import factory
from flow.shell_command import resource_types
from flow.shell_command.lsf import resource
import injector
@injector.inject(
resource_type_definitions=setting('shell_command.resource_types'),
resource_definitions=setting('shell_command.lsf.supported_resources'))
class LSFResourceManager(object):
def __init__(self):
self.resource_types = resource_types.make_resource_types(
self.resource_type_definitions)
self.available_resources = {}
self.available_resources['limit'] = factory.build_objects(
self.resource_definitions.get('limit', {}), resource)
self.available_resources['request'] = factory.build_objects(
self.resource_definitions.get('request', {}), resource)
self.available_resources['reserve'] = factory.build_objects(
self.resource_definitions.get('reserve', {}), resource)
def set_resources(self, request, executor_data):
resources = resource_types.make_all_resource_objects(
executor_data.get('resources', {}), self.resource_types)
resource.set_all_resources(request, resources, self.available_resources)
from flow.configuration.settings.injector import setting
from flow.handler import Handler
from flow.orchestrator.messages import CreateTokenMessage, NotifyPlaceMessage
from flow.orchestrator.messages import NotifyTransitionMessage
from flow.redisom import get_object
from injector import inject
import flow.interfaces
import logging
LOG = logging.getLogger(__name__)
@inject(redis=flow.interfaces.IStorage,
service_interfaces=flow.interfaces.IServiceLocator,
queue_name=setting('orchestrator.create_token_queue'))
class PetriCreateTokenHandler(Handler):
message_class = CreateTokenMessage
def _handle_message(self, message):
net = get_object(self.redis, message.net_key)
create_token_kwargs = getattr(message, 'create_token_kwargs', {})
return net.create_put_notify(message.place_idx,
self.service_interfaces,
color=message.color,
color_group_idx=message.color_group_idx,
data=getattr(message, 'data', {}))
from flow.configuration.settings.injector import setting
import injector
import json
import re
import requests
@injector.inject(bindings=setting('bindings'),
hostname=setting('amqp.hostname'),
port=setting('amqp.api_port'), virtual_host=setting('amqp.vhost'))
class RabbitMQAPI(object):
@property
def auth(self):
return ('guest', 'guest')
@property
def parameters(self):
return {
'hostname': self.hostname,
'port': self.port,
'virtual_host': self.virtual_host,
}
def request_string(self, api_substring):
template = 'http://%(hostname)s:%(port)s/api/' + api_substring
return template % self.parameters
def vhost_status(self):
response = requests.get(self.request_string('vhosts/%(virtual_host)s'),
