def load_config(self, path, is_reload=False):
if self.config_class() is None:
return None
config = self.config_class()() # Create a empty config
if path is not None:
try:
with open(self.config_path, "r") as f:
try:
config = self.config_class()(yaml.load(f), strict=False)
except ModelConversionError as e:
self.logger.error("Could not create config for worker "+self.framework_name + " "+json.dumps(e.message))
return config
try:
config.validate()
except ModelValidationError as e:
config = self.config_class()() # Reset back to empty config
self.logger.error("Could not validate config for worker "+self.framework_name + " "+json.dumps(e.message))
except IOError as e:
self.logger.error("Could not load config for worker "+self.framework_name+". "+e.message)
else:
self.logger.warning("Config Path is not defined for worker "+self.framework_name)
self.config = config
if is_reload:
publisher = RoutingPublisher(self.rabbitmq, "worker", "reload")
publisher.publish({'target': self.worker_name(), 'framework': self.framework_name, 'tags': self.get_tags()})
publisher.close()
class RPCPublisher(object):
def __init__(self, rabbitmq, exchange_name, routingkey):
self.logger = logging.getLogger("hq.warehouse.rabbitmq.rpc.publisher")
self.publisher = RoutingPublisher(rabbitmq, exchange_name, routingkey)
global rpc_subscriber
if rpc_subscriber is None:
rpc_subscriber = RPCSubscriber(rabbitmq)
rpc_subscriber.start()
def publish(self, dict):
global rpc_subscriber
while rpc_subscriber.consumer_tag is None:
continue
props = BasicProperties()
props.reply_to = rpc_subscriber.queue_name
props.correlation_id = str(uuid.uuid4())
try:
rpc_subscriber.rpc_data[props.correlation_id] = Event()
self.publisher.publish(dict, props)
except:
self.logger.exception("Error publishing rpc message")
del rpc_subscriber.rpc_data[props.correlation_id]
props.correlation_id = None
self.publisher.close()
return props.correlation_id
def get_data(self, correlation_id, wait=5, force=False):
global rpc_subscriber
if correlation_id in rpc_subscriber.rpc_data:
if isinstance(rpc_subscriber.rpc_data[correlation_id], _Event):
if force: # Already waited and data did not show up
del rpc_subscriber.rpc_data[correlation_id]
return None
try:
rpc_subscriber.rpc_data[correlation_id].wait(wait)
except KeyboardInterrupt:
return None
return self.get_data(correlation_id, force=True)
else:
data = rpc_subscriber.rpc_data[correlation_id]
del rpc_subscriber.rpc_data[correlation_id]
return data
return None
def __init__(self, rabbitmq, exchange_name, routingkey):
self.logger = logging.getLogger("hq.warehouse.rabbitmq.rpc.publisher")
self.publisher = RoutingPublisher(rabbitmq, exchange_name, routingkey)
global rpc_subscriber
if rpc_subscriber is None:
rpc_subscriber = RPCSubscriber(rabbitmq)
rpc_subscriber.start()
