def run(self): consumer = messaging.MessageConsumer( messaging.buildRabbitMQConnectionLink(), self.queue, self.callback, ) consumer.constantConsuming()
def _launchedApp(self, kwargs): web_interface_id = kwargs['web_interface_id'] try: vm_info = VMInfo() vm_info.save() app = Application.objects.get(app_id=web_interface_id) app.interface_bits = AppState.bits['STARTED'] app.vm_info = vm_info app.save() if app.const_output_update: producerAppManager = messaging.MessageProducer( messaging.buildRabbitMQConnectionLink(), settings.APPLICATION_MANAGER_QUEUE, ) message = messaging.createMessage( 'const_output_update', return_queue=settings.WEB_INTERFACE_QUEUE, app_manager_id=app.app_manager_id, update_output=app.const_output_update, ) producerAppManager.publish(message) except ObjectDoesNotExist: pass
def _createdApp(self,kwargs): web_interface_id = kwargs['web_interface_id'] app_manager_id = kwargs['app_manager_id'] try: app = Application.objects.get(app_id=web_interface_id) app.app_manager_id = app_manager_id app.save() producerAppManager = messaging.MessageProducer( messaging.buildRabbitMQConnectionLink(), settings.APPLICATION_MANAGER_QUEUE, ) message = messaging.createMessage( 'launch_app', return_queue=settings.WEB_INTERFACE_QUEUE, app_manager_id=app.app_manager_id, ) producerAppManager.publish(message) app.save() except ObjectDoesNotExist: pass
def _preprocess(self, return_params, kwargs): prefix = 's3://' aws_storage_bucket_name = auxiliary.read_option(self.config, 'aws', 'aws_storage_bucket_name') aws_storage_head = os.path.join(prefix, aws_storage_bucket_name) s3_app_input = auxiliary.read_option(self.config, 'preprocessor', 's3_app_input') s3_app_output = auxiliary.read_option(self.config, 'preprocessor', 's3_app_output') s3_app_scripts = auxiliary.read_option(self.config, 'preprocessor', 's3_app_scripts') s3cfg_path = os.path.join(self.CLAUDE_ROOT, auxiliary.read_option(self.config, 'preprocessor', 's3cfg')) group_id = kwargs['group_id'] input_param = kwargs['input_param'] producer = messaging.MessageProducer( messaging.buildRabbitMQConnectionLink(address=return_params['return_address']), return_params['return_queue'], ) scripts_dir = scripts.__path__[0] with auxiliary.TemporaryDirectory() as tmp_dir: input_s3_path_full = os.path.join(aws_storage_head, kwargs['input_file']) input_arch_name = os.path.basename(input_s3_path_full) input_arch_path = os.path.join(tmp_dir, input_arch_name) s3tools.s3cmdGET(s3cfg_path, input_s3_path_full, input_arch_path) ready_archs = self._handle_local_arch(input_arch_path) for arch in ready_archs: app_type = AppParamsProvider(arch['app_type'], input_param) script_name = app_type.getAppScriptName() script_path = os.path.join(scripts_dir, script_name) script_s3_path_full = os.path.join(*[aws_storage_head, s3_app_scripts, script_name]) s3tools.s3cmdPUT(s3cfg_path, script_path, script_s3_path_full) app_input_path = arch['arch_path'] app_input_name = os.path.basename(app_input_path) app_input_s3_path = os.path.join(s3_app_input, app_input_name) app_input_s3_path_full = os.path.join(aws_storage_head, app_input_s3_path) s3tools.s3cmdPUT(s3cfg_path, app_input_path, app_input_s3_path_full) app_output_s3_path = os.path.join(s3_app_output, 'output_' + app_input_name) app_output_s3_path_full = os.path.join(aws_storage_head, app_output_s3_path) app_params = { 'app_input_file' : app_input_s3_path_full, 'app_output_file' : app_output_s3_path_full, 'app_script' : script_s3_path_full, } app_params.update(app_type.getAppSpecificParams()) message = messaging.createMessage( 'preprocessed', group_id=group_id, app_type=arch['app_type'], app_name=arch['app_name'], app_input_file=app_input_s3_path, app_output_file=app_output_s3_path, app_params=app_params, ) producer.publish(message) message = messaging.createMessage( 'preprocessing_completed', input_id=kwargs['input_id'], group_id=group_id, ) producer.publish(message)
from .forms import InputUploadForm from claudei.states import InputState, GroupState, AppState from claudei.AJAXRequests import AJAXRequests import os import logging import json import tools.messaging as messaging import tools.auxiliary as auxiliary import tools.s3tools as s3tools from manager.tools.messaging import Producer, form_message, claude_msgs_pb2 producerPreprocessor = messaging.MessageProducer( messaging.buildRabbitMQConnectionLink(), settings.PREPROCESSOR_QUEUE, ) producerPostprocessor = messaging.MessageProducer( messaging.buildRabbitMQConnectionLink(), settings.POSTPROCESSOR_QUEUE, ) producerAppExecutor = messaging.MessageProducer( messaging.buildRabbitMQConnectionLink(), settings.APP_EXECUTOR_QUEUE, ) '''producerResourceManager = messaging.MessageProducer( messaging.buildRabbitMQConnectionLink(), settings.RESOURCE_MANAGER_QUEUE, )'''