def run(self):
consumer = messaging.MessageConsumer(
messaging.buildRabbitMQConnectionLink(),
self.queue,
self.callback,
)
consumer.constantConsuming()
def _launchedApp(self, kwargs):
web_interface_id = kwargs['web_interface_id']
try:
vm_info = VMInfo()
vm_info.save()
app = Application.objects.get(app_id=web_interface_id)
app.interface_bits = AppState.bits['STARTED']
app.vm_info = vm_info
app.save()
if app.const_output_update:
producerAppManager = messaging.MessageProducer(
messaging.buildRabbitMQConnectionLink(),
settings.APPLICATION_MANAGER_QUEUE,
)
message = messaging.createMessage(
'const_output_update',
return_queue=settings.WEB_INTERFACE_QUEUE,
app_manager_id=app.app_manager_id,
update_output=app.const_output_update,
)
producerAppManager.publish(message)
except ObjectDoesNotExist:
pass
def _createdApp(self,kwargs):
web_interface_id = kwargs['web_interface_id']
app_manager_id = kwargs['app_manager_id']
try:
app = Application.objects.get(app_id=web_interface_id)
app.app_manager_id = app_manager_id
app.save()
producerAppManager = messaging.MessageProducer(
messaging.buildRabbitMQConnectionLink(),
settings.APPLICATION_MANAGER_QUEUE,
)
message = messaging.createMessage(
'launch_app',
return_queue=settings.WEB_INTERFACE_QUEUE,
app_manager_id=app.app_manager_id,
)
producerAppManager.publish(message)
app.save()
except ObjectDoesNotExist:
pass
def _preprocess(self, return_params, kwargs):
prefix = 's3://'
aws_storage_bucket_name = auxiliary.read_option(self.config, 'aws', 'aws_storage_bucket_name')
aws_storage_head = os.path.join(prefix, aws_storage_bucket_name)
s3_app_input = auxiliary.read_option(self.config, 'preprocessor', 's3_app_input')
s3_app_output = auxiliary.read_option(self.config, 'preprocessor', 's3_app_output')
s3_app_scripts = auxiliary.read_option(self.config, 'preprocessor', 's3_app_scripts')
s3cfg_path = os.path.join(self.CLAUDE_ROOT, auxiliary.read_option(self.config, 'preprocessor', 's3cfg'))
group_id = kwargs['group_id']
input_param = kwargs['input_param']
producer = messaging.MessageProducer(
messaging.buildRabbitMQConnectionLink(address=return_params['return_address']),
return_params['return_queue'],
)
scripts_dir = scripts.__path__[0]
with auxiliary.TemporaryDirectory() as tmp_dir:
input_s3_path_full = os.path.join(aws_storage_head, kwargs['input_file'])
input_arch_name = os.path.basename(input_s3_path_full)
input_arch_path = os.path.join(tmp_dir, input_arch_name)
s3tools.s3cmdGET(s3cfg_path, input_s3_path_full, input_arch_path)
ready_archs = self._handle_local_arch(input_arch_path)
for arch in ready_archs:
app_type = AppParamsProvider(arch['app_type'], input_param)
script_name = app_type.getAppScriptName()
script_path = os.path.join(scripts_dir, script_name)
script_s3_path_full = os.path.join(*[aws_storage_head, s3_app_scripts, script_name])
s3tools.s3cmdPUT(s3cfg_path, script_path, script_s3_path_full)
app_input_path = arch['arch_path']
app_input_name = os.path.basename(app_input_path)
app_input_s3_path = os.path.join(s3_app_input, app_input_name)
app_input_s3_path_full = os.path.join(aws_storage_head, app_input_s3_path)
s3tools.s3cmdPUT(s3cfg_path, app_input_path, app_input_s3_path_full)
app_output_s3_path = os.path.join(s3_app_output, 'output_' + app_input_name)
app_output_s3_path_full = os.path.join(aws_storage_head, app_output_s3_path)
app_params = {
'app_input_file' : app_input_s3_path_full,
'app_output_file' : app_output_s3_path_full,
'app_script' : script_s3_path_full,
}
app_params.update(app_type.getAppSpecificParams())
message = messaging.createMessage(
'preprocessed',
group_id=group_id,
app_type=arch['app_type'],
app_name=arch['app_name'],
app_input_file=app_input_s3_path,
app_output_file=app_output_s3_path,
app_params=app_params,
)
producer.publish(message)
message = messaging.createMessage(
'preprocessing_completed',
input_id=kwargs['input_id'],
group_id=group_id,
)
producer.publish(message)
from .forms import InputUploadForm
from claudei.states import InputState, GroupState, AppState
from claudei.AJAXRequests import AJAXRequests
import os
import logging
import json
import tools.messaging as messaging
import tools.auxiliary as auxiliary
import tools.s3tools as s3tools
from manager.tools.messaging import Producer, form_message, claude_msgs_pb2
producerPreprocessor = messaging.MessageProducer(
messaging.buildRabbitMQConnectionLink(),
settings.PREPROCESSOR_QUEUE,
)
producerPostprocessor = messaging.MessageProducer(
messaging.buildRabbitMQConnectionLink(),
settings.POSTPROCESSOR_QUEUE,
)
producerAppExecutor = messaging.MessageProducer(
messaging.buildRabbitMQConnectionLink(),
settings.APP_EXECUTOR_QUEUE,
)
'''producerResourceManager = messaging.MessageProducer(
messaging.buildRabbitMQConnectionLink(),
settings.RESOURCE_MANAGER_QUEUE,
)'''
