def _launchedApp(self, kwargs):
web_interface_id = kwargs['web_interface_id']
try:
vm_info = VMInfo()
vm_info.save()
app = Application.objects.get(app_id=web_interface_id)
app.interface_bits = AppState.bits['STARTED']
app.vm_info = vm_info
app.save()
if app.const_output_update:
producerAppManager = messaging.MessageProducer(
messaging.buildRabbitMQConnectionLink(),
settings.APPLICATION_MANAGER_QUEUE,
)
message = messaging.createMessage(
'const_output_update',
return_queue=settings.WEB_INTERFACE_QUEUE,
app_manager_id=app.app_manager_id,
update_output=app.const_output_update,
)
producerAppManager.publish(message)
except ObjectDoesNotExist:
pass
def post(self, request, *args, **kwargs):
if request.is_ajax:
if 'params' in request.POST:
ajax = AJAXRequests(json.loads(request.POST['params']))
if ajax.isPlay() and ajax.isTargetInput():
try:
input_element = InputElement.objects.get(input_id=ajax.id)
if input_element.interface_bits == InputState.bits['NEW']:
app_group = ApplicationGroup.create()
app_group.save()
input_element.interface_bits = InputState.bits['PREPROCESSING']
input_element.save()
message = messaging.createMessage(
'preprocess',
return_queue=settings.WEB_INTERFACE_QUEUE,
input_id=input_element.input_id,
input_file=input_element.input_file.name,
group_id=app_group.group_id,
input_param='old',
)
producerPreprocessor.publish(message)
except ObjectDoesNotExist:
pass
elif ajax.isTrash() and ajax.isTargetInput():
try:
input_element = InputElement.objects.get(input_id=ajax.id)
if input_element.interface_bits == InputState.bits['NEW']:
input_element.delete()
except ObjectDoesNotExist:
pass
return redirect(reverse('input_upload'))
def _trashApp(self, app):
app_group = app.app_group
if app_group.interface_bits != GroupState.bits['TRANSITIONING']:
app_group.prev_interface_bits = app_group.interface_bits
app_group.interface_bits = GroupState.bits['TRANSITIONING']
app_group.transitioning_apps_counter += 1
app_group.save()
message = messaging.createMessage(
'delete_app',
return_queue=settings.WEB_INTERFACE_QUEUE,
app_manager_id=app.app_manager_id,
web_interface_id=app.app_id,
)
#producerAppManager.publish(message)
app.interface_bits = AppState.bits['TRANSITIONING']
app.save()
def _createdApp(self,kwargs):
web_interface_id = kwargs['web_interface_id']
app_manager_id = kwargs['app_manager_id']
try:
app = Application.objects.get(app_id=web_interface_id)
app.app_manager_id = app_manager_id
app.save()
producerAppManager = messaging.MessageProducer(
messaging.buildRabbitMQConnectionLink(),
settings.APPLICATION_MANAGER_QUEUE,
)
message = messaging.createMessage(
'launch_app',
return_queue=settings.WEB_INTERFACE_QUEUE,
app_manager_id=app.app_manager_id,
)
producerAppManager.publish(message)
app.save()
except ObjectDoesNotExist:
pass
def _preprocess(self, return_params, kwargs):
prefix = 's3://'
aws_storage_bucket_name = auxiliary.read_option(self.config, 'aws', 'aws_storage_bucket_name')
aws_storage_head = os.path.join(prefix, aws_storage_bucket_name)
s3_app_input = auxiliary.read_option(self.config, 'preprocessor', 's3_app_input')
s3_app_output = auxiliary.read_option(self.config, 'preprocessor', 's3_app_output')
s3_app_scripts = auxiliary.read_option(self.config, 'preprocessor', 's3_app_scripts')
s3cfg_path = os.path.join(self.CLAUDE_ROOT, auxiliary.read_option(self.config, 'preprocessor', 's3cfg'))
group_id = kwargs['group_id']
input_param = kwargs['input_param']
producer = messaging.MessageProducer(
messaging.buildRabbitMQConnectionLink(address=return_params['return_address']),
return_params['return_queue'],
)
scripts_dir = scripts.__path__[0]
with auxiliary.TemporaryDirectory() as tmp_dir:
input_s3_path_full = os.path.join(aws_storage_head, kwargs['input_file'])
input_arch_name = os.path.basename(input_s3_path_full)
input_arch_path = os.path.join(tmp_dir, input_arch_name)
s3tools.s3cmdGET(s3cfg_path, input_s3_path_full, input_arch_path)
ready_archs = self._handle_local_arch(input_arch_path)
for arch in ready_archs:
app_type = AppParamsProvider(arch['app_type'], input_param)
script_name = app_type.getAppScriptName()
script_path = os.path.join(scripts_dir, script_name)
script_s3_path_full = os.path.join(*[aws_storage_head, s3_app_scripts, script_name])
s3tools.s3cmdPUT(s3cfg_path, script_path, script_s3_path_full)
app_input_path = arch['arch_path']
app_input_name = os.path.basename(app_input_path)
app_input_s3_path = os.path.join(s3_app_input, app_input_name)
app_input_s3_path_full = os.path.join(aws_storage_head, app_input_s3_path)
s3tools.s3cmdPUT(s3cfg_path, app_input_path, app_input_s3_path_full)
app_output_s3_path = os.path.join(s3_app_output, 'output_' + app_input_name)
app_output_s3_path_full = os.path.join(aws_storage_head, app_output_s3_path)
app_params = {
'app_input_file' : app_input_s3_path_full,
'app_output_file' : app_output_s3_path_full,
'app_script' : script_s3_path_full,
}
app_params.update(app_type.getAppSpecificParams())
message = messaging.createMessage(
'preprocessed',
group_id=group_id,
app_type=arch['app_type'],
app_name=arch['app_name'],
app_input_file=app_input_s3_path,
app_output_file=app_output_s3_path,
app_params=app_params,
)
producer.publish(message)
message = messaging.createMessage(
'preprocessing_completed',
input_id=kwargs['input_id'],
group_id=group_id,
)
producer.publish(message)
def post(self, request, *args, **kwargs):
if request.is_ajax:
if 'params' in request.POST:
ajax = AJAXRequests(json.loads(request.POST['params']))
if ajax.isTrash() and ajax.isTargetApp():
try:
app = Application.objects.get(app_id=ajax.id)
self._trashApp(app)
except ObjectDoesNotExist:
pass
elif ajax.isTrash() and ajax.isTargetGroup():
try:
app_group = ApplicationGroup.objects.get(group_id=ajax.id)
apps_list = Application.objects.filter(app_group=app_group)
for app in apps_list:
self._trashApp(app)
except ObjectDoesNotExist:
pass
elif ajax.isPlay() and ajax.isTargetApp():
try:
app = Application.objects.get(app_id=ajax.id)
self._playApp(app)
except ObjectDoesNotExist:
pass
elif ajax.isPlay() and ajax.isTargetGroup():
try:
app_group = ApplicationGroup.objects.get(group_id=ajax.id)
apps_list = Application.objects.filter(app_group=app_group)
for app in apps_list:
self._playApp(app)
app_group.interface_bits = GroupState.bits['STARTED']
app_group.save()
except ObjectDoesNotExist:
pass
elif ajax.isInfo() and ajax.isTargetApp():
return redirect(reverse('view_application', kwargs={'app_id': ajax.id}))
elif ajax.isRequest() and ajax.isTargetGroup():
try:
app_group = ApplicationGroup.objects.get(group_id=ajax.id)
try:
default_storage.delete(app_group.output.name)
app_group.output_file_ready = False
app_group.interface_bits = GroupState.bits['FINISHED']
app_group.save()
except S3ResponseError:
pass
except:
pass
apps_list = Application.objects.filter(app_group=app_group)
outputs_list = []
apps_ids_list = []
for app in apps_list:
if app.app_output_file_ready:
app.interface_bits = AppState.bits['TRANSITIONING']
app.save()
apps_ids_list.append(app.app_id)
outputs_list.append(app.app_output_file.name)
message = messaging.createMessage(
'postprocess',
return_queue=settings.WEB_INTERFACE_QUEUE,
group_id=app_group.group_id,
outputs_list=outputs_list,
apps_ids_list=apps_ids_list,
)
producerPostprocessor.publish(message)
app_group.interface_bits = GroupState.bits['TRANSITIONING']
app_group.save()
except ObjectDoesNotExist:
pass
elif ajax.isDownload() and ajax.isTargetApp():
try:
app = Application.objects.get(app_id=ajax.id)
fileopen = default_storage.open(app.app_output_file.name, 'r')
fileopen.close()
url = s3tools.botoGetURL(settings.CONFIG, app.app_output_file.name)
# a temporary hack
url_split = url.split('/', 3)
n_url = settings.OUTSIDE_ADDRESS + url_split[3]
return HttpResponse(json.dumps(n_url))
except ObjectDoesNotExist:
pass
except IOError:
app.app_output_file_ready = False
app.interface_bits = AppState.bits['NEW']
app.save()
except S3ResponseError:
app.app_output_file_ready = False
app.interface_bits = AppState.bits['NEW']
app.save()
elif ajax.isDownload() and ajax.isTargetGroup():
try:
app_group = ApplicationGroup.objects.get(group_id=ajax.id)
fileopen = default_storage.open(app_group.output.name, 'r')
fileopen.close()
url = s3tools.botoGetURL(settings.CONFIG, app_group.output.name)
# a temporary hack
url_split = url.split('/', 3)
n_url = settings.OUTSIDE_ADDRESS + url_split[3]
return HttpResponse(json.dumps(n_url))
except ObjectDoesNotExist:
pass
except IOError:
app_group.output_file_ready = False
app_group.interface_bits = GroupState.bits['FINISHED']
app_group.save()
except S3ResponseError:
app_group.output_file_ready = False
app_group.interface_bits = GroupState.bits['FINISHED']
app_group.save()
elif ajax.isTrashOutput() and ajax.isTargetGroup():
try:
app_group = ApplicationGroup.objects.get(group_id=ajax.id)
default_storage.delete(app_group.output.name)
app_group.output_file_ready = False
app_group.interface_bits = GroupState.bits['FINISHED']
app_group.save()
except ObjectDoesNotExist:
pass
except S3ResponseError:
pass
return redirect(reverse('application_list'))