def _get_url(cls, path):
FILE_SERVER_TYPE = get_setting('FILE_SERVER_TYPE')
if FILE_SERVER_TYPE == 'LOCAL':
return 'file://' + path
elif FILE_SERVER_TYPE == 'GOOGLE_CLOUD':
if get_setting('BUCKET_ID').strip() == '':
raise Exception('Bucket ID is not set.')
return 'gs://' + get_setting('BUCKET_ID') + path
else:
raise Exception(
'Couldn\'t recognize value for setting FILE_SERVER_TYPE="%s"'\
% FILE_SERVER_TYPE)
def run(cls, task_run):
from analysis.models.task_runs import TaskRunAttempt
task_run_attempt = TaskRunAttempt.create_from_task_run(task_run)
cmd = [
sys.executable, TASK_RUNNER_EXECUTABLE, '--run_attempt_id',
task_run_attempt.id.hex, '--master_url',
get_setting('MASTER_URL_FOR_WORKER')
]
logger.debug(cmd)
proc = subprocess.Popen(cmd, stderr=subprocess.STDOUT)
def _get_path_for_import(cls, file_data_object):
if get_setting('KEEP_DUPLICATE_FILES') and get_setting('FORCE_RERUN'):
# If both are True, we can organize the directory structure in
# a human browsable way
return os.path.join(
'/', get_setting('FILE_ROOT'),
cls._get_browsable_path(file_data_object),
"%s-%s-%s" % (timezone.now().strftime('%Y%m%d%H%M%S'),
file_data_object.id.hex,
file_data_object.file_content.filename))
elif get_setting('KEEP_DUPLICATE_FILES'):
# Use a flat directory structure but give files with
# identical content distinctive names
return os.path.join(
'/', get_setting('FILE_ROOT'),
'%s-%s-%s' % (timezone.now().strftime('%Y%m%d%H%M%S'),
file_data_object.id.hex,
file_data_object.file_content.filename))
else:
# Use a flat directory structure and use file names that
# reflect content
return os.path.join(
'/',
get_setting('FILE_ROOT'),
'%s-%s' % (
file_data_object.file_content.unnamed_file_content\
.hash_function,
file_data_object.file_content.unnamed_file_content\
.hash_value
)
)
def add_file_location(self):
# A FileLocation should be generated once file_content is set
if self.file_content and not self.file_location:
# If a file with identical content has already been uploaded,
# re-use it if permitted by settings.
if not get_setting('KEEP_DUPLICATE_FILES'):
file_location = self.file_content.get_valid_location()
if file_location is not None:
self.file_location = file_location
self.save()
return
# No existing file to use. Create a new location for upload.
self.file_location \
= FileLocation.create_location_for_import(self)
self.save()
def worker_settings(request, id):
try:
WORKING_DIR = models.TaskRunAttempt.get_working_dir(id)
LOG_DIR = models.TaskRunAttempt.get_log_dir(id)
return JsonResponse({
'LOG_LEVEL':
get_setting('LOG_LEVEL'),
'WORKING_DIR':
WORKING_DIR,
'WORKER_LOG_FILE':
os.path.join(LOG_DIR, 'worker.log'),
'STDOUT_LOG_FILE':
os.path.join(LOG_DIR, 'stdout.log'),
'STDERR_LOG_FILE':
os.path.join(LOG_DIR, 'stderr.log'),
})
except Exception as e:
return JsonResponse({"message": e.message}, status=500)
def get_task_manager(cls):
logger.debug('Getting task manager of type "%s"' %
get_setting('WORKER_TYPE'))
if get_setting('WORKER_TYPE') == cls.LOCAL:
return LocalTaskManager()
elif get_setting('WORKER_TYPE') == cls.CLUSTER:
return ClusterTaskManager()
elif get_setting('WORKER_TYPE') == cls.GOOGLE_CLOUD:
return CloudTaskManager()
elif get_setting('WORKER_TYPE') == cls.MOCK:
return MockTaskManager()
else:
raise Exception('Invalid selection WORKER_TYPE=%s' %
get_setting('WORKER_TYPE'))
def get_log_dir(cls, task_run_attempt_id):
return os.path.join(get_setting('FILE_ROOT_FOR_WORKER'),
'runtime_volumes', task_run_attempt_id, 'logs')
def filehandler_settings(request):
return JsonResponse({
'HASH_FUNCTION': get_setting('HASH_FUNCTION'),
'PROJECT_ID': get_setting('PROJECT_ID'),
})
import logging
import os
import requests
import subprocess
import sys
from analysis import get_setting
logger = logging.getLogger('LoomDaemon')
TASK_RUNNER_EXECUTABLE = os.path.abspath(
os.path.join(
get_setting('BASE_DIR'),
'../../worker/task_runner.py',
))
class LocalTaskManager:
@classmethod
def run(cls, task_run):
from analysis.models.task_runs import TaskRunAttempt
task_run_attempt = TaskRunAttempt.create_from_task_run(task_run)
cmd = [
sys.executable, TASK_RUNNER_EXECUTABLE, '--run_attempt_id',
task_run_attempt.id.hex, '--master_url',
get_setting('MASTER_URL_FOR_WORKER')
]
logger.debug(cmd)
proc = subprocess.Popen(cmd, stderr=subprocess.STDOUT)