def _read_bool_from_config(key, default):
"""
Helper to read Docker specific bool config values from the worker config files.
"""
if config.has_option('docker', key):
return config.getboolean('docker', key)
else:
return default
def _read_bool_from_config(key, default):
"""
Helper to read Docker specific bool config values from the worker config files.
"""
if config.has_option('docker', key):
return config.getboolean('docker', key)
else:
return default
def _get_cache_settings(spec):
if not spec.get("use_cache", True):
return None
if not config.getboolean("girder_io", "diskcache_enabled"):
return None
return dict(
directory=config.get("girder_io", "diskcache_directory"),
eviction_policy=config.get("girder_io", "diskcache_eviction_policy"),
size_limit=config.getint("girder_io", "diskcache_size_limit"),
cull_limit=config.getint("girder_io", "diskcache_cull_limit"),
large_value_threshold=config.getint("girder_io", "diskcache_large_value_threshold"),
)
def _get_cache_settings(spec):
if not spec.get('use_cache', True):
return None
if not config.getboolean('girder_io', 'diskcache_enabled'):
return None
return dict(
directory=config.get('girder_io', 'diskcache_directory'),
eviction_policy=config.get('girder_io', 'diskcache_eviction_policy'),
size_limit=config.getint('girder_io', 'diskcache_size_limit'),
cull_limit=config.getint('girder_io', 'diskcache_cull_limit'),
large_value_threshold=config.getint(
'girder_io', 'diskcache_large_value_threshold'),
)
def _get_cache_settings(spec):
if not spec.get('use_cache', True):
return None
if not config.getboolean('girder_io', 'diskcache_enabled'):
return None
return dict(
directory=config.get('girder_io', 'diskcache_directory'),
eviction_policy=config.get('girder_io', 'diskcache_eviction_policy'),
size_limit=config.getint('girder_io', 'diskcache_size_limit'),
cull_limit=config.getint('girder_io', 'diskcache_cull_limit'),
large_value_threshold=config.getint(
'girder_io', 'diskcache_large_value_threshold'),
)
def fetch_handler(spec, **kwargs):
resource_type = spec.get('resource_type', 'file').lower()
task_input = kwargs.get('task_input', {})
target = task_input.get('target', 'filepath')
fetch_parent = spec.get('fetch_parent', False)
direct_path = spec.get('direct_path')
if 'id' not in spec:
raise Exception('Must pass a resource ID for girder inputs.')
if 'name' not in spec:
raise Exception('Must pass a name for girder inputs.')
client = _init_client(spec)
filename = client.transformFilename(spec['name'])
dest = os.path.join(kwargs['_tempdir'], filename)
if resource_type == 'folder':
client.downloadFolderRecursive(spec['id'], dest)
elif resource_type == 'item':
client.downloadItem(spec['id'], kwargs['_tempdir'], filename)
elif resource_type == 'file':
if fetch_parent:
# If we fetch the parent, we can't use direct paths as the
# task may needs all of the siblings next to each other
dest = _fetch_parent_item(spec['id'], client, kwargs['_tempdir'])
elif (direct_path and config.getboolean('girder_io', 'allow_direct_path') and
os.path.isfile(direct_path)):
# If the specification includes a direct path AND it is allowed by
# the worker configuration AND it is a reachable file, use it.
dest = direct_path
else:
client.downloadFile(spec['id'], dest)
else:
raise Exception('Invalid resource type: ' + resource_type)
if target == 'filepath':
return dest
elif target == 'memory':
with open(dest, 'rb') as fd:
return fd.read()
else:
raise Exception('Invalid Girder push target: ' + target)
@task_revoked.connect
def gw_task_revoked(sender=None, request=None, **rest):
try:
sender.job_manager = _job_manager(headers=request.message.headers,
kwargs=request.kwargsrepr)
_update_status(sender, JobStatus.CANCELED)
except AttributeError:
pass
except JobSpecNotFound:
logger.warn(
'No jobInfoSpec. Unable to move \'%s\' into CANCELED state.')
register('girder_io', jsonpickle.encode, jsonpickle.decode,
content_type='application/json',
content_encoding='utf-8')
app = Celery(
main=girder_worker.config.get('celery', 'app_main'),
task_cls='girder_worker.task:Task')
try:
include_core_tasks = config.getboolean(
'girder_worker', 'core_tasks')
except (NoSectionError, NoOptionError):
include_core_tasks = True
discover_tasks(app, core=include_core_tasks)
app.config_from_object('girder_worker.celeryconfig', force=True)