def insert_provider(provider):
"""Insert the given provider into the database.
Args:
provider (Provider): The provider model.
Returns:
ObjectId: The inserted provider id
Raises:
ValueError: If an invalid provider type is specified
"""
# We validate in case the provider was created outside our create_provider method.
values = set(item.value for item in ProviderClass)
if not provider.provider_class in values:
raise errors.ValidationError('Unregistered provider class specified')
provider.created = datetime.datetime.utcnow()
provider.modified = datetime.datetime.utcnow()
provider.validate()
provider.validate_permissions()
# All was good, create the mapper and insert, errors will bubble up
mapper = mappers.Providers()
return mapper.insert(provider)
def validate_job_compute_provider(job_map, request_handler, validate_provider=False):
"""Verify that the user can set compute_provider_id, if provided.
Checks if compute_provider_id is set in job_map, and if so verifies that
the user has the correct permissions.
Returns:
str: The compute_provider_id if specified, otherwise None
Raises:
APIPermissionException: If a non-admin user attempts to override provider
APIValidataionException: If validate_provider is true and the provider either
doesn't exist, or is not a compute provider
"""
compute_provider_id = job_map.get('compute_provider_id')
if compute_provider_id:
if not request_handler.user_is_admin:
raise errors.PermissionError('Only admin can override job provider!')
if validate_provider:
try:
validate_provider_class(compute_provider_id, 'compute')
except errors.ResourceNotFound:
raise errors.ValidationError('Provider id is not a regsitered provider on this system')
return compute_provider_id
def update_provider(provider_id, doc):
"""Update the given provider instance, with fields from doc.
Args:
provider_id (ObjectId|str): The provider id
doc (dict): The update fields
Raises:
APINotFoundException: If the provider does not exist.
APIValidationException: If the update would result in an invalid provider
configuration, or an invalid field was specified
(e.g. attempt to change provider type)
"""
mapper = mappers.Providers()
current_provider = mapper.get(provider_id)
if not current_provider:
raise errors.ResourceNotFound(provider_id,
'Provider {path} not found!')
# NOTE: We do NOT permit updating provider class or type
if 'provider_class' in doc:
raise errors.ValidationError('Cannot change provider class!')
if 'provider_type' in doc:
raise errors.ValidationError('Cannot change provider type!')
if 'label' in doc:
current_provider.label = doc['label']
# If we do it this way we can only ever update keys not delete them
if 'config' in doc:
current_provider.config = doc['config']
#for key in doc['config']:
# current_provider.config[key] = doc['config'][key]
current_provider.modifed = datetime.datetime.utcnow()
current_provider.validate()
if 'creds' in doc:
# Do full validation if the creds are changed to confirm they are correct
provider = create_provider(current_provider.provider_class,
current_provider.provider_type,
current_provider.label,
current_provider.config, doc['creds'],
provider_id)
provider.validate_permissions()
mapper.patch(provider_id, current_provider)
def validate_provider_class(provider_id, provider_class):
"""Validate that the given provider exists, and has the given class.
Args:
provider_id (str): The provider id
provider_class (str|ProviderClass): The provider class
Raises:
APIValidationException: If the provider either doesn't exist or is not of the specified class.
"""
provider_class = ProviderClass(provider_class).value
mapper = mappers.Providers()
result = mapper.get(provider_id)
if not result:
raise errors.ResourceNotFound(provider_id,
'Provider {path} does not exist')
if result.provider_class != provider_class:
raise errors.ValidationError(
'Provider {} is not a {} provider!'.format(provider_id,
provider_class.value))
def validate_provider_updates(container, provider_ids, is_admin):
"""Validate an update (or setting) of provider ids.
Allows setting or changing a compute provider on the container as long
as the user is an admin and the provider exists.
Allows setting the storage provider on the container as long as:
1. User is a site admin
2. The provider exists
3. A provider isn't already set
Setting either provider to the current value is a no-op and doesn't
trigger authorization errors.
Side-effect: This will convert any IDs in the provider_ids parameter to ObjectIds.
Args:
container (dict): The current container (or empty if it doesn't exist)
provider_ids (dict): The provider ids to update.
is_admin (bool): Whether or not the user is a site administrator
Raises:
APIPermissionException: If the user is unautorized to make the change.
APIValidationException: If the user attempted an invalid transition.
APINotFoundException: If the given storage provider does not exist.
"""
# Early return for empty provider_ids object
if not provider_ids:
return
# First check if this is a change
updates = {}
current_provider_ids = container.get('providers') or {}
for provider_class in ('compute', 'storage'):
updates[provider_class] = False
if provider_class in provider_ids:
# Ensure ObjectId
provider_ids[provider_class] = bson.ObjectId(
provider_ids[provider_class])
current_id = current_provider_ids.get(provider_class)
if current_id != provider_ids[provider_class]:
if current_id:
raise errors.ValidationError(
'Cannot change {} provider once set!'.format(
provider_class))
# Its only an update if current is set
updates[provider_class] = True
if (updates['storage'] or updates['compute']) and not is_admin:
raise errors.PermissionError('Changing providers requires site-admin!')
# Verify that provider exists and is the correct type
for provider_class in ('compute', 'storage'):
if not updates[provider_class]:
continue
provider = get_provider(provider_ids[provider_class])
if provider.provider_class != ProviderClass(provider_class).value:
raise errors.ValidationError('Invalid provider class: {}'.format(
provider.provider_class))
def validate(self):
# Only empty configuration is valid
if self.config:
raise errors.ValidationError(
'Static Compute should have NO configuration!')
def enqueue_job(job_map, origin, perm_check_uid=None):
"""
Using a payload for a proposed job, creates and returns (but does not insert)
a Job object. This preperation includes:
- confirms gear exists
- validates config against gear manifest
- creating file reference objects for inputs
- if given a perm_check_uid, method will check if user has proper access to inputs
- confirming inputs exist
- creating container reference object for destination
- preparing file contexts
- job api key generation, if requested
"""
# gear and config manifest check
gear_id = job_map.get('gear_id')
if not gear_id:
raise errors.InputValidationException('Job must specify gear')
gear = get_gear(gear_id)
# Invalid disables a gear from running entirely.
# https://github.com/flywheel-io/gears/tree/master/spec#reserved-custom-keys
if gear.get('gear', {}).get('custom', {}).get('flywheel', {}).get('invalid', False):
raise errors.InputValidationException('Gear marked as invalid, will not run!')
config_ = job_map.get('config', {})
validate_gear_config(gear, config_)
# Translate maps to FileReferences
inputs = {}
for x in job_map.get('inputs', {}).keys():
# Ensure input is in gear manifest
if x not in gear['gear']['inputs']:
raise errors.InputValidationException('Job input {} is not listed in gear manifest'.format(x))
input_map = job_map['inputs'][x]
if gear['gear']['inputs'][x]['base'] == 'file':
try:
inputs[x] = create_filereference_from_dictionary(input_map)
except KeyError:
raise errors.InputValidationException('Input {} does not have a properly formatted file reference.'.format(x))
else:
inputs[x] = input_map
# Add job tags, config, attempt number, and/or previous job ID, if present
tags = job_map.get('tags', [])
attempt = job_map.get('attempt', 1)
previous_job_id = job_map.get('previous_job_id', None)
batch = job_map.get('batch', None) # A batch id if this job is part of a batch run
label = job_map.get('label', "")
# Add destination container, or select one
destination = None
if job_map.get('destination', None) is not None:
destination = create_containerreference_from_dictionary(job_map['destination'])
else:
destination = None
for key in inputs.keys():
if isinstance(inputs[key], FileReference):
destination = create_containerreference_from_filereference(inputs[key])
break
if not destination:
raise errors.InputValidationException('Must specify destination if gear has no inputs.')
elif destination.type == 'analysis':
raise errors.InputValidationException('Cannot use analysis for destination of a job, container was inferred.')
# Get parents from destination, also checks that destination exists
destination_container = destination.get()
# Permission check
if perm_check_uid:
for x in inputs:
if hasattr(inputs[x], 'check_access'):
inputs[x].check_access(perm_check_uid, 'ro')
destination.check_access(perm_check_uid, 'rw', cont=destination_container)
# Config options are stored on the job object under the "config" key
config_ = {
'config': fill_gear_default_values(gear, config_),
'inputs': { },
'destination': {
'type': destination.type,
'id': destination.id,
}
}
# Implementation notes: with regard to sending the gear file information, we have two options:
#
# 1) Send the file object as it existed when you enqueued the job
# 2) Send the file object as it existed when the job was started
#
# Option #2 is possibly more convenient - it's more up to date - but the only file modifications after a job is enqueued would be from
#
# A) a gear finishing, and updating the file object
# B) a user editing the file object
#
# You can count on neither occurring before a job starts, because the queue is not globally FIFO.
# So option #2 is potentially more convenient, but unintuitive and prone to user confusion.
input_file_count = 0
input_file_size_bytes = 0
related_containers = set()
add_related_containers(related_containers, destination_container)
file_inputs = []
for x in inputs:
input_type = gear['gear']['inputs'][x]['base']
if input_type == 'file':
input_container = inputs[x].get()
add_related_containers(related_containers, input_container)
obj = inputs[x].get_file(container=input_container)
file_inputs.append(obj)
cr = create_containerreference_from_filereference(inputs[x])
# Whitelist file fields passed to gear to those that are scientific-relevant
whitelisted_keys = ['info', 'tags', 'measurements', 'classification', 'mimetype', 'type', 'modality', 'size']
obj_projection = { key: obj.get(key) for key in whitelisted_keys }
input_file_count += 1
input_file_size_bytes += obj.get('size', 0)
###
# recreate `measurements` list on object
# Can be removed when `classification` key has been adopted everywhere
if not obj_projection.get('measurements', None):
obj_projection['measurements'] = []
if obj_projection.get('classification'):
for v in obj_projection['classification'].itervalues():
obj_projection['measurements'].extend(v)
#
###
config_['inputs'][x] = {
'base': 'file',
'hierarchy': cr.__dict__,
'location': {
'name': obj['name'],
'path': '/flywheel/v0/input/' + x + '/' + obj['name'],
},
'object': obj_projection,
}
elif input_type == 'context':
config_['inputs'][x] = inputs[x]
else:
# Note: API key inputs should not be passed as input
raise Exception('Non-file input base type')
# Populate any context inputs for the gear
resolve_context_inputs(config_, gear, destination.type, destination.id, perm_check_uid)
# Populate parents (including destination)
parents = destination_container.get('parents', {})
parents[destination.type] = bson.ObjectId(destination.id)
# Determine compute provider, if not provided
compute_provider_id = job_map.get('compute_provider_id')
if compute_provider_id is None:
compute_provider_id = providers.get_compute_provider_id_for_job(gear, destination_container, file_inputs)
# If compute provider is still undetermined, then we need to raise
if compute_provider_id is None:
raise errors.APIPreconditionFailed('Cannot determine compute provider for job. '
'gear={}, destination.id={}'.format(gear['_id'], destination.id))
else:
# Validate the provided compute provider
try:
providers.validate_provider_class(compute_provider_id, 'compute')
except flywheel_errors.ResourceNotFound:
raise flywheel_errors.ValidationError('Provider id is not valid')
# Initialize profile
profile = {
'total_input_files': input_file_count,
'total_input_size_bytes': input_file_size_bytes
}
release_version = config.get_release_version()
if release_version:
profile['versions'] = { 'core': release_version }
gear_name = gear['gear']['name']
if gear_name not in tags:
tags.append(gear_name)
job = Job(gear, inputs, destination=destination, tags=tags, config_=config_, attempt=attempt,
previous_job_id=previous_job_id, origin=origin, batch=batch, parents=parents, profile=profile,
related_container_ids=list(related_containers), label=label, compute_provider_id=compute_provider_id)
return job