def insert_provider(provider): """Insert the given provider into the database. Args: provider (Provider): The provider model. Returns: ObjectId: The inserted provider id Raises: ValueError: If an invalid provider type is specified """ # We validate in case the provider was created outside our create_provider method. values = set(item.value for item in ProviderClass) if not provider.provider_class in values: raise errors.ValidationError('Unregistered provider class specified') provider.created = datetime.datetime.utcnow() provider.modified = datetime.datetime.utcnow() provider.validate() provider.validate_permissions() # All was good, create the mapper and insert, errors will bubble up mapper = mappers.Providers() return mapper.insert(provider)
def validate_job_compute_provider(job_map, request_handler, validate_provider=False): """Verify that the user can set compute_provider_id, if provided. Checks if compute_provider_id is set in job_map, and if so verifies that the user has the correct permissions. Returns: str: The compute_provider_id if specified, otherwise None Raises: APIPermissionException: If a non-admin user attempts to override provider APIValidataionException: If validate_provider is true and the provider either doesn't exist, or is not a compute provider """ compute_provider_id = job_map.get('compute_provider_id') if compute_provider_id: if not request_handler.user_is_admin: raise errors.PermissionError('Only admin can override job provider!') if validate_provider: try: validate_provider_class(compute_provider_id, 'compute') except errors.ResourceNotFound: raise errors.ValidationError('Provider id is not a regsitered provider on this system') return compute_provider_id
def update_provider(provider_id, doc): """Update the given provider instance, with fields from doc. Args: provider_id (ObjectId|str): The provider id doc (dict): The update fields Raises: APINotFoundException: If the provider does not exist. APIValidationException: If the update would result in an invalid provider configuration, or an invalid field was specified (e.g. attempt to change provider type) """ mapper = mappers.Providers() current_provider = mapper.get(provider_id) if not current_provider: raise errors.ResourceNotFound(provider_id, 'Provider {path} not found!') # NOTE: We do NOT permit updating provider class or type if 'provider_class' in doc: raise errors.ValidationError('Cannot change provider class!') if 'provider_type' in doc: raise errors.ValidationError('Cannot change provider type!') if 'label' in doc: current_provider.label = doc['label'] # If we do it this way we can only ever update keys not delete them if 'config' in doc: current_provider.config = doc['config'] #for key in doc['config']: # current_provider.config[key] = doc['config'][key] current_provider.modifed = datetime.datetime.utcnow() current_provider.validate() if 'creds' in doc: # Do full validation if the creds are changed to confirm they are correct provider = create_provider(current_provider.provider_class, current_provider.provider_type, current_provider.label, current_provider.config, doc['creds'], provider_id) provider.validate_permissions() mapper.patch(provider_id, current_provider)
def validate_provider_class(provider_id, provider_class): """Validate that the given provider exists, and has the given class. Args: provider_id (str): The provider id provider_class (str|ProviderClass): The provider class Raises: APIValidationException: If the provider either doesn't exist or is not of the specified class. """ provider_class = ProviderClass(provider_class).value mapper = mappers.Providers() result = mapper.get(provider_id) if not result: raise errors.ResourceNotFound(provider_id, 'Provider {path} does not exist') if result.provider_class != provider_class: raise errors.ValidationError( 'Provider {} is not a {} provider!'.format(provider_id, provider_class.value))
def validate_provider_updates(container, provider_ids, is_admin): """Validate an update (or setting) of provider ids. Allows setting or changing a compute provider on the container as long as the user is an admin and the provider exists. Allows setting the storage provider on the container as long as: 1. User is a site admin 2. The provider exists 3. A provider isn't already set Setting either provider to the current value is a no-op and doesn't trigger authorization errors. Side-effect: This will convert any IDs in the provider_ids parameter to ObjectIds. Args: container (dict): The current container (or empty if it doesn't exist) provider_ids (dict): The provider ids to update. is_admin (bool): Whether or not the user is a site administrator Raises: APIPermissionException: If the user is unautorized to make the change. APIValidationException: If the user attempted an invalid transition. APINotFoundException: If the given storage provider does not exist. """ # Early return for empty provider_ids object if not provider_ids: return # First check if this is a change updates = {} current_provider_ids = container.get('providers') or {} for provider_class in ('compute', 'storage'): updates[provider_class] = False if provider_class in provider_ids: # Ensure ObjectId provider_ids[provider_class] = bson.ObjectId( provider_ids[provider_class]) current_id = current_provider_ids.get(provider_class) if current_id != provider_ids[provider_class]: if current_id: raise errors.ValidationError( 'Cannot change {} provider once set!'.format( provider_class)) # Its only an update if current is set updates[provider_class] = True if (updates['storage'] or updates['compute']) and not is_admin: raise errors.PermissionError('Changing providers requires site-admin!') # Verify that provider exists and is the correct type for provider_class in ('compute', 'storage'): if not updates[provider_class]: continue provider = get_provider(provider_ids[provider_class]) if provider.provider_class != ProviderClass(provider_class).value: raise errors.ValidationError('Invalid provider class: {}'.format( provider.provider_class))
def validate(self): # Only empty configuration is valid if self.config: raise errors.ValidationError( 'Static Compute should have NO configuration!')
def enqueue_job(job_map, origin, perm_check_uid=None): """ Using a payload for a proposed job, creates and returns (but does not insert) a Job object. This preperation includes: - confirms gear exists - validates config against gear manifest - creating file reference objects for inputs - if given a perm_check_uid, method will check if user has proper access to inputs - confirming inputs exist - creating container reference object for destination - preparing file contexts - job api key generation, if requested """ # gear and config manifest check gear_id = job_map.get('gear_id') if not gear_id: raise errors.InputValidationException('Job must specify gear') gear = get_gear(gear_id) # Invalid disables a gear from running entirely. # https://github.com/flywheel-io/gears/tree/master/spec#reserved-custom-keys if gear.get('gear', {}).get('custom', {}).get('flywheel', {}).get('invalid', False): raise errors.InputValidationException('Gear marked as invalid, will not run!') config_ = job_map.get('config', {}) validate_gear_config(gear, config_) # Translate maps to FileReferences inputs = {} for x in job_map.get('inputs', {}).keys(): # Ensure input is in gear manifest if x not in gear['gear']['inputs']: raise errors.InputValidationException('Job input {} is not listed in gear manifest'.format(x)) input_map = job_map['inputs'][x] if gear['gear']['inputs'][x]['base'] == 'file': try: inputs[x] = create_filereference_from_dictionary(input_map) except KeyError: raise errors.InputValidationException('Input {} does not have a properly formatted file reference.'.format(x)) else: inputs[x] = input_map # Add job tags, config, attempt number, and/or previous job ID, if present tags = job_map.get('tags', []) attempt = job_map.get('attempt', 1) previous_job_id = job_map.get('previous_job_id', None) batch = job_map.get('batch', None) # A batch id if this job is part of a batch run label = job_map.get('label', "") # Add destination container, or select one destination = None if job_map.get('destination', None) is not None: destination = create_containerreference_from_dictionary(job_map['destination']) else: destination = None for key in inputs.keys(): if isinstance(inputs[key], FileReference): destination = create_containerreference_from_filereference(inputs[key]) break if not destination: raise errors.InputValidationException('Must specify destination if gear has no inputs.') elif destination.type == 'analysis': raise errors.InputValidationException('Cannot use analysis for destination of a job, container was inferred.') # Get parents from destination, also checks that destination exists destination_container = destination.get() # Permission check if perm_check_uid: for x in inputs: if hasattr(inputs[x], 'check_access'): inputs[x].check_access(perm_check_uid, 'ro') destination.check_access(perm_check_uid, 'rw', cont=destination_container) # Config options are stored on the job object under the "config" key config_ = { 'config': fill_gear_default_values(gear, config_), 'inputs': { }, 'destination': { 'type': destination.type, 'id': destination.id, } } # Implementation notes: with regard to sending the gear file information, we have two options: # # 1) Send the file object as it existed when you enqueued the job # 2) Send the file object as it existed when the job was started # # Option #2 is possibly more convenient - it's more up to date - but the only file modifications after a job is enqueued would be from # # A) a gear finishing, and updating the file object # B) a user editing the file object # # You can count on neither occurring before a job starts, because the queue is not globally FIFO. # So option #2 is potentially more convenient, but unintuitive and prone to user confusion. input_file_count = 0 input_file_size_bytes = 0 related_containers = set() add_related_containers(related_containers, destination_container) file_inputs = [] for x in inputs: input_type = gear['gear']['inputs'][x]['base'] if input_type == 'file': input_container = inputs[x].get() add_related_containers(related_containers, input_container) obj = inputs[x].get_file(container=input_container) file_inputs.append(obj) cr = create_containerreference_from_filereference(inputs[x]) # Whitelist file fields passed to gear to those that are scientific-relevant whitelisted_keys = ['info', 'tags', 'measurements', 'classification', 'mimetype', 'type', 'modality', 'size'] obj_projection = { key: obj.get(key) for key in whitelisted_keys } input_file_count += 1 input_file_size_bytes += obj.get('size', 0) ### # recreate `measurements` list on object # Can be removed when `classification` key has been adopted everywhere if not obj_projection.get('measurements', None): obj_projection['measurements'] = [] if obj_projection.get('classification'): for v in obj_projection['classification'].itervalues(): obj_projection['measurements'].extend(v) # ### config_['inputs'][x] = { 'base': 'file', 'hierarchy': cr.__dict__, 'location': { 'name': obj['name'], 'path': '/flywheel/v0/input/' + x + '/' + obj['name'], }, 'object': obj_projection, } elif input_type == 'context': config_['inputs'][x] = inputs[x] else: # Note: API key inputs should not be passed as input raise Exception('Non-file input base type') # Populate any context inputs for the gear resolve_context_inputs(config_, gear, destination.type, destination.id, perm_check_uid) # Populate parents (including destination) parents = destination_container.get('parents', {}) parents[destination.type] = bson.ObjectId(destination.id) # Determine compute provider, if not provided compute_provider_id = job_map.get('compute_provider_id') if compute_provider_id is None: compute_provider_id = providers.get_compute_provider_id_for_job(gear, destination_container, file_inputs) # If compute provider is still undetermined, then we need to raise if compute_provider_id is None: raise errors.APIPreconditionFailed('Cannot determine compute provider for job. ' 'gear={}, destination.id={}'.format(gear['_id'], destination.id)) else: # Validate the provided compute provider try: providers.validate_provider_class(compute_provider_id, 'compute') except flywheel_errors.ResourceNotFound: raise flywheel_errors.ValidationError('Provider id is not valid') # Initialize profile profile = { 'total_input_files': input_file_count, 'total_input_size_bytes': input_file_size_bytes } release_version = config.get_release_version() if release_version: profile['versions'] = { 'core': release_version } gear_name = gear['gear']['name'] if gear_name not in tags: tags.append(gear_name) job = Job(gear, inputs, destination=destination, tags=tags, config_=config_, attempt=attempt, previous_job_id=previous_job_id, origin=origin, batch=batch, parents=parents, profile=profile, related_container_ids=list(related_containers), label=label, compute_provider_id=compute_provider_id) return job