def _construct_run_message(self, shared_file_system, bundle, bundle_resources): """ Constructs the run message that is sent to the given worker to tell it to run the given bundle. """ message = {} message['type'] = 'run' message['bundle'] = bundle_util.bundle_to_bundle_info( self._model, bundle) if shared_file_system: bundle_link_url = getattr(bundle.metadata, "link_url", None) message['bundle'][ 'location'] = bundle_link_url or self._bundle_store.get_bundle_location( bundle.uuid) parent_bundle_link_urls = self._model.get_bundle_metadata([ dep['parent_uuid'] for dep in message['bundle']['dependencies'] ], "link_url") for dependency in message['bundle']['dependencies']: parent_bundle_link_url = parent_bundle_link_urls.get( dependency['parent_uuid']) dependency['location'] = ( parent_bundle_link_url or self._bundle_store.get_bundle_location( dependency['parent_uuid'])) # Figure out the resource requirements. message['resources'] = bundle_resources.as_dict return message
def _construct_run_message(self, worker, bundle): """ Constructs the run message that is sent to the given worker to tell it to run the given bundle. """ message = {} message['type'] = 'run' message['bundle'] = bundle_util.bundle_to_bundle_info(self._model, bundle) if self._worker_model.shared_file_system and worker['user_id'] == self._model.root_user_id: message['bundle']['location'] = self._bundle_store.get_bundle_location(bundle.uuid) for dependency in message['bundle']['dependencies']: dependency['location'] = self._bundle_store.get_bundle_location(dependency['parent_uuid']) # Figure out the resource requirements. resources = message['resources'] = {} resources['request_cpus'] = self._compute_request_cpus(bundle) resources['request_gpus'] = self._compute_request_gpus(bundle) resources['docker_image'] = bundle.metadata.request_docker_image resources['request_time'] = self._compute_request_time(bundle) resources['request_memory'] = self._compute_request_memory(bundle) resources['request_disk'] = self._compute_request_disk(bundle) resources['request_network'] = bundle.metadata.request_network return message
def _construct_run_message(self, worker, bundle): """ Constructs the run message that is sent to the given worker to tell it to run the given bundle. """ message = {} message['type'] = 'run' message['bundle'] = bundle_util.bundle_to_bundle_info(self._model, bundle) if self._worker_model.shared_file_system and worker['user_id'] == self._model.root_user_id: message['bundle']['location'] = self._bundle_store.get_bundle_location(bundle.uuid) for dependency in message['bundle']['dependencies']: dependency['location'] = self._bundle_store.get_bundle_location( dependency['parent_uuid'] ) # Figure out the resource requirements. resources = message['resources'] = {} resources['request_cpus'] = self._compute_request_cpus(bundle) resources['request_gpus'] = self._compute_request_gpus(bundle) resources['docker_image'] = self._get_docker_image(bundle) resources['request_time'] = self._compute_request_time(bundle) resources['request_memory'] = self._compute_request_memory(bundle) resources['request_disk'] = self._compute_request_disk(bundle) resources['request_network'] = bundle.metadata.request_network return message
def _construct_run_message(self, worker, bundle): """ Constructs the run message that is sent to the given worker to tell it to run the given bundle. """ message = {} message['type'] = 'run' message['bundle'] = bundle_util.bundle_to_bundle_info(self._model, bundle) if self._worker_model.shared_file_system and worker['user_id'] == self._model.root_user_id: message['bundle']['location'] = self._bundle_store.get_bundle_location(bundle.uuid) for dependency in message['bundle']['dependencies']: dependency['location'] = self._bundle_store.get_bundle_location(dependency['parent_uuid']) # Figure out the resource requirements. resources = message['resources'] = {} resources['docker_image'] = (bundle.metadata.request_docker_image or self._default_docker_image) # Parse |request_string| using |to_value|, but don't exceed |max_value|. def parse_and_min(to_value, request_string, default_value, max_value): # On user-owned workers, ignore the maximum value. Users are free to # use as many resources as they wish on their own machines. if worker['user_id'] != self._model.root_user_id: max_value = None # Use default if request value doesn't exist if request_string: request_value = to_value(request_string) else: request_value = default_value if request_value and max_value: return int(min(request_value, max_value)) elif request_value: return int(request_value) elif max_value: return int(max_value) else: return None # These limits are used for killing runs that use too many resources. resources['request_time'] = parse_and_min(formatting.parse_duration, bundle.metadata.request_time, self._default_request_time, self._max_request_time) resources['request_memory'] = parse_and_min(formatting.parse_size, bundle.metadata.request_memory, self._default_request_memory, self._max_request_memory) resources['request_disk'] = parse_and_min(formatting.parse_size, bundle.metadata.request_disk, self._default_request_disk, self._max_request_disk) resources['request_network'] = (bundle.metadata.request_network or self._default_request_network) return message
def _construct_run_message(self, worker, bundle, bundle_resources): """ Constructs the run message that is sent to the given worker to tell it to run the given bundle. """ message = {} message['type'] = 'run' message['bundle'] = bundle_util.bundle_to_bundle_info(self._model, bundle) if worker['shared_file_system']: message['bundle']['location'] = self._bundle_store.get_bundle_location(bundle.uuid) for dependency in message['bundle']['dependencies']: dependency['location'] = self._bundle_store.get_bundle_location( dependency['parent_uuid'] ) # Figure out the resource requirements. message['resources'] = bundle_resources.as_dict return message
def get_bundle_infos( uuids, get_children=False, get_host_worksheets=False, get_permissions=False, ignore_not_found=True, ): """ Return a map from bundle uuid to info. :param Collection[str] uuids: uuids of bundles to fetch :param bool get_children: include children :param bool get_host_worksheets: include host worksheets :param bool get_permissions: include group permissions :param bool ignore_not_found: abort with 404 NOT FOUND when False and bundle doesn't exist :rtype: dict[str, dict] """ if len(uuids) == 0: return {} bundles = local.model.batch_get_bundles(uuid=uuids) bundle_infos = { bundle.uuid: bundle_util.bundle_to_bundle_info(local.model, bundle) for bundle in bundles } # Implement permissions policies perms = _get_user_bundle_permissions(uuids) readable = { u for u, perm in perms.items() if perm >= GROUP_OBJECT_PERMISSION_READ } anonymous = { u for u, perm in perms.items() if u in bundle_infos and (perm < GROUP_OBJECT_PERMISSION_READ or bundle_infos[u]['is_anonymous']) } for uuid in uuids: bundle = bundle_infos.get(uuid) # Bundle doesn't exist; abort or skip if bundle is None: if ignore_not_found: continue else: abort(http.client.NOT_FOUND, "Bundle %s not found" % uuid) # Replace bundles that the user does not have read access to elif uuid not in readable: bundle_infos[uuid] = bundle_util.bundle_to_bundle_info( local.model, PrivateBundle.construct(uuid)) # Mask owners of anonymous bundles that user does not have all acccess to elif uuid in anonymous: bundle['owner_id'] = None # Set permission bundle['permission'] = perms[uuid] if get_children: parent2children = local.model.get_children_uuids(readable) # Gather all children bundle uuids and fetch permissions child_uuids = [uuid for l in parent2children.values() for uuid in l] child_perms = _get_user_bundle_permissions(child_uuids) # Lookup bundle names child_names = local.model.get_bundle_names(child_uuids) # Set children infos for parent_uuid, children in parent2children.items(): bundle_infos[parent_uuid]['children'] = [ { 'uuid': child_uuid, 'metadata': { 'name': child_names[child_uuid] } } for child_uuid in children if child_perms[child_uuid] >= GROUP_OBJECT_PERMISSION_READ ] if get_host_worksheets: # bundle_uuids -> list of worksheet_uuids host_worksheets = local.model.get_host_worksheet_uuids(readable) # Gather all worksheet uuids worksheet_uuids = [ uuid for l in host_worksheets.values() for uuid in l ] wpermissions = local.model.get_user_worksheet_permissions( request.user.user_id, worksheet_uuids, local.model.get_worksheet_owner_ids(worksheet_uuids), ) readable_worksheet_uuids = set( uuid for uuid, permission in wpermissions.items() if permission >= GROUP_OBJECT_PERMISSION_READ) # Lookup names worksheets = dict( (worksheet.uuid, worksheet) for worksheet in local.model.batch_get_worksheets( fetch_items=False, uuid=readable_worksheet_uuids)) # Fill the info for bundle_uuid, host_uuids in host_worksheets.items(): bundle_infos[bundle_uuid]['host_worksheets'] = [ { 'uuid': host_uuid, 'name': worksheets[host_uuid].name } for host_uuid in host_uuids if host_uuid in readable_worksheet_uuids ] if get_permissions: # Fill the permissions info bundle2group_perms = local.model.batch_get_group_bundle_permissions( request.user.user_id, readable) for uuid, group_perms in bundle2group_perms.items(): # Only show group permissions to the user is they have # at least read permission on this bundle. if uuid in anonymous: bundle_infos[uuid]['group_permissions'] = [] else: bundle_infos[uuid]['group_permissions'] = group_perms return bundle_infos
def get_bundle_infos( uuids, get_children=False, get_single_host_worksheet=False, get_host_worksheets=False, get_permissions=False, ignore_not_found=True, model=None, ): """ Return a map from bundle uuid to info. :param Collection[str] uuids: uuids of bundles to fetch :param bool get_children: include children :param bool get_single_host_worksheet: include one host_worksheet per bundle uuid :param bool get_host_worksheets: include all host worksheets :param bool get_permissions: include group permissions :param bool ignore_not_found: abort with 404 NOT FOUND when False and bundle doesn't exist :param BundleModel model: model used to make database queries :rtype: dict[str, dict] """ if model is None: model = local.model if len(uuids) == 0: return {} bundles = model.batch_get_bundles(uuid=uuids) bundle_infos = { bundle.uuid: bundle_util.bundle_to_bundle_info(model, bundle) for bundle in bundles } # Implement permissions policies perms = _get_user_bundle_permissions(model, uuids) readable = { u for u, perm in perms.items() if perm >= GROUP_OBJECT_PERMISSION_READ } anonymous = { u for u, perm in perms.items() if u in bundle_infos and (perm < GROUP_OBJECT_PERMISSION_READ or bundle_infos[u]['is_anonymous']) } for uuid in uuids: bundle = bundle_infos.get(uuid) # Bundle doesn't exist; abort or skip if bundle is None: if ignore_not_found: continue else: abort(http.client.NOT_FOUND, "Bundle %s not found" % uuid) # Replace bundles that the user does not have read access to elif uuid not in readable: bundle_infos[uuid] = bundle_util.bundle_to_bundle_info( model, PrivateBundle.construct(uuid)) # Mask owners of anonymous bundles that user does not have all access to elif uuid in anonymous: bundle['owner_id'] = None # Set permission bundle['permission'] = perms[uuid] if get_children: parent2children = model.get_children_uuids(readable) # Gather all children bundle uuids and fetch permissions child_uuids = [uuid for l in parent2children.values() for uuid in l] child_perms = _get_user_bundle_permissions(model, child_uuids) # Lookup bundle names child_names = model.get_bundle_names(child_uuids) # Set children infos for parent_uuid, children in parent2children.items(): bundle_infos[parent_uuid]['children'] = [ { 'uuid': child_uuid, 'metadata': { 'name': child_names[child_uuid] } } for child_uuid in children if child_perms[child_uuid] >= GROUP_OBJECT_PERMISSION_READ ] if get_single_host_worksheet: # Query for 5 worksheet uuids per bundle to check the read permissions for, since we # just need a single host worksheet per bundle uuid. This is much faster than fetching all # worksheet uuid's per bundle. host_worksheets = model.get_host_worksheet_uuids(readable, 5) worksheet_uuids = [ uuid for l in host_worksheets.values() for uuid in l ] worksheet_names = _get_readable_worksheet_names(model, worksheet_uuids) for bundle_uuid, host_uuids in host_worksheets.items(): if bundle_uuid not in bundle_infos: continue for host_uuid in host_uuids: if host_uuid in worksheet_names: bundle_infos[bundle_uuid]['host_worksheet'] = { 'uuid': host_uuid, 'name': worksheet_names[host_uuid], } # Just set a single host worksheet per bundle uuid break if get_host_worksheets: host_worksheets = model.get_all_host_worksheet_uuids(readable) # Gather all worksheet uuids worksheet_uuids = [ uuid for l in host_worksheets.values() for uuid in l ] worksheet_names = _get_readable_worksheet_names(model, worksheet_uuids) # Fill the info for bundle_uuid, host_uuids in host_worksheets.items(): if bundle_uuid not in bundle_infos: continue bundle_infos[bundle_uuid]['host_worksheets'] = [{ 'uuid': host_uuid, 'name': worksheet_names[host_uuid] } for host_uuid in host_uuids if host_uuid in worksheet_names] if get_permissions: # Fill the permissions info bundle2group_perms = model.batch_get_group_bundle_permissions( request.user.user_id, readable) for uuid, group_perms in bundle2group_perms.items(): # Only show group permissions to the user is they have # at least read permission on this bundle. if uuid in anonymous: bundle_infos[uuid]['group_permissions'] = [] else: bundle_infos[uuid]['group_permissions'] = group_perms return bundle_infos
def _construct_run_message(self, worker, bundle): """ Constructs the run message that is sent to the given worker to tell it to run the given bundle. """ message = {} message['type'] = 'run' message['bundle'] = bundle_util.bundle_to_bundle_info( self._model, bundle) if self._worker_model.shared_file_system and worker[ 'user_id'] == self._model.root_user_id: message['bundle'][ 'location'] = self._bundle_store.get_bundle_location( bundle.uuid) for dependency in message['bundle']['dependencies']: dependency[ 'location'] = self._bundle_store.get_bundle_location( dependency['parent_uuid']) # Figure out the resource requirements. resources = message['resources'] = {} resources['docker_image'] = (bundle.metadata.request_docker_image or self._default_docker_image) # Parse |request_string| using |to_value|, but don't exceed |max_value|. def parse_and_min(to_value, request_string, default_value, max_value): # On user-owned workers, ignore the maximum value. Users are free to # use as many resources as they wish on their own machines. if worker['user_id'] != self._model.root_user_id: max_value = None # Use default if request value doesn't exist if request_string: request_value = to_value(request_string) else: request_value = default_value if request_value and max_value: return int(min(request_value, max_value)) elif request_value: return int(request_value) elif max_value: return int(max_value) else: return None # These limits are used for killing runs that use too many resources. resources['request_time'] = parse_and_min(formatting.parse_duration, bundle.metadata.request_time, self._default_request_time, self._max_request_time) resources['request_memory'] = parse_and_min( formatting.parse_size, bundle.metadata.request_memory, self._default_request_memory, self._max_request_memory) resources['request_disk'] = parse_and_min(formatting.parse_size, bundle.metadata.request_disk, self._default_request_disk, self._max_request_disk) resources['request_network'] = (bundle.metadata.request_network or self._default_request_network) return message
def get_bundle_infos(uuids, get_children=False, get_host_worksheets=False, get_permissions=False): """ get_children, get_host_worksheets, get_permissions: whether we want to return more detailed information. Return map from bundle uuid to info. """ if len(uuids) == 0: return {} bundles = local.model.batch_get_bundles(uuid=uuids) bundle_dict = { bundle.uuid: bundle_util.bundle_to_bundle_info(local.model, bundle) for bundle in bundles } # Filter out bundles that we don't have read permission on def select_unreadable_bundles(uuids): permissions = local.model.get_user_bundle_permissions( request.user.user_id, uuids, local.model.get_bundle_owner_ids(uuids)) return [ uuid for uuid, permission in permissions.items() if permission < GROUP_OBJECT_PERMISSION_READ ] def select_unreadable_worksheets(uuids): permissions = local.model.get_user_worksheet_permissions( request.user.user_id, uuids, local.model.get_worksheet_owner_ids(uuids)) return [ uuid for uuid, permission in permissions.items() if permission < GROUP_OBJECT_PERMISSION_READ ] # Lookup the user names of all the owners user_ids = [info['owner_id'] for info in bundle_dict.values()] users = local.model.get_users( user_ids=user_ids) if len(user_ids) > 0 else [] users = {u.user_id: u for u in users} if users: for info in bundle_dict.values(): user = users[info['owner_id']] info['owner_name'] = user.user_name if user else None info['owner'] = '%s(%s)' % (info['owner_name'], info['owner_id']) # Mask bundles that we can't access for uuid in select_unreadable_bundles(uuids): if uuid in bundle_dict: bundle_dict[uuid] = bundle_util.bundle_to_bundle_info( local.model, PrivateBundle.construct(uuid)) if get_children: result = local.model.get_children_uuids(uuids) # Gather all children bundle uuids children_uuids = [uuid for l in result.values() for uuid in l] unreadable = set(select_unreadable_bundles(children_uuids)) # Lookup bundle names names = local.model.get_bundle_names(children_uuids) # Fill in info for uuid, info in bundle_dict.items(): info['children'] = [{ 'uuid': child_uuid, 'metadata': { 'name': names[child_uuid] } } for child_uuid in result[uuid] if child_uuid not in unreadable] if get_host_worksheets: # bundle_uuids -> list of worksheet_uuids result = local.model.get_host_worksheet_uuids(uuids) # Gather all worksheet uuids worksheet_uuids = [uuid for l in result.values() for uuid in l] unreadable = set(select_unreadable_worksheets(worksheet_uuids)) worksheet_uuids = [ uuid for uuid in worksheet_uuids if uuid not in unreadable ] # Lookup names worksheets = dict((worksheet.uuid, worksheet) for worksheet in local.model.batch_get_worksheets( fetch_items=False, uuid=worksheet_uuids)) # Fill the info for uuid, info in bundle_dict.items(): info['host_worksheets'] = [{ 'uuid': worksheet_uuid, 'name': worksheets[worksheet_uuid].name } for worksheet_uuid in result[uuid] if worksheet_uuid not in unreadable] if get_permissions: # Fill the info group_result = local.model.batch_get_group_bundle_permissions( request.user.user_id, uuids) result = local.model.get_user_bundle_permissions( request.user.user_id, uuids, local.model.get_bundle_owner_ids(uuids)) for uuid, info in bundle_dict.items(): info['group_permissions'] = group_result[uuid] info['permission'] = result[uuid] return bundle_dict