def test_unpack_document(self): obj = self.client._unpack_document( { 'data': { 'id': '123', 'type': 'bundles', 'attributes': {'name': 'hello'}, 'relationships': { 'owner': {'data': {'id': '345', 'type': 'users'}}, 'parent': {'data': {'id': '567', 'type': 'bundles'}}, }, }, 'included': [ { 'type': 'users', 'id': '345', 'attributes': {'name': 'percy', 'affiliation': 'stanford'}, } ], } ) self.assertDictEqual( obj, { 'owner': JsonApiRelationship( 'users', '345', {'name': 'percy', 'affiliation': 'stanford'} ), 'parent': JsonApiRelationship('bundles', '567'), 'id': '123', 'name': 'hello', }, )
def _make_public_readable(self, bundle): """ Make the given bundle readable to the public. """ # Get public group info public = self.client.fetch('groups', 'public') # Set permissions self.client.create( 'bundle-permissions', { 'group': JsonApiRelationship('groups', public['id']), 'bundle': JsonApiRelationship('bundles', bundle['id']), 'permission': 1, })
def test_pack_document(self): doc = self.client._pack_document({ 'owner': JsonApiRelationship('users', '345'), 'friend': EmptyJsonApiRelationship(), 'id': '123', 'name': 'hello' }, 'bundles') self.assertDictEqual(doc, { 'data': { 'id': '123', 'type': 'bundles', 'attributes': { 'name': 'hello' }, 'relationships': { 'owner': { 'data': { 'id': '345', 'type': 'users' } }, 'friend': { 'data': None } }, } })
def newline(): if not skip_prelude: client.create('worksheet-items', data={ 'type': worksheet_util.TYPE_MARKUP, 'worksheet': JsonApiRelationship('worksheets', worksheet_uuid), 'value': '', })
def ensure_log_worksheet_private(self): """ Ensure that the leaderboard worksheet is private, so that all bundles created on it are automatically private. """ if self.config['make_predictions_public']: return # Get public group info public = self.client.fetch('groups', 'public') # Set permissions self.client.create( 'worksheet-permissions', { 'group': JsonApiRelationship('groups', public['id']), 'worksheet': JsonApiRelationship('worksheets', self.config['log_worksheet_uuid']), 'permission': 0, })
def mimic_bundles( client, old_inputs, old_output, new_inputs, new_output_name, worksheet_uuid, depth, shadow, dry_run, metadata_override=None, skip_prelude=False, ): """ :param JsonApiClient client: client :param old_inputs: list of bundle uuids :param old_output: bundle uuid that we produced :param new_inputs: list of bundle uuids that are analogous to old_inputs :param new_output_name: name of the bundle to create to be analogous to old_output (possibly None) :param worksheet_uuid: add newly created bundles to this worksheet :param depth: how far to do a BFS up from old_output. :param shadow: whether to add the new inputs right after all occurrences of the old inputs in worksheets. :param dry_run: return plan without executing anything if True. :param skip_prelude: don't include preludes in mimicked items if True. :param metadata_override: new metadata fields replace old ones in the newly mimicked bundles. """ if metadata_override is None: metadata_override = {} # Build the graph (get all the infos). # If old_output is given, look at ancestors of old_output until we # reached some depth. If it's not given, we first get all the # descendants first, and then get their ancestors. if old_output: infos = client.fetch('bundles', params={'specs': old_output}) assert isinstance(infos, list) else: # Fetch bundles specified in `old_inputs` and their descendants # down by `depth` levesl infos = client.fetch('bundles', params={'specs': old_inputs, 'depth': depth}) infos = {b['uuid']: b for b in infos} # uuid -> bundle info def get_self_and_ancestors(bundle_uuids): # Traverse up ancestors by at most `depth` levels and returns # the set of all bundles visited, as well as updating the `info` # dictionary along the way. result = bundle_uuids visited = set() for _ in range(depth): next_bundle_uuids = [] for bundle_uuid in bundle_uuids: if bundle_uuid in visited: continue # Add to infos if not there yet if bundle_uuid not in infos: infos[bundle_uuid] = client.fetch('bundles', bundle_uuid) # Append all of the parents to the next batch of bundles to look at info = infos[bundle_uuid] for dep in info['dependencies']: parent_uuid = dep['parent_uuid'] if parent_uuid not in infos: next_bundle_uuids.append(parent_uuid) # Mark this bundle as visited visited.add(bundle_uuid) # Prepend to the running list of all bundles result = next_bundle_uuids + result # Swap in the next batch of bundles for next iteration bundle_uuids = next_bundle_uuids return result all_bundle_uuids = get_self_and_ancestors(list(infos.keys())) # Now go recursively create the bundles. old_to_new = {} # old_uuid -> new_uuid downstream = ( set() ) # old_uuid -> whether we're downstream of an input (and actually needs to be mapped onto a new uuid) created_uuids = set() # set of uuids which were newly created plan = [] # sequence of (old, new) bundle infos to make for old, new in zip(old_inputs, new_inputs): old_to_new[old] = new downstream.add(old) # Return corresponding new_bundle_uuid def recurse(old_bundle_uuid): if old_bundle_uuid in old_to_new: return old_to_new[old_bundle_uuid] # Don't have any more information (because we probably hit the maximum depth) if old_bundle_uuid not in infos: return old_bundle_uuid # Get information about the old bundle. old_info = infos[old_bundle_uuid] new_dependencies = [ { 'parent_uuid': recurse(dep['parent_uuid']), 'parent_path': dep['parent_path'], 'child_uuid': dep[ 'child_uuid' ], # This is just a placeholder to do the equality test 'child_path': dep['child_path'], } for dep in old_info['dependencies'] ] # If there are no inputs or if we're downstream of any inputs, we need to make a new bundle. lone_output = len(old_inputs) == 0 and old_bundle_uuid == old_output downstream_of_inputs = any( dep['parent_uuid'] in downstream for dep in old_info['dependencies'] ) if lone_output or downstream_of_inputs: # Now create a new bundle that mimics the old bundle. new_info = copy.deepcopy(old_info) # Make sure that new uuids are generated new_info.pop('uuid', None) new_info.pop('id', None) # Only change the name if the output name is supplied. new_metadata = new_info['metadata'] if new_output_name: if old_bundle_uuid == old_output: new_metadata['name'] = new_output_name else: # Just make up a name heuristically new_metadata['name'] = new_output_name + '-' + old_info['metadata']['name'] # By default, the mimic bundle uses whatever image the old bundle uses # Preferably it uses the SHA256 image digest, but it may simply copy request_docker_image # if it is not present if new_info['bundle_type'] == 'run' and new_metadata.get('docker_image', ''): # Put docker_image in requested_docker_image if it is present and this is a run bundle new_metadata['request_docker_image'] = new_metadata['docker_image'] cls = get_bundle_subclass(new_info['bundle_type']) for spec in cls.METADATA_SPECS: # Remove automatically generated keys if spec.generated and spec.key in new_metadata: del new_metadata[spec.key] # Override original metadata keys if spec.key in metadata_override: new_metadata[spec.key] = metadata_override[spec.key] # Set up info dict new_info['metadata'] = new_metadata new_info['dependencies'] = new_dependencies if dry_run: new_info['uuid'] = None else: if new_info['bundle_type'] not in ('make', 'run'): raise UsageError( 'Can\'t mimic %s since it is not make or run' % old_bundle_uuid ) # Create the new bundle, requesting to shadow the old # bundle in its worksheet if shadow is specified, otherwise # leave the bundle detached, to be added later below. params = {} params['worksheet'] = worksheet_uuid if shadow: params['shadow'] = old_info['uuid'] else: params['detached'] = True new_info = client.create('bundles', new_info, params=params) new_bundle_uuid = new_info['uuid'] plan.append((old_info, new_info)) downstream.add(old_bundle_uuid) created_uuids.add(new_bundle_uuid) else: new_bundle_uuid = old_bundle_uuid old_to_new[old_bundle_uuid] = new_bundle_uuid # Cache it return new_bundle_uuid if old_output: recurse(old_output) else: # Don't have a particular output we're targetting, so just create # new versions of all the uuids. for uuid in all_bundle_uuids: recurse(uuid) # Add to worksheet if not dry_run and not shadow: # A prelude of a bundle on a worksheet is the set of items (markup, directives, etc.) # that occur immediately before it, until the last preceding newline. # Let W be the first worksheet containing the old_inputs[0]. # Add all items on that worksheet that appear in old_to_new along with their preludes. # For items not on this worksheet, add them at the end (instead of making them floating). if old_output: anchor_uuid = old_output elif len(old_inputs) > 0: anchor_uuid = old_inputs[0] # Find worksheets that contain the anchor bundle host_worksheets = client.fetch('worksheets', params={'keywords': 'bundle=' + anchor_uuid}) host_worksheet_uuids = [hw['id'] for hw in host_worksheets] new_bundle_uuids_added = set() if len(host_worksheet_uuids) > 0: # Choose a single worksheet. if worksheet_uuid in host_worksheet_uuids: # If current worksheet is one of them, favor that one. host_worksheet_uuid = worksheet_uuid else: # Choose an arbitrary one (in the future, have a better way of canonicalizing). host_worksheet_uuid = host_worksheet_uuids[0] # Fetch the worksheet worksheet_info = client.fetch( 'worksheets', host_worksheet_uuid, params={'include': ['items', 'items.bundle']} ) prelude_items = [] # The prelude that we're building up for item in worksheet_info['items']: just_added = False if item['type'] == worksheet_util.TYPE_BUNDLE: old_bundle_uuid = item['bundle']['id'] if old_bundle_uuid in old_to_new: # Flush the prelude gathered so far. new_bundle_uuid = old_to_new[old_bundle_uuid] if new_bundle_uuid in created_uuids: # Only add novel bundles # Add prelude if not skip_prelude: for item2 in prelude_items: # Create a copy of the item on the destination worksheet item2 = item2.copy() item2['worksheet'] = JsonApiRelationship( 'worksheets', worksheet_uuid ) client.create( 'worksheet-items', data=item2, params={'uuid': worksheet_uuid}, ) # Add the bundle item client.create( 'worksheet-items', data={ 'type': worksheet_util.TYPE_BUNDLE, 'worksheet': JsonApiRelationship('worksheets', worksheet_uuid), 'bundle': JsonApiRelationship('bundles', new_bundle_uuid), }, params={'uuid': worksheet_uuid}, ) new_bundle_uuids_added.add(new_bundle_uuid) just_added = True if (item['type'] == worksheet_util.TYPE_MARKUP and item['value'] != '') or item[ 'type' ] == worksheet_util.TYPE_DIRECTIVE: prelude_items.append(item) # Include in prelude else: prelude_items = [] # Reset # Add the bundles that haven't been added yet for info, new_info in plan: new_bundle_uuid = new_info['uuid'] if new_bundle_uuid not in new_bundle_uuids_added: print('adding: ' + new_bundle_uuid) client.create( 'worksheet-items', data={ 'type': worksheet_util.TYPE_BUNDLE, 'worksheet': JsonApiRelationship('worksheets', worksheet_uuid), 'bundle': JsonApiRelationship('bundles', new_bundle_uuid), }, params={'uuid': worksheet_uuid}, ) return plan