def test_is_url(): """test the ability to determine whether a string is a URL""" assert utils.is_url("http://mydomain.com/foo/bar/bat?asdf=1234&qewr=ooo") assert utils.is_url("http://xkcd.com/1193/") assert not utils.is_url("syn123445") assert not utils.is_url("wasssuuuup???") assert utils.is_url('file://foo.com/path/to/file.xyz') assert utils.is_url('file:///path/to/file.xyz') assert utils.is_url('file:/path/to/file.xyz') assert utils.is_url('file:///c:/WINDOWS/clock.avi') assert utils.is_url('file:c:/WINDOWS/clock.avi') assert not utils.is_url('c:/WINDOWS/ugh/ugh.ugh')
def test_windows_file_urls(): url = 'file:///c:/WINDOWS/clock.avi' assert utils.is_url(url) assert utils.file_url_to_path( url, verify_exists=False) == 'c:/WINDOWS/clock.avi', utils.file_url_to_path( url)
def _check_path_and_normalize(f): sys.stdout.write('.') if is_url(f): return f path_normalized = os.path.abspath(os.path.expandvars(os.path.expanduser(f))) if not os.path.isfile(path_normalized): print('\nThe specified path "%s" is either not a file path or does not exist.', f) raise IOError('The path %s is not a file or does not exist' %f) return path_normalized
def local_file_has_changed(entityBundle, checkIndirect, path=None): """ Checks the local cache to see if the given file has been modified. :param entityBundle : A dictionary with 'fileHandles' and 'entity'. Typically created via:: syn._getEntityBundle() :param checkIndirect: Whether or not the cache should be checked for unmodified files. Should be True when getting, False when storing. :param path: Path to the local file. May be in any format. If not given, the information from the 'entityBundle' is used to derive a cached location for the file. :returns: True if the file has been modified. """ # Find the directory of the '.cacheMap' for the file cacheDir, filepath, _ = determine_local_file_location(entityBundle) if path is None: path = filepath # If there is no file path, there is nothing to download if path is None: return False # For external URLs, if the path has not changed # then the file handle does not have to change if utils.is_url(path): for handle in entityBundle['fileHandles']: if handle['id'] == entityBundle['entity']['dataFileHandleId'] \ and handle['concreteType'] == 'org.sagebionetworks.repo.model.file.ExternalFileHandle' \ and handle['externalURL'] == path: return False return True # Compare the modification times path = utils.normalize_path(path) fileMTime = get_modification_time(path) unmodifiedFileExists = False for file, cacheTime, cachedFileMTime in iterator_over_cache_map(cacheDir): # When there is a direct match, return if it is modified if path == file and os.path.exists(path): return not fileMTime == cacheTime # If there is no direct match, but a copy exists, return False after checking all entries if checkIndirect and cachedFileMTime == cacheTime: unmodifiedFileExists = True # The file is cached but not in the right place copy it add it to the cache if checkIndirect and unmodifiedFileExists and not path.startswith(CACHE_DIR): add_local_file_to_cache(path=path, **entityBundle['entity']) return not unmodifiedFileExists
def local_file_has_changed(entityBundle, path=None): """ Checks the local cache to see if the given file has been modified. :param entityBundle: A dictionary with 'fileHandles' and 'entity'. Typically created via:: syn._getEntityBundle() :param path: Path to the local file. May be in any format. If not given, the information from the 'entityBundle' is used to derive a cached location for the file. :returns: True if the file has been modified. """ # Find the directory of the '.cacheMap' for the file cacheDir, filepath, _ = determine_local_file_location(entityBundle) if path is None: path = filepath # If there is no file path, there is nothing to download if path is None: return False # External URLs will be ignored if utils.is_url(path): return True # Compare the modification times path = normalize_path(path) fileMTime = get_modification_time(path) unmodifiedFileExists = False for file, cacheTime, _ in iterator_over_cache_map(cacheDir): # When there is a direct match, return if it is modified if path == file and os.path.exists(path): return not fileMTime == cacheTime # If there is no direct match, but a pristine copy exists, return False (after checking all entries) # The filecmp is necessary for Windows machines since their clocks do not keep millisecond information # i.e. Two files created quickly may have the same timestamp if fileMTime == cacheTime and os.path.exists(file) and filecmp.cmp( path, file): unmodifiedFileExists = True # The file is not cached or has been changed return not unmodifiedFileExists
def local_file_has_changed(entityBundle, checkIndirect, path=None): """ Checks the local cache to see if the given file has been modified. :param entityBundle : A dictionary with 'fileHandles' and 'entity'. Typically created via:: syn._getEntityBundle() :param checkIndirect: Whether or not the cache should be checked for unmodified files. Should be True when getting, False when storing. :param path: Path to the local file. May be in any format. If not given, the information from the 'entityBundle' is used to derive a cached location for the file. :returns: True if the file has been modified. """ # Find the directory of the '.cacheMap' for the file cacheDir, filepath, _ = determine_local_file_location(entityBundle) if path is None: path = filepath # If there is no file path, there is nothing to download if path is None: return False # External URLs will be ignored if utils.is_url(path): return True # Compare the modification times path = normalize_path(path) fileMTime = get_modification_time(path) unmodifiedFileExists = False for file, cacheTime, cachedFileMTime in iterator_over_cache_map(cacheDir): # When there is a direct match, return if it is modified if path == file and os.path.exists(path): return not fileMTime == cacheTime # If there is no direct match, but a pristine copy exists, return False (after checking all entries) if checkIndirect and cachedFileMTime == cacheTime: unmodifiedFileExists = True # The file is not cached or has been changed return not unmodifiedFileExists
def local_file_has_changed(entityBundle, path=None): """ Checks the local cache to see if the given file has been modified. :param entityBundle: A dictionary with 'fileHandles' and 'entity'. Typically created via:: syn._getEntityBundle() :param path: Path to the local file. May be in any format. If not given, the information from the 'entityBundle' is used to derive a cached location for the file. :returns: True if the file has been modified. """ # Find the directory of the '.cacheMap' for the file cacheDir, filepath, _ = determine_local_file_location(entityBundle) if path is None: path = filepath # If there is no file path, there is nothing to download if path is None: return False # External URLs will be ignored if utils.is_url(path): return True # Compare the modification times path = normalize_path(path) fileMTime = get_modification_time(path) unmodifiedFileExists = False for file, cacheTime, _ in iterator_over_cache_map(cacheDir): # When there is a direct match, return if it is modified if path == file and os.path.exists(path): return not fileMTime == cacheTime # If there is no direct match, but a pristine copy exists, return False (after checking all entries) # The filecmp is necessary for Windows machines since their clocks do not keep millisecond information # i.e. Two files created quickly may have the same timestamp if fileMTime == cacheTime and os.path.exists(file) and filecmp.cmp(path, file): unmodifiedFileExists = True # The file is not cached or has been changed return not unmodifiedFileExists
def add_local_file_to_cache(**entity): """ Makes a '.cacheMap' entry in the cache. :param entity: A Synapse Entity object or dictionary with a 'path'. FileEntities require a 'dataFileHandleID'. Locationables require a 'id' and 'versionNumber'. Example:: foo = File('/path/to/file/xyz.txt') cache.add_local_file_to_cache(bar="Something to include in dict", **foo) Note: Since neither FileEntities nor Locationables have a 'path' in their properties, calls to this method should look like:: cache.add_local_file_to_cache(path=entity['path'], **entity) """ # External URLs will be ignored if utils.is_url(entity['path']) or entity['path'] is None: return # Get the '.cacheMap' cacheDir = determine_cache_directory(entity) entity['path'] = utils.normalize_path(entity['path']) # If the file to-be-added does not exist, search the cache for a pristine copy if not os.path.exists(entity['path']): for file, cacheTime, fileMTime in iterator_over_cache_map(cacheDir): if fileMTime == cacheTime: shutil.copyfile(file, entity['path']) break # Update the cache if os.path.exists(entity['path']): cache = obtain_lock_and_read_cache(cacheDir) cache[entity['path']] = time.strftime( utils.ISO_FORMAT, time.gmtime(os.path.getmtime(entity['path']))) write_cache_then_release_lock(cacheDir, cache)
def add_local_file_to_cache(**entity): """ Makes a '.cacheMap' entry in the cache. :param entity: A Synapse Entity object or dictionary with a 'path'. FileEntities require a 'dataFileHandleID'. Locationables require a 'id' and 'versionNumber'. Example:: foo = File('/path/to/file/xyz.txt') cache.add_local_file_to_cache(bar="Something to include in dict", **foo) Note: Since neither FileEntities nor Locationables have a 'path' in their properties, calls to this method should look like:: cache.add_local_file_to_cache(path=entity['path'], **entity) """ # External URLs will be ignored if utils.is_url(entity['path']): return True # Get the '.cacheMap' cacheDir = determine_cache_directory(entity) entity['path'] = normalize_path(entity['path']) # If the file to-be-added does not exist, search the cache for a pristine copy if not os.path.exists(entity['path']): for file, cacheTime, fileMTime in iterator_over_cache_map(cacheDir): if fileMTime == cacheTime: shutil.copyfile(file, entity['path']) break # Update the cache if os.path.exists(entity['path']): cache = obtain_lock_and_read_cache(cacheDir) cache[entity['path']] = time.strftime(utils.ISO_FORMAT, time.gmtime(os.path.getmtime(entity['path']))) write_cache_then_release_lock(cacheDir, cache)
def used(self, target=None, targetVersion=None, wasExecuted=None, url=None, name=None): """ Add a resource used by the activity. This method tries to be as permissive as possible. It accepts a string which might be a synapse ID or a URL, a synapse entity, a UsedEntity or UsedURL dictionary or a list containing any combination of these. In addition, named parameters can be used to specify the fields of either a UsedEntity or a UsedURL. If target and optionally targetVersion are specified, create a UsedEntity. If url and optionally name are specified, create a UsedURL. It is an error to specify both target/targetVersion parameters and url/name parameters in the same call. To add multiple UsedEntities and UsedURLs, make a separate call for each or pass in a list. In case of conflicting settings for wasExecuted both inside an object and with a parameter, the parameter wins. For example, this UsedURL will have wasExecuted set to False:: activity.used({'url':'http://google.com', 'name':'Goog', 'wasExecuted':True}, wasExecuted=False) Entity examples:: activity.used('syn12345') activity.used(entity) activity.used(target=entity, targetVersion=2) activity.used(codeEntity, wasExecuted=True) activity.used({'reference':{'target':'syn12345', 'targetVersion':1}, 'wasExecuted':False}) URL examples:: activity.used('http://mydomain.com/my/awesome/data.RData') activity.used(url='http://mydomain.com/my/awesome/data.RData', name='Awesome Data') activity.used(url='https://github.com/joe_hacker/code_repo', name='Gnarly hacks', wasExecuted=True) activity.used({'url':'https://github.com/joe_hacker/code_repo', 'name':'Gnarly hacks'}, wasExecuted=True) List example:: activity.used(['syn12345', 'syn23456', entity, \ {'reference':{'target':'syn100009', 'targetVersion':2}, 'wasExecuted':True}, \ 'http://mydomain.com/my/awesome/data.RData']) """ # -- A list of targets if isinstance(target, list): badargs = _get_any_bad_args(['targetVersion', 'url', 'name'], locals()) _raise_incorrect_used_usage(badargs, 'list of used resources') for item in target: self.used(item, wasExecuted=wasExecuted) return # -- UsedEntity elif is_used_entity(target): badargs = _get_any_bad_args(['targetVersion', 'url', 'name'], locals()) _raise_incorrect_used_usage( badargs, 'dictionary representing a used resource') resource = target if 'concreteType' not in resource: resource[ 'concreteType'] = 'org.sagebionetworks.repo.model.provenance.UsedEntity' # -- Used URL elif is_used_url(target): badargs = _get_any_bad_args(['targetVersion', 'url', 'name'], locals()) _raise_incorrect_used_usage(badargs, 'URL') resource = target if 'concreteType' not in resource: resource[ 'concreteType'] = 'org.sagebionetworks.repo.model.provenance.UsedURL' # -- Synapse Entity elif is_synapse_entity(target): badargs = _get_any_bad_args(['url', 'name'], locals()) _raise_incorrect_used_usage(badargs, 'Synapse entity') reference = {'targetId': target['id']} if 'versionNumber' in target: reference['targetVersionNumber'] = target['versionNumber'] if targetVersion: reference['targetVersionNumber'] = int(targetVersion) resource = { 'reference': reference, 'concreteType': 'org.sagebionetworks.repo.model.provenance.UsedEntity' } # -- URL parameter elif url: badargs = _get_any_bad_args(['target', 'targetVersion'], locals()) _raise_incorrect_used_usage(badargs, 'URL') resource = { 'url': url, 'name': name if name else target, 'concreteType': 'org.sagebionetworks.repo.model.provenance.UsedURL' } # -- URL as a string elif is_url(target): badargs = _get_any_bad_args(['targetVersion'], locals()) _raise_incorrect_used_usage(badargs, 'URL') resource = { 'url': target, 'name': name if name else target, 'concreteType': 'org.sagebionetworks.repo.model.provenance.UsedURL' } # -- Synapse Entity ID (assuming the string is an ID) elif isinstance(target, six.string_types): badargs = _get_any_bad_args(['url', 'name'], locals()) _raise_incorrect_used_usage(badargs, 'Synapse entity') vals = target.split('.') # Handle synapseIds of from syn234.4 if not is_synapse_id(vals[0]): raise ValueError('%s is not a valid Synapse id' % target) if len(vals) == 2: if targetVersion and int(targetVersion) != int(vals[1]): raise ValueError( 'Two conflicting versions for %s were specified' % target) targetVersion = int(vals[1]) reference = {'targetId': vals[0]} if targetVersion: reference['targetVersionNumber'] = int(targetVersion) resource = { 'reference': reference, 'concreteType': 'org.sagebionetworks.repo.model.provenance.UsedEntity' } else: raise SynapseError( 'Unexpected parameters in call to Activity.used().') # Set wasExecuted if wasExecuted is None: # Default to False if 'wasExecuted' not in resource: resource['wasExecuted'] = False else: # wasExecuted parameter overrides setting in an object resource['wasExecuted'] = wasExecuted # Add the used resource to the activity self['used'].append(resource)
def test_windows_file_urls(): url = 'file:///c:/WINDOWS/clock.avi' assert utils.is_url(url) assert utils.file_url_to_path(url, verify_exists=False).get('path',None) == 'c:/WINDOWS/clock.avi', utils.file_url_to_path(url)
def test_windows_file_urls(): url = 'file:///c:/WINDOWS/clock.avi' assert_true(utils.is_url(url)) assert_equals(utils.file_url_to_path(url, verify_exists=False), 'c:/WINDOWS/clock.avi', utils.file_url_to_path(url))
def syncToSynapse(syn, manifestFile, dryRun=False, sendMessages=True, retries=MAX_RETRIES): """Synchronizes files specified in the manifest file to Synapse :param syn: A synapse object as obtained with syn = synapseclient.login() :param manifestFile: A tsv file with file locations and metadata to be pushed to Synapse. See below for details :param dryRun: Performs validation without uploading if set to True (default is False) Given a file describing all of the uploads uploads the content to Synapse and optionally notifies you via Synapse messagging (email) at specific intervals, on errors and on completion. **Manifest file format** The format of the manifest file is a tab delimited file with one row per file to upload and columns describing the file. The minimum required columns are **path** and **parent** where path is the local file path and parent is the Synapse Id of the project or folder where the file is uploaded to. In addition to these columns you can specify any of the parameters to the File constructor (**name**, **synapseStore**, **contentType**) as well as parameters to the syn.store command (**used**, **executed**, **activityName**, **activityDescription**, **forceVersion**). Used and executed can be semi-colon (";") separated lists of Synapse ids, urls and/or local filepaths of files already stored in Synapse (or being stored in Synapse by the manifest). Any additional columns will be added as annotations. **Required fields:** ====== ====================== ============================ Field Meaning Example ====== ====================== ============================ path local file path or URL /path/to/local/file.txt parent synapse id syn1235 ====== ====================== ============================ **Common fields:** =============== =========================== ============ Field Meaning Example =============== =========================== ============ name name of file in Synapse Example_file forceVersion whether to update version False =============== =========================== ============ **Provenance fields:** ==================== ===================================== ========================================== Field Meaning Example ==================== ===================================== ========================================== used List of items used to generate file syn1235; /path/to_local/file.txt executed List of items exectued https://github.org/; /path/to_local/code.py activityName Name of activity in provenance "Ran normalization" activityDescription Text description on what was done "Ran algorithm xyx with parameters..." ==================== ===================================== ========================================== Annotations: **Annotations:** Any columns that are not in the reserved names described above will be interpreted as annotations of the file **Other optional fields:** =============== ========================================== ============ Field Meaning Example =============== ========================================== ============ synapseStore Boolean describing whether to upload files True contentType content type of file to overload defaults text/html =============== ========================================== ============ **Example manifest file** =============== ======== ======= ======= =========================== ============================ path parent annot1 annot2 used executed =============== ======== ======= ======= =========================== ============================ /path/file1.txt syn1243 "bar" 3.1415 "syn124; /path/file2.txt" "https://github.org/foo/bar" /path/file2.txt syn12433 "baz" 2.71 "" "https://github.org/foo/baz" =============== ======== ======= ======= =========================== ============================ """ df = readManifestFile(syn, manifestFile) sizes = [ os.stat(os.path.expandvars(os.path.expanduser(f))).st_size for f in df.path if not is_url(f) ] # Write output on what is getting pushed and estimated times - send out message. sys.stdout.write('=' * 50 + '\n') sys.stdout.write( 'We are about to upload %i files with a total size of %s.\n ' % (len(df), utils.humanizeBytes(sum(sizes)))) sys.stdout.write('=' * 50 + '\n') if dryRun: return sys.stdout.write('Starting upload...\n') if sendMessages: notify_decorator = notifyMe(syn, 'Upload of %s' % manifestFile, retries=retries) upload = notify_decorator(_manifest_upload) upload(syn, df) else: _manifest_upload(syn, df)
def used(self, target=None, targetVersion=None, wasExecuted=None, url=None, name=None): """ Add a resource used by the activity. This method tries to be as permissive as possible. It accepts a string which might be a synapse ID or a URL, a synapse entity, a UsedEntity or UsedURL dictionary or a list containing any combination of these. In addition, named parameters can be used to specify the fields of either a UsedEntity or a UsedURL. If target and optionally targetVersion are specified, create a UsedEntity. If url and optionally name are specified, create a UsedURL. It is an error to specify both target/targetVersion parameters and url/name parameters in the same call. To add multiple UsedEntities and UsedURLs, make a separate call for each or pass in a list. In case of conflicting settings for wasExecuted both inside an object and with a parameter, the parameter wins. For example, this UsedURL will have wasExecuted set to False:: activity.used({'url':'http://google.com', 'name':'Goog', 'wasExecuted':True}, wasExecuted=False) Entity examples:: activity.used('syn12345') activity.used(entity) activity.used(target=entity, targetVersion=2) activity.used(codeEntity, wasExecuted=True) activity.used({'reference':{'target':'syn12345', 'targetVersion':1}, 'wasExecuted':False}) URL examples:: activity.used('http://mydomain.com/my/awesome/data.RData') activity.used(url='http://mydomain.com/my/awesome/data.RData', name='Awesome Data') activity.used(url='https://github.com/joe_hacker/code_repo', name='Gnarly hacks', wasExecuted=True) activity.used({'url':'https://github.com/joe_hacker/code_repo', 'name':'Gnarly hacks'}, wasExecuted=True) List example:: used( [ 'syn12345', 'syn23456', entity, {'reference':{'target':'syn100009', 'targetVersion':2}, 'wasExecuted':True}, 'http://mydomain.com/my/awesome/data.RData' ] ) """ # Check for allowed combinations of parameters and generate specific error message # based on the context. For example, if we specify a URL, it's illegal to specify # a version. def check_for_invalid_parameters(context=None, params={}): err_msg = 'Error in call to Activity.used()' if context == 'list': illegal_params = ('targetVersion', 'url', 'name',) context_msg = 'list of used resources' elif context == 'dict': illegal_params = ('targetVersion', 'url', 'name',) context_msg = 'dictionary representing a used resource' elif context == 'url_param': illegal_params = ('target', 'targetVersion',) context_msg = 'URL' elif context == 'url_string': illegal_params = ('targetVersion',) context_msg = 'URL' elif context == 'entity': illegal_params = ('url', 'name',) context_msg = 'Synapse entity' else: illegal_params = () context_msg = '?' for param in illegal_params: if param in params and params[param] is not None: raise Exception('%s: It is an error to specify the \'%s\' parameter in combination with a %s.' % (err_msg, str(param), context_msg)) # List if isinstance(target, list): check_for_invalid_parameters(context='list', params=locals()) for item in target: self.used(item, wasExecuted=wasExecuted) return # Used Entity elif is_used_entity(target): check_for_invalid_parameters(context='dict', params=locals()) resource = target if 'concreteType' not in resource: resource['concreteType'] = 'org.sagebionetworks.repo.model.provenance.UsedEntity' # Used URL elif is_used_url(target): check_for_invalid_parameters(context='dict', params=locals()) resource = target if 'concreteType' not in resource: resource['concreteType'] = 'org.sagebionetworks.repo.model.provenance.UsedURL' # Synapse Entity elif is_synapse_entity(target): check_for_invalid_parameters(context='entity', params=locals()) reference = {'targetId':target['id']} if 'versionNumber' in target: reference['targetVersionNumber'] = target['versionNumber'] ## if targetVersion is specified as a parameter, it overrides the version in the object if targetVersion: reference['targetVersionNumber'] = int(targetVersion) resource = {'reference':reference, 'concreteType':'org.sagebionetworks.repo.model.provenance.UsedEntity'} # URL parameter elif url: check_for_invalid_parameters(context='url_param', params=locals()) resource = {'url':url, 'name':name if name else target, 'concreteType':'org.sagebionetworks.repo.model.provenance.UsedURL'} # URL as a string elif is_url(target): check_for_invalid_parameters(context='url_string', params=locals()) resource = {'url':target, 'name':name if name else target, 'concreteType':'org.sagebionetworks.repo.model.provenance.UsedURL'} # If it's a string and isn't a URL, assume it's a Synapse Entity ID elif isinstance(target, basestring): check_for_invalid_parameters(context='entity', params=locals()) reference = {'targetId':target} if targetVersion: reference['targetVersionNumber'] = int(targetVersion) resource = {'reference':reference, 'concreteType':'org.sagebionetworks.repo.model.provenance.UsedEntity'} else: raise Exception('Unexpected parameters in call to Activity.used().') # Set wasExecuted if wasExecuted is None: # Default to False if 'wasExecuted' not in resource: resource['wasExecuted'] = False else: # wasExecuted parameter overrides setting in an object resource['wasExecuted'] = wasExecuted # Add the used resource to the activity self['used'].append(resource)
def used(self, target=None, targetVersion=None, wasExecuted=None, url=None, name=None): """ Add a resource used by the activity. This method tries to be as permissive as possible. It accepts a string which might be a synapse ID or a URL, a synapse entity, a UsedEntity or UsedURL dictionary or a list containing any combination of these. In addition, named parameters can be used to specify the fields of either a UsedEntity or a UsedURL. If target and optionally targetVersion are specified, create a UsedEntity. If url and optionally name are specified, create a UsedURL. It is an error to specify both target/targetVersion parameters and url/name parameters in the same call. To add multiple UsedEntities and UsedURLs, make a separate call for each or pass in a list. In case of conflicting settings for wasExecuted both inside an object and with a parameter, the parameter wins. For example, this UsedURL will have wasExecuted set to False:: activity.used({'url':'http://google.com', 'name':'Goog', 'wasExecuted':True}, wasExecuted=False) Entity examples:: activity.used('syn12345') activity.used(entity) activity.used(target=entity, targetVersion=2) activity.used(codeEntity, wasExecuted=True) activity.used({'reference':{'target':'syn12345', 'targetVersion':1}, 'wasExecuted':False}) URL examples:: activity.used('http://mydomain.com/my/awesome/data.RData') activity.used(url='http://mydomain.com/my/awesome/data.RData', name='Awesome Data') activity.used(url='https://github.com/joe_hacker/code_repo', name='Gnarly hacks', wasExecuted=True) activity.used({'url':'https://github.com/joe_hacker/code_repo', 'name':'Gnarly hacks'}, wasExecuted=True) List example:: used( [ 'syn12345', 'syn23456', entity, {'reference':{'target':'syn100009', 'targetVersion':2}, 'wasExecuted':True}, 'http://mydomain.com/my/awesome/data.RData' ] ) """ # Check for allowed combinations of parameters and generate specific error message # based on the context. For example, if we specify a URL, it's illegal to specify # a version. def check_for_invalid_parameters(context=None, params={}): err_msg = 'Error in call to Activity.used()' if context == 'list': illegal_params = ( 'targetVersion', 'url', 'name', ) context_msg = 'list of used resources' elif context == 'dict': illegal_params = ( 'targetVersion', 'url', 'name', ) context_msg = 'dictionary representing a used resource' elif context == 'url_param': illegal_params = ( 'target', 'targetVersion', ) context_msg = 'URL' elif context == 'url_string': illegal_params = ('targetVersion', ) context_msg = 'URL' elif context == 'entity': illegal_params = ( 'url', 'name', ) context_msg = 'Synapse entity' else: illegal_params = () context_msg = '?' for param in illegal_params: if param in params and params[param] is not None: raise Exception( '%s: It is an error to specify the \'%s\' parameter in combination with a %s.' % (err_msg, str(param), context_msg)) # List if isinstance(target, list): check_for_invalid_parameters(context='list', params=locals()) for item in target: self.used(item, wasExecuted=wasExecuted) return # Used Entity elif is_used_entity(target): check_for_invalid_parameters(context='dict', params=locals()) resource = target if 'concreteType' not in resource: resource[ 'concreteType'] = 'org.sagebionetworks.repo.model.provenance.UsedEntity' # Used URL elif is_used_url(target): check_for_invalid_parameters(context='dict', params=locals()) resource = target if 'concreteType' not in resource: resource[ 'concreteType'] = 'org.sagebionetworks.repo.model.provenance.UsedURL' # Synapse Entity elif is_synapse_entity(target): check_for_invalid_parameters(context='entity', params=locals()) reference = {'targetId': target['id']} if 'versionNumber' in target: reference['targetVersionNumber'] = target['versionNumber'] ## if targetVersion is specified as a parameter, it overrides the version in the object if targetVersion: reference['targetVersionNumber'] = int(targetVersion) resource = { 'reference': reference, 'concreteType': 'org.sagebionetworks.repo.model.provenance.UsedEntity' } # URL parameter elif url: check_for_invalid_parameters(context='url_param', params=locals()) resource = { 'url': url, 'name': name if name else target, 'concreteType': 'org.sagebionetworks.repo.model.provenance.UsedURL' } # URL as a string elif is_url(target): check_for_invalid_parameters(context='url_string', params=locals()) resource = { 'url': target, 'name': name if name else target, 'concreteType': 'org.sagebionetworks.repo.model.provenance.UsedURL' } # If it's a string and isn't a URL, assume it's a Synapse Entity ID elif isinstance(target, basestring): check_for_invalid_parameters(context='entity', params=locals()) reference = {'targetId': target} if targetVersion: reference['targetVersionNumber'] = int(targetVersion) resource = { 'reference': reference, 'concreteType': 'org.sagebionetworks.repo.model.provenance.UsedEntity' } else: raise Exception( 'Unexpected parameters in call to Activity.used().') # Set wasExecuted if wasExecuted is None: # Default to False if 'wasExecuted' not in resource: resource['wasExecuted'] = False else: # wasExecuted parameter overrides setting in an object resource['wasExecuted'] = wasExecuted # Add the used resource to the activity self['used'].append(resource)
def used(self, target=None, targetVersion=None, wasExecuted=None, url=None, name=None): """ Add a resource used by the activity. This method tries to be as permissive as possible. It accepts a string which might be a synapse ID or a URL, a synapse entity, a UsedEntity or UsedURL dictionary or a list containing any combination of these. In addition, named parameters can be used to specify the fields of either a UsedEntity or a UsedURL. If target and optionally targetVersion are specified, create a UsedEntity. If url and optionally name are specified, create a UsedURL. It is an error to specify both target/targetVersion parameters and url/name parameters in the same call. To add multiple UsedEntities and UsedURLs, make a separate call for each or pass in a list. In case of conflicting settings for wasExecuted both inside an object and with a parameter, the parameter wins. For example, this UsedURL will have wasExecuted set to False:: activity.used({'url':'http://google.com', 'name':'Goog', 'wasExecuted':True}, wasExecuted=False) Entity examples:: activity.used('syn12345') activity.used(entity) activity.used(target=entity, targetVersion=2) activity.used(codeEntity, wasExecuted=True) activity.used({'reference':{'target':'syn12345', 'targetVersion':1}, 'wasExecuted':False}) URL examples:: activity.used('http://mydomain.com/my/awesome/data.RData') activity.used(url='http://mydomain.com/my/awesome/data.RData', name='Awesome Data') activity.used(url='https://github.com/joe_hacker/code_repo', name='Gnarly hacks', wasExecuted=True) activity.used({'url':'https://github.com/joe_hacker/code_repo', 'name':'Gnarly hacks'}, wasExecuted=True) List example:: used( [ 'syn12345', 'syn23456', entity, {'reference':{'target':'syn100009', 'targetVersion':2}, 'wasExecuted':True}, 'http://mydomain.com/my/awesome/data.RData' ] ) """ # -- A list of targets if isinstance(target, list): badargs = _get_any_bad_args(['targetVersion', 'url', 'name'], locals()) _raise_incorrect_used_usage(badargs, 'list of used resources') for item in target: self.used(item, wasExecuted=wasExecuted) return # -- UsedEntity elif is_used_entity(target): badargs = _get_any_bad_args(['targetVersion', 'url', 'name'], locals()) _raise_incorrect_used_usage(badargs, 'dictionary representing a used resource') resource = target if 'concreteType' not in resource: resource['concreteType'] = 'org.sagebionetworks.repo.model.provenance.UsedEntity' # -- Used URL elif is_used_url(target): badargs = _get_any_bad_args(['targetVersion', 'url', 'name'], locals()) _raise_incorrect_used_usage(badargs, 'URL') resource = target if 'concreteType' not in resource: resource['concreteType'] = 'org.sagebionetworks.repo.model.provenance.UsedURL' # -- Synapse Entity elif is_synapse_entity(target): badargs = _get_any_bad_args(['url', 'name'], locals()) _raise_incorrect_used_usage(badargs, 'Synapse entity') reference = {'targetId':target['id']} if 'versionNumber' in target: reference['targetVersionNumber'] = target['versionNumber'] if targetVersion: reference['targetVersionNumber'] = int(targetVersion) resource = {'reference':reference, 'concreteType':'org.sagebionetworks.repo.model.provenance.UsedEntity'} # -- URL parameter elif url: badargs = _get_any_bad_args(['target', 'targetVersion'], locals()) _raise_incorrect_used_usage(badargs, 'URL') resource = {'url':url, 'name':name if name else target, 'concreteType':'org.sagebionetworks.repo.model.provenance.UsedURL'} # -- URL as a string elif is_url(target): badargs = _get_any_bad_args(['targetVersion'], locals()) _raise_incorrect_used_usage(badargs, 'URL') resource = {'url':target, 'name':name if name else target, 'concreteType':'org.sagebionetworks.repo.model.provenance.UsedURL'} # -- Synapse Entity ID (assuming the string is an ID) elif isinstance(target, basestring): badargs = _get_any_bad_args(['url', 'name'], locals()) _raise_incorrect_used_usage(badargs, 'Synapse entity') reference = {'targetId':target} if targetVersion: reference['targetVersionNumber'] = int(targetVersion) resource = {'reference':reference, 'concreteType':'org.sagebionetworks.repo.model.provenance.UsedEntity'} else: raise SynapseError('Unexpected parameters in call to Activity.used().') # Set wasExecuted if wasExecuted is None: # Default to False if 'wasExecuted' not in resource: resource['wasExecuted'] = False else: # wasExecuted parameter overrides setting in an object resource['wasExecuted'] = wasExecuted # Add the used resource to the activity self['used'].append(resource)
def test_windows_file_urls(): url = "file:///c:/WINDOWS/clock.avi" assert utils.is_url(url) assert utils.file_url_to_path(url, verify_exists=False) == "c:/WINDOWS/clock.avi", utils.file_url_to_path(url)