def test_is_url():
"""test the ability to determine whether a string is a URL"""
assert utils.is_url("http://mydomain.com/foo/bar/bat?asdf=1234&qewr=ooo")
assert utils.is_url("http://xkcd.com/1193/")
assert not utils.is_url("syn123445")
assert not utils.is_url("wasssuuuup???")
assert utils.is_url('file://foo.com/path/to/file.xyz')
assert utils.is_url('file:///path/to/file.xyz')
assert utils.is_url('file:/path/to/file.xyz')
assert utils.is_url('file:///c:/WINDOWS/clock.avi')
assert utils.is_url('file:c:/WINDOWS/clock.avi')
assert not utils.is_url('c:/WINDOWS/ugh/ugh.ugh')
def test_is_url():
"""test the ability to determine whether a string is a URL"""
assert utils.is_url("http://mydomain.com/foo/bar/bat?asdf=1234&qewr=ooo")
assert utils.is_url("http://xkcd.com/1193/")
assert not utils.is_url("syn123445")
assert not utils.is_url("wasssuuuup???")
assert utils.is_url('file://foo.com/path/to/file.xyz')
assert utils.is_url('file:///path/to/file.xyz')
assert utils.is_url('file:/path/to/file.xyz')
assert utils.is_url('file:///c:/WINDOWS/clock.avi')
assert utils.is_url('file:c:/WINDOWS/clock.avi')
assert not utils.is_url('c:/WINDOWS/ugh/ugh.ugh')
def test_windows_file_urls():
url = 'file:///c:/WINDOWS/clock.avi'
assert utils.is_url(url)
assert utils.file_url_to_path(
url,
verify_exists=False) == 'c:/WINDOWS/clock.avi', utils.file_url_to_path(
url)
def _check_path_and_normalize(f):
sys.stdout.write('.')
if is_url(f):
return f
path_normalized = os.path.abspath(os.path.expandvars(os.path.expanduser(f)))
if not os.path.isfile(path_normalized):
print('\nThe specified path "%s" is either not a file path or does not exist.', f)
raise IOError('The path %s is not a file or does not exist' %f)
return path_normalized
def local_file_has_changed(entityBundle, checkIndirect, path=None):
"""
Checks the local cache to see if the given file has been modified.
:param entityBundle : A dictionary with 'fileHandles' and 'entity'.
Typically created via::
syn._getEntityBundle()
:param checkIndirect: Whether or not the cache should be checked for unmodified files.
Should be True when getting, False when storing.
:param path: Path to the local file. May be in any format.
If not given, the information from the 'entityBundle'
is used to derive a cached location for the file.
:returns: True if the file has been modified.
"""
# Find the directory of the '.cacheMap' for the file
cacheDir, filepath, _ = determine_local_file_location(entityBundle)
if path is None:
path = filepath
# If there is no file path, there is nothing to download
if path is None:
return False
# For external URLs, if the path has not changed
# then the file handle does not have to change
if utils.is_url(path):
for handle in entityBundle['fileHandles']:
if handle['id'] == entityBundle['entity']['dataFileHandleId'] \
and handle['concreteType'] == 'org.sagebionetworks.repo.model.file.ExternalFileHandle' \
and handle['externalURL'] == path:
return False
return True
# Compare the modification times
path = utils.normalize_path(path)
fileMTime = get_modification_time(path)
unmodifiedFileExists = False
for file, cacheTime, cachedFileMTime in iterator_over_cache_map(cacheDir):
# When there is a direct match, return if it is modified
if path == file and os.path.exists(path):
return not fileMTime == cacheTime
# If there is no direct match, but a copy exists, return False after checking all entries
if checkIndirect and cachedFileMTime == cacheTime:
unmodifiedFileExists = True
# The file is cached but not in the right place copy it add it to the cache
if checkIndirect and unmodifiedFileExists and not path.startswith(CACHE_DIR):
add_local_file_to_cache(path=path, **entityBundle['entity'])
return not unmodifiedFileExists
def local_file_has_changed(entityBundle, path=None):
"""
Checks the local cache to see if the given file has been modified.
:param entityBundle: A dictionary with 'fileHandles' and 'entity'.
Typically created via::
syn._getEntityBundle()
:param path: Path to the local file. May be in any format.
If not given, the information from the 'entityBundle'
is used to derive a cached location for the file.
:returns: True if the file has been modified.
"""
# Find the directory of the '.cacheMap' for the file
cacheDir, filepath, _ = determine_local_file_location(entityBundle)
if path is None:
path = filepath
# If there is no file path, there is nothing to download
if path is None:
return False
# External URLs will be ignored
if utils.is_url(path):
return True
# Compare the modification times
path = normalize_path(path)
fileMTime = get_modification_time(path)
unmodifiedFileExists = False
for file, cacheTime, _ in iterator_over_cache_map(cacheDir):
# When there is a direct match, return if it is modified
if path == file and os.path.exists(path):
return not fileMTime == cacheTime
# If there is no direct match, but a pristine copy exists, return False (after checking all entries)
# The filecmp is necessary for Windows machines since their clocks do not keep millisecond information
# i.e. Two files created quickly may have the same timestamp
if fileMTime == cacheTime and os.path.exists(file) and filecmp.cmp(
path, file):
unmodifiedFileExists = True
# The file is not cached or has been changed
return not unmodifiedFileExists
def local_file_has_changed(entityBundle, checkIndirect, path=None):
"""
Checks the local cache to see if the given file has been modified.
:param entityBundle : A dictionary with 'fileHandles' and 'entity'.
Typically created via::
syn._getEntityBundle()
:param checkIndirect: Whether or not the cache should be checked for unmodified files.
Should be True when getting, False when storing.
:param path: Path to the local file. May be in any format.
If not given, the information from the 'entityBundle'
is used to derive a cached location for the file.
:returns: True if the file has been modified.
"""
# Find the directory of the '.cacheMap' for the file
cacheDir, filepath, _ = determine_local_file_location(entityBundle)
if path is None:
path = filepath
# If there is no file path, there is nothing to download
if path is None:
return False
# External URLs will be ignored
if utils.is_url(path):
return True
# Compare the modification times
path = normalize_path(path)
fileMTime = get_modification_time(path)
unmodifiedFileExists = False
for file, cacheTime, cachedFileMTime in iterator_over_cache_map(cacheDir):
# When there is a direct match, return if it is modified
if path == file and os.path.exists(path):
return not fileMTime == cacheTime
# If there is no direct match, but a pristine copy exists, return False (after checking all entries)
if checkIndirect and cachedFileMTime == cacheTime:
unmodifiedFileExists = True
# The file is not cached or has been changed
return not unmodifiedFileExists
def local_file_has_changed(entityBundle, path=None):
"""
Checks the local cache to see if the given file has been modified.
:param entityBundle: A dictionary with 'fileHandles' and 'entity'.
Typically created via::
syn._getEntityBundle()
:param path: Path to the local file. May be in any format.
If not given, the information from the 'entityBundle'
is used to derive a cached location for the file.
:returns: True if the file has been modified.
"""
# Find the directory of the '.cacheMap' for the file
cacheDir, filepath, _ = determine_local_file_location(entityBundle)
if path is None:
path = filepath
# If there is no file path, there is nothing to download
if path is None:
return False
# External URLs will be ignored
if utils.is_url(path):
return True
# Compare the modification times
path = normalize_path(path)
fileMTime = get_modification_time(path)
unmodifiedFileExists = False
for file, cacheTime, _ in iterator_over_cache_map(cacheDir):
# When there is a direct match, return if it is modified
if path == file and os.path.exists(path):
return not fileMTime == cacheTime
# If there is no direct match, but a pristine copy exists, return False (after checking all entries)
# The filecmp is necessary for Windows machines since their clocks do not keep millisecond information
# i.e. Two files created quickly may have the same timestamp
if fileMTime == cacheTime and os.path.exists(file) and filecmp.cmp(path, file):
unmodifiedFileExists = True
# The file is not cached or has been changed
return not unmodifiedFileExists
def add_local_file_to_cache(**entity):
"""
Makes a '.cacheMap' entry in the cache.
:param entity: A Synapse Entity object or dictionary with a 'path'.
FileEntities require a 'dataFileHandleID'.
Locationables require a 'id' and 'versionNumber'.
Example::
foo = File('/path/to/file/xyz.txt')
cache.add_local_file_to_cache(bar="Something to include in dict", **foo)
Note: Since neither FileEntities nor Locationables have a 'path' in their properties,
calls to this method should look like::
cache.add_local_file_to_cache(path=entity['path'], **entity)
"""
# External URLs will be ignored
if utils.is_url(entity['path']) or entity['path'] is None:
return
# Get the '.cacheMap'
cacheDir = determine_cache_directory(entity)
entity['path'] = utils.normalize_path(entity['path'])
# If the file to-be-added does not exist, search the cache for a pristine copy
if not os.path.exists(entity['path']):
for file, cacheTime, fileMTime in iterator_over_cache_map(cacheDir):
if fileMTime == cacheTime:
shutil.copyfile(file, entity['path'])
break
# Update the cache
if os.path.exists(entity['path']):
cache = obtain_lock_and_read_cache(cacheDir)
cache[entity['path']] = time.strftime(
utils.ISO_FORMAT, time.gmtime(os.path.getmtime(entity['path'])))
write_cache_then_release_lock(cacheDir, cache)
def add_local_file_to_cache(**entity):
"""
Makes a '.cacheMap' entry in the cache.
:param entity: A Synapse Entity object or dictionary with a 'path'.
FileEntities require a 'dataFileHandleID'.
Locationables require a 'id' and 'versionNumber'.
Example::
foo = File('/path/to/file/xyz.txt')
cache.add_local_file_to_cache(bar="Something to include in dict", **foo)
Note: Since neither FileEntities nor Locationables have a 'path' in their properties,
calls to this method should look like::
cache.add_local_file_to_cache(path=entity['path'], **entity)
"""
# External URLs will be ignored
if utils.is_url(entity['path']):
return True
# Get the '.cacheMap'
cacheDir = determine_cache_directory(entity)
entity['path'] = normalize_path(entity['path'])
# If the file to-be-added does not exist, search the cache for a pristine copy
if not os.path.exists(entity['path']):
for file, cacheTime, fileMTime in iterator_over_cache_map(cacheDir):
if fileMTime == cacheTime:
shutil.copyfile(file, entity['path'])
break
# Update the cache
if os.path.exists(entity['path']):
cache = obtain_lock_and_read_cache(cacheDir)
cache[entity['path']] = time.strftime(utils.ISO_FORMAT, time.gmtime(os.path.getmtime(entity['path'])))
write_cache_then_release_lock(cacheDir, cache)
def used(self,
target=None,
targetVersion=None,
wasExecuted=None,
url=None,
name=None):
"""
Add a resource used by the activity.
This method tries to be as permissive as possible. It accepts a string which might be a synapse ID or a URL,
a synapse entity, a UsedEntity or UsedURL dictionary or a list containing any combination of these.
In addition, named parameters can be used to specify the fields of either a UsedEntity or a UsedURL.
If target and optionally targetVersion are specified, create a UsedEntity.
If url and optionally name are specified, create a UsedURL.
It is an error to specify both target/targetVersion parameters and url/name parameters in the same call.
To add multiple UsedEntities and UsedURLs, make a separate call for each or pass in a list.
In case of conflicting settings for wasExecuted both inside an object and with a parameter, the parameter wins.
For example, this UsedURL will have wasExecuted set to False::
activity.used({'url':'http://google.com', 'name':'Goog', 'wasExecuted':True}, wasExecuted=False)
Entity examples::
activity.used('syn12345')
activity.used(entity)
activity.used(target=entity, targetVersion=2)
activity.used(codeEntity, wasExecuted=True)
activity.used({'reference':{'target':'syn12345', 'targetVersion':1}, 'wasExecuted':False})
URL examples::
activity.used('http://mydomain.com/my/awesome/data.RData')
activity.used(url='http://mydomain.com/my/awesome/data.RData', name='Awesome Data')
activity.used(url='https://github.com/joe_hacker/code_repo', name='Gnarly hacks', wasExecuted=True)
activity.used({'url':'https://github.com/joe_hacker/code_repo', 'name':'Gnarly hacks'}, wasExecuted=True)
List example::
activity.used(['syn12345', 'syn23456', entity, \
{'reference':{'target':'syn100009', 'targetVersion':2}, 'wasExecuted':True}, \
'http://mydomain.com/my/awesome/data.RData'])
"""
# -- A list of targets
if isinstance(target, list):
badargs = _get_any_bad_args(['targetVersion', 'url', 'name'],
locals())
_raise_incorrect_used_usage(badargs, 'list of used resources')
for item in target:
self.used(item, wasExecuted=wasExecuted)
return
# -- UsedEntity
elif is_used_entity(target):
badargs = _get_any_bad_args(['targetVersion', 'url', 'name'],
locals())
_raise_incorrect_used_usage(
badargs, 'dictionary representing a used resource')
resource = target
if 'concreteType' not in resource:
resource[
'concreteType'] = 'org.sagebionetworks.repo.model.provenance.UsedEntity'
# -- Used URL
elif is_used_url(target):
badargs = _get_any_bad_args(['targetVersion', 'url', 'name'],
locals())
_raise_incorrect_used_usage(badargs, 'URL')
resource = target
if 'concreteType' not in resource:
resource[
'concreteType'] = 'org.sagebionetworks.repo.model.provenance.UsedURL'
# -- Synapse Entity
elif is_synapse_entity(target):
badargs = _get_any_bad_args(['url', 'name'], locals())
_raise_incorrect_used_usage(badargs, 'Synapse entity')
reference = {'targetId': target['id']}
if 'versionNumber' in target:
reference['targetVersionNumber'] = target['versionNumber']
if targetVersion:
reference['targetVersionNumber'] = int(targetVersion)
resource = {
'reference':
reference,
'concreteType':
'org.sagebionetworks.repo.model.provenance.UsedEntity'
}
# -- URL parameter
elif url:
badargs = _get_any_bad_args(['target', 'targetVersion'], locals())
_raise_incorrect_used_usage(badargs, 'URL')
resource = {
'url': url,
'name': name if name else target,
'concreteType':
'org.sagebionetworks.repo.model.provenance.UsedURL'
}
# -- URL as a string
elif is_url(target):
badargs = _get_any_bad_args(['targetVersion'], locals())
_raise_incorrect_used_usage(badargs, 'URL')
resource = {
'url': target,
'name': name if name else target,
'concreteType':
'org.sagebionetworks.repo.model.provenance.UsedURL'
}
# -- Synapse Entity ID (assuming the string is an ID)
elif isinstance(target, six.string_types):
badargs = _get_any_bad_args(['url', 'name'], locals())
_raise_incorrect_used_usage(badargs, 'Synapse entity')
vals = target.split('.') # Handle synapseIds of from syn234.4
if not is_synapse_id(vals[0]):
raise ValueError('%s is not a valid Synapse id' % target)
if len(vals) == 2:
if targetVersion and int(targetVersion) != int(vals[1]):
raise ValueError(
'Two conflicting versions for %s were specified' %
target)
targetVersion = int(vals[1])
reference = {'targetId': vals[0]}
if targetVersion:
reference['targetVersionNumber'] = int(targetVersion)
resource = {
'reference':
reference,
'concreteType':
'org.sagebionetworks.repo.model.provenance.UsedEntity'
}
else:
raise SynapseError(
'Unexpected parameters in call to Activity.used().')
# Set wasExecuted
if wasExecuted is None:
# Default to False
if 'wasExecuted' not in resource:
resource['wasExecuted'] = False
else:
# wasExecuted parameter overrides setting in an object
resource['wasExecuted'] = wasExecuted
# Add the used resource to the activity
self['used'].append(resource)
def test_windows_file_urls():
url = 'file:///c:/WINDOWS/clock.avi'
assert utils.is_url(url)
assert utils.file_url_to_path(url, verify_exists=False).get('path',None) == 'c:/WINDOWS/clock.avi', utils.file_url_to_path(url)
def test_windows_file_urls():
url = 'file:///c:/WINDOWS/clock.avi'
assert_true(utils.is_url(url))
assert_equals(utils.file_url_to_path(url, verify_exists=False), 'c:/WINDOWS/clock.avi', utils.file_url_to_path(url))
def syncToSynapse(syn,
manifestFile,
dryRun=False,
sendMessages=True,
retries=MAX_RETRIES):
"""Synchronizes files specified in the manifest file to Synapse
:param syn: A synapse object as obtained with syn = synapseclient.login()
:param manifestFile: A tsv file with file locations and metadata to be pushed to Synapse.
See below for details
:param dryRun: Performs validation without uploading if set to True (default is False)
Given a file describing all of the uploads uploads the content to Synapse and optionally notifies you via Synapse
messagging (email) at specific intervals, on errors and on completion.
**Manifest file format**
The format of the manifest file is a tab delimited file with one row per file to upload and columns describing the
file. The minimum required columns are **path** and **parent** where path is the local file path and parent is the
Synapse Id of the project or folder where the file is uploaded to. In addition to these columns you can specify any
of the parameters to the File constructor (**name**, **synapseStore**, **contentType**) as well as parameters to the
syn.store command (**used**, **executed**, **activityName**, **activityDescription**, **forceVersion**).
Used and executed can be semi-colon (";") separated lists of Synapse ids, urls and/or local filepaths of files
already stored in Synapse (or being stored in Synapse by the manifest).
Any additional columns will be added as annotations.
**Required fields:**
====== ====================== ============================
Field Meaning Example
====== ====================== ============================
path local file path or URL /path/to/local/file.txt
parent synapse id syn1235
====== ====================== ============================
**Common fields:**
=============== =========================== ============
Field Meaning Example
=============== =========================== ============
name name of file in Synapse Example_file
forceVersion whether to update version False
=============== =========================== ============
**Provenance fields:**
==================== ===================================== ==========================================
Field Meaning Example
==================== ===================================== ==========================================
used List of items used to generate file syn1235; /path/to_local/file.txt
executed List of items exectued https://github.org/; /path/to_local/code.py
activityName Name of activity in provenance "Ran normalization"
activityDescription Text description on what was done "Ran algorithm xyx with parameters..."
==================== ===================================== ==========================================
Annotations:
**Annotations:**
Any columns that are not in the reserved names described above will be interpreted as annotations of the file
**Other optional fields:**
=============== ========================================== ============
Field Meaning Example
=============== ========================================== ============
synapseStore Boolean describing whether to upload files True
contentType content type of file to overload defaults text/html
=============== ========================================== ============
**Example manifest file**
=============== ======== ======= ======= =========================== ============================
path parent annot1 annot2 used executed
=============== ======== ======= ======= =========================== ============================
/path/file1.txt syn1243 "bar" 3.1415 "syn124; /path/file2.txt" "https://github.org/foo/bar"
/path/file2.txt syn12433 "baz" 2.71 "" "https://github.org/foo/baz"
=============== ======== ======= ======= =========================== ============================
"""
df = readManifestFile(syn, manifestFile)
sizes = [
os.stat(os.path.expandvars(os.path.expanduser(f))).st_size
for f in df.path if not is_url(f)
]
# Write output on what is getting pushed and estimated times - send out message.
sys.stdout.write('=' * 50 + '\n')
sys.stdout.write(
'We are about to upload %i files with a total size of %s.\n ' %
(len(df), utils.humanizeBytes(sum(sizes))))
sys.stdout.write('=' * 50 + '\n')
if dryRun:
return
sys.stdout.write('Starting upload...\n')
if sendMessages:
notify_decorator = notifyMe(syn,
'Upload of %s' % manifestFile,
retries=retries)
upload = notify_decorator(_manifest_upload)
upload(syn, df)
else:
_manifest_upload(syn, df)
def used(self, target=None, targetVersion=None, wasExecuted=None, url=None, name=None):
"""
Add a resource used by the activity.
This method tries to be as permissive as possible. It accepts a string which might
be a synapse ID or a URL, a synapse entity, a UsedEntity or UsedURL dictionary or
a list containing any combination of these.
In addition, named parameters can be used to specify the fields of either a
UsedEntity or a UsedURL. If target and optionally targetVersion are specified,
create a UsedEntity. If url and optionally name are specified, create a UsedURL.
It is an error to specify both target/targetVersion parameters and url/name
parameters in the same call. To add multiple UsedEntities and UsedURLs, make a
separate call for each or pass in a list.
In case of conflicting settings for wasExecuted both inside an object and with a
parameter, the parameter wins. For example, this UsedURL will have wasExecuted set
to False::
activity.used({'url':'http://google.com', 'name':'Goog', 'wasExecuted':True}, wasExecuted=False)
Entity examples::
activity.used('syn12345')
activity.used(entity)
activity.used(target=entity, targetVersion=2)
activity.used(codeEntity, wasExecuted=True)
activity.used({'reference':{'target':'syn12345', 'targetVersion':1}, 'wasExecuted':False})
URL examples::
activity.used('http://mydomain.com/my/awesome/data.RData')
activity.used(url='http://mydomain.com/my/awesome/data.RData', name='Awesome Data')
activity.used(url='https://github.com/joe_hacker/code_repo', name='Gnarly hacks', wasExecuted=True)
activity.used({'url':'https://github.com/joe_hacker/code_repo', 'name':'Gnarly hacks'}, wasExecuted=True)
List example::
used( [ 'syn12345', 'syn23456', entity,
{'reference':{'target':'syn100009', 'targetVersion':2}, 'wasExecuted':True},
'http://mydomain.com/my/awesome/data.RData' ] )
"""
# Check for allowed combinations of parameters and generate specific error message
# based on the context. For example, if we specify a URL, it's illegal to specify
# a version.
def check_for_invalid_parameters(context=None, params={}):
err_msg = 'Error in call to Activity.used()'
if context == 'list':
illegal_params = ('targetVersion', 'url', 'name',)
context_msg = 'list of used resources'
elif context == 'dict':
illegal_params = ('targetVersion', 'url', 'name',)
context_msg = 'dictionary representing a used resource'
elif context == 'url_param':
illegal_params = ('target', 'targetVersion',)
context_msg = 'URL'
elif context == 'url_string':
illegal_params = ('targetVersion',)
context_msg = 'URL'
elif context == 'entity':
illegal_params = ('url', 'name',)
context_msg = 'Synapse entity'
else:
illegal_params = ()
context_msg = '?'
for param in illegal_params:
if param in params and params[param] is not None:
raise Exception('%s: It is an error to specify the \'%s\' parameter in combination with a %s.' % (err_msg, str(param), context_msg))
# List
if isinstance(target, list):
check_for_invalid_parameters(context='list', params=locals())
for item in target:
self.used(item, wasExecuted=wasExecuted)
return
# Used Entity
elif is_used_entity(target):
check_for_invalid_parameters(context='dict', params=locals())
resource = target
if 'concreteType' not in resource:
resource['concreteType'] = 'org.sagebionetworks.repo.model.provenance.UsedEntity'
# Used URL
elif is_used_url(target):
check_for_invalid_parameters(context='dict', params=locals())
resource = target
if 'concreteType' not in resource:
resource['concreteType'] = 'org.sagebionetworks.repo.model.provenance.UsedURL'
# Synapse Entity
elif is_synapse_entity(target):
check_for_invalid_parameters(context='entity', params=locals())
reference = {'targetId':target['id']}
if 'versionNumber' in target:
reference['targetVersionNumber'] = target['versionNumber']
## if targetVersion is specified as a parameter, it overrides the version in the object
if targetVersion:
reference['targetVersionNumber'] = int(targetVersion)
resource = {'reference':reference, 'concreteType':'org.sagebionetworks.repo.model.provenance.UsedEntity'}
# URL parameter
elif url:
check_for_invalid_parameters(context='url_param', params=locals())
resource = {'url':url, 'name':name if name else target, 'concreteType':'org.sagebionetworks.repo.model.provenance.UsedURL'}
# URL as a string
elif is_url(target):
check_for_invalid_parameters(context='url_string', params=locals())
resource = {'url':target, 'name':name if name else target, 'concreteType':'org.sagebionetworks.repo.model.provenance.UsedURL'}
# If it's a string and isn't a URL, assume it's a Synapse Entity ID
elif isinstance(target, basestring):
check_for_invalid_parameters(context='entity', params=locals())
reference = {'targetId':target}
if targetVersion:
reference['targetVersionNumber'] = int(targetVersion)
resource = {'reference':reference, 'concreteType':'org.sagebionetworks.repo.model.provenance.UsedEntity'}
else:
raise Exception('Unexpected parameters in call to Activity.used().')
# Set wasExecuted
if wasExecuted is None:
# Default to False
if 'wasExecuted' not in resource:
resource['wasExecuted'] = False
else:
# wasExecuted parameter overrides setting in an object
resource['wasExecuted'] = wasExecuted
# Add the used resource to the activity
self['used'].append(resource)
def used(self,
target=None,
targetVersion=None,
wasExecuted=None,
url=None,
name=None):
"""
Add a resource used by the activity.
This method tries to be as permissive as possible. It accepts a string which might
be a synapse ID or a URL, a synapse entity, a UsedEntity or UsedURL dictionary or
a list containing any combination of these.
In addition, named parameters can be used to specify the fields of either a
UsedEntity or a UsedURL. If target and optionally targetVersion are specified,
create a UsedEntity. If url and optionally name are specified, create a UsedURL.
It is an error to specify both target/targetVersion parameters and url/name
parameters in the same call. To add multiple UsedEntities and UsedURLs, make a
separate call for each or pass in a list.
In case of conflicting settings for wasExecuted both inside an object and with a
parameter, the parameter wins. For example, this UsedURL will have wasExecuted set
to False::
activity.used({'url':'http://google.com', 'name':'Goog', 'wasExecuted':True}, wasExecuted=False)
Entity examples::
activity.used('syn12345')
activity.used(entity)
activity.used(target=entity, targetVersion=2)
activity.used(codeEntity, wasExecuted=True)
activity.used({'reference':{'target':'syn12345', 'targetVersion':1}, 'wasExecuted':False})
URL examples::
activity.used('http://mydomain.com/my/awesome/data.RData')
activity.used(url='http://mydomain.com/my/awesome/data.RData', name='Awesome Data')
activity.used(url='https://github.com/joe_hacker/code_repo', name='Gnarly hacks', wasExecuted=True)
activity.used({'url':'https://github.com/joe_hacker/code_repo', 'name':'Gnarly hacks'}, wasExecuted=True)
List example::
used( [ 'syn12345', 'syn23456', entity,
{'reference':{'target':'syn100009', 'targetVersion':2}, 'wasExecuted':True},
'http://mydomain.com/my/awesome/data.RData' ] )
"""
# Check for allowed combinations of parameters and generate specific error message
# based on the context. For example, if we specify a URL, it's illegal to specify
# a version.
def check_for_invalid_parameters(context=None, params={}):
err_msg = 'Error in call to Activity.used()'
if context == 'list':
illegal_params = (
'targetVersion',
'url',
'name',
)
context_msg = 'list of used resources'
elif context == 'dict':
illegal_params = (
'targetVersion',
'url',
'name',
)
context_msg = 'dictionary representing a used resource'
elif context == 'url_param':
illegal_params = (
'target',
'targetVersion',
)
context_msg = 'URL'
elif context == 'url_string':
illegal_params = ('targetVersion', )
context_msg = 'URL'
elif context == 'entity':
illegal_params = (
'url',
'name',
)
context_msg = 'Synapse entity'
else:
illegal_params = ()
context_msg = '?'
for param in illegal_params:
if param in params and params[param] is not None:
raise Exception(
'%s: It is an error to specify the \'%s\' parameter in combination with a %s.'
% (err_msg, str(param), context_msg))
# List
if isinstance(target, list):
check_for_invalid_parameters(context='list', params=locals())
for item in target:
self.used(item, wasExecuted=wasExecuted)
return
# Used Entity
elif is_used_entity(target):
check_for_invalid_parameters(context='dict', params=locals())
resource = target
if 'concreteType' not in resource:
resource[
'concreteType'] = 'org.sagebionetworks.repo.model.provenance.UsedEntity'
# Used URL
elif is_used_url(target):
check_for_invalid_parameters(context='dict', params=locals())
resource = target
if 'concreteType' not in resource:
resource[
'concreteType'] = 'org.sagebionetworks.repo.model.provenance.UsedURL'
# Synapse Entity
elif is_synapse_entity(target):
check_for_invalid_parameters(context='entity', params=locals())
reference = {'targetId': target['id']}
if 'versionNumber' in target:
reference['targetVersionNumber'] = target['versionNumber']
## if targetVersion is specified as a parameter, it overrides the version in the object
if targetVersion:
reference['targetVersionNumber'] = int(targetVersion)
resource = {
'reference':
reference,
'concreteType':
'org.sagebionetworks.repo.model.provenance.UsedEntity'
}
# URL parameter
elif url:
check_for_invalid_parameters(context='url_param', params=locals())
resource = {
'url': url,
'name': name if name else target,
'concreteType':
'org.sagebionetworks.repo.model.provenance.UsedURL'
}
# URL as a string
elif is_url(target):
check_for_invalid_parameters(context='url_string', params=locals())
resource = {
'url': target,
'name': name if name else target,
'concreteType':
'org.sagebionetworks.repo.model.provenance.UsedURL'
}
# If it's a string and isn't a URL, assume it's a Synapse Entity ID
elif isinstance(target, basestring):
check_for_invalid_parameters(context='entity', params=locals())
reference = {'targetId': target}
if targetVersion:
reference['targetVersionNumber'] = int(targetVersion)
resource = {
'reference':
reference,
'concreteType':
'org.sagebionetworks.repo.model.provenance.UsedEntity'
}
else:
raise Exception(
'Unexpected parameters in call to Activity.used().')
# Set wasExecuted
if wasExecuted is None:
# Default to False
if 'wasExecuted' not in resource:
resource['wasExecuted'] = False
else:
# wasExecuted parameter overrides setting in an object
resource['wasExecuted'] = wasExecuted
# Add the used resource to the activity
self['used'].append(resource)
def used(self, target=None, targetVersion=None, wasExecuted=None, url=None, name=None):
"""
Add a resource used by the activity.
This method tries to be as permissive as possible. It accepts a string which might
be a synapse ID or a URL, a synapse entity, a UsedEntity or UsedURL dictionary or
a list containing any combination of these.
In addition, named parameters can be used to specify the fields of either a
UsedEntity or a UsedURL. If target and optionally targetVersion are specified,
create a UsedEntity. If url and optionally name are specified, create a UsedURL.
It is an error to specify both target/targetVersion parameters and url/name
parameters in the same call. To add multiple UsedEntities and UsedURLs, make a
separate call for each or pass in a list.
In case of conflicting settings for wasExecuted both inside an object and with a
parameter, the parameter wins. For example, this UsedURL will have wasExecuted set
to False::
activity.used({'url':'http://google.com', 'name':'Goog', 'wasExecuted':True}, wasExecuted=False)
Entity examples::
activity.used('syn12345')
activity.used(entity)
activity.used(target=entity, targetVersion=2)
activity.used(codeEntity, wasExecuted=True)
activity.used({'reference':{'target':'syn12345', 'targetVersion':1}, 'wasExecuted':False})
URL examples::
activity.used('http://mydomain.com/my/awesome/data.RData')
activity.used(url='http://mydomain.com/my/awesome/data.RData', name='Awesome Data')
activity.used(url='https://github.com/joe_hacker/code_repo', name='Gnarly hacks', wasExecuted=True)
activity.used({'url':'https://github.com/joe_hacker/code_repo', 'name':'Gnarly hacks'}, wasExecuted=True)
List example::
used( [ 'syn12345', 'syn23456', entity,
{'reference':{'target':'syn100009', 'targetVersion':2}, 'wasExecuted':True},
'http://mydomain.com/my/awesome/data.RData' ] )
"""
# -- A list of targets
if isinstance(target, list):
badargs = _get_any_bad_args(['targetVersion', 'url', 'name'], locals())
_raise_incorrect_used_usage(badargs, 'list of used resources')
for item in target:
self.used(item, wasExecuted=wasExecuted)
return
# -- UsedEntity
elif is_used_entity(target):
badargs = _get_any_bad_args(['targetVersion', 'url', 'name'], locals())
_raise_incorrect_used_usage(badargs, 'dictionary representing a used resource')
resource = target
if 'concreteType' not in resource:
resource['concreteType'] = 'org.sagebionetworks.repo.model.provenance.UsedEntity'
# -- Used URL
elif is_used_url(target):
badargs = _get_any_bad_args(['targetVersion', 'url', 'name'], locals())
_raise_incorrect_used_usage(badargs, 'URL')
resource = target
if 'concreteType' not in resource:
resource['concreteType'] = 'org.sagebionetworks.repo.model.provenance.UsedURL'
# -- Synapse Entity
elif is_synapse_entity(target):
badargs = _get_any_bad_args(['url', 'name'], locals())
_raise_incorrect_used_usage(badargs, 'Synapse entity')
reference = {'targetId':target['id']}
if 'versionNumber' in target:
reference['targetVersionNumber'] = target['versionNumber']
if targetVersion:
reference['targetVersionNumber'] = int(targetVersion)
resource = {'reference':reference, 'concreteType':'org.sagebionetworks.repo.model.provenance.UsedEntity'}
# -- URL parameter
elif url:
badargs = _get_any_bad_args(['target', 'targetVersion'], locals())
_raise_incorrect_used_usage(badargs, 'URL')
resource = {'url':url, 'name':name if name else target, 'concreteType':'org.sagebionetworks.repo.model.provenance.UsedURL'}
# -- URL as a string
elif is_url(target):
badargs = _get_any_bad_args(['targetVersion'], locals())
_raise_incorrect_used_usage(badargs, 'URL')
resource = {'url':target, 'name':name if name else target, 'concreteType':'org.sagebionetworks.repo.model.provenance.UsedURL'}
# -- Synapse Entity ID (assuming the string is an ID)
elif isinstance(target, basestring):
badargs = _get_any_bad_args(['url', 'name'], locals())
_raise_incorrect_used_usage(badargs, 'Synapse entity')
reference = {'targetId':target}
if targetVersion:
reference['targetVersionNumber'] = int(targetVersion)
resource = {'reference':reference, 'concreteType':'org.sagebionetworks.repo.model.provenance.UsedEntity'}
else:
raise SynapseError('Unexpected parameters in call to Activity.used().')
# Set wasExecuted
if wasExecuted is None:
# Default to False
if 'wasExecuted' not in resource:
resource['wasExecuted'] = False
else:
# wasExecuted parameter overrides setting in an object
resource['wasExecuted'] = wasExecuted
# Add the used resource to the activity
self['used'].append(resource)
def test_windows_file_urls():
url = "file:///c:/WINDOWS/clock.avi"
assert utils.is_url(url)
assert utils.file_url_to_path(url, verify_exists=False) == "c:/WINDOWS/clock.avi", utils.file_url_to_path(url)
