def test_getChildren():
# setup a hierarchy for folders
# PROJECT
# | \
# File Folder
# |
# File
project_name = str(uuid.uuid1())
test_project = syn.store(Project(name=project_name))
folder = syn.store(Folder(name="firstFolder", parent=test_project))
syn.store(
File(path="~/doesntMatter.txt",
name="file inside folders",
parent=folder,
synapseStore=False))
project_file = syn.store(
File(path="~/doesntMatterAgain.txt",
name="file inside project",
parent=test_project,
synapseStore=False))
schedule_for_cleanup(test_project)
expected_id_set = {project_file.id, folder.id}
children_id_set = {x['id'] for x in syn.getChildren(test_project.id)}
assert_equals(expected_id_set, children_id_set)
def test_File_update_file_handle__External_non_sftp():
external_file_handle = {'concreteType': 'org.sagebionetworks.repo.model.file.ExternalFileHandle',
'externalURL': "https://some.website"}
f = File(parent="idk")
assert_true(f.synapseStore)
f._update_file_handle(external_file_handle)
assert_false(f.synapseStore)
def test_File_update_file_handle__External_non_sftp():
external_file_handle = {
'concreteType':
'org.sagebionetworks.repo.model.file.ExternalFileHandle',
'externalURL': "https://some.website"
}
f = File(parent="idk")
assert_true(f.synapseStore)
f._update_file_handle(external_file_handle)
assert_false(f.synapseStore)
def test_is_synapse_annotation():
assert is_synapse_annotations({'id': 'syn123', 'etag': '0f2977b9-0261-4811-a89e-c13e37ce4604', 'annotations': {}})
# missing id
assert not is_synapse_annotations({'etag': '0f2977b9-0261-4811-a89e-c13e37ce4604', 'annotations': {}})
# missing etag
assert not is_synapse_annotations({'id': 'syn123', 'annotations': {}})
# missing annotations
assert not is_synapse_annotations({'id': 'syn123', 'etag': '0f2977b9-0261-4811-a89e-c13e37ce4604'})
# has additional keys
assert is_synapse_annotations(
{'id': 'syn123', 'etag': '0f2977b9-0261-4811-a89e-c13e37ce4604', 'annotations': {}, 'foo': 'bar'})
# annotations only
assert not is_synapse_annotations({'annotations': {}})
# annotations + other keys
assert not is_synapse_annotations({'annotations': {}, 'bar': 'baz'})
# sanity check: Any Entity subclass has id etag and annotations,
# but its annotations are not in the format Synapse expects
assert not is_synapse_annotations(
File(
'~/asdf.txt',
id='syn123',
etag='0f2977b9-0261-4811-a89e-c13e37ce4604',
parentId='syn135'
)
)
def test_entity_version():
# Make an Entity and make sure the version is one
entity = File(parent=project['id'])
entity['path'] = utils.make_bogus_data_file()
schedule_for_cleanup(entity['path'])
entity = syn.createEntity(entity)
syn.setAnnotations(entity, {'fizzbuzz': 111222})
entity = syn.getEntity(entity)
assert entity.versionNumber == 1
# Update the Entity and make sure the version is incremented
entity.foo = 998877
entity['name'] = 'foobarbat'
entity['description'] = 'This is a test entity...'
entity = syn.updateEntity(entity,
incrementVersion=True,
versionLabel="Prada remix")
assert entity.versionNumber == 2
# Get the older data and verify the random stuff is still there
annotations = syn.getAnnotations(entity, version=1)
assert annotations['fizzbuzz'][0] == 111222
returnEntity = syn.getEntity(entity, version=1)
assert returnEntity.versionNumber == 1
assert returnEntity['fizzbuzz'][0] == 111222
assert 'foo' not in returnEntity
# Try the newer Entity
returnEntity = syn.getEntity(entity)
assert returnEntity.versionNumber == 2
assert returnEntity['foo'][0] == 998877
assert returnEntity['name'] == 'foobarbat'
assert returnEntity['description'] == 'This is a test entity...'
assert returnEntity['versionLabel'] == 'Prada remix'
# Try the older Entity again
returnEntity = syn.downloadEntity(entity, version=1)
assert returnEntity.versionNumber == 1
assert returnEntity['fizzbuzz'][0] == 111222
assert 'foo' not in returnEntity
# Delete version 2
syn.delete(entity, version=2)
returnEntity = syn.getEntity(entity)
assert returnEntity.versionNumber == 1
def test_provenance():
# Create a File Entity
fname = utils.make_bogus_data_file()
schedule_for_cleanup(fname)
data_entity = syn.store(File(fname, parent=project['id']))
# Create a File Entity of Code
fd, path = tempfile.mkstemp(suffix=".py")
os.write(
fd, """
## Chris's fabulous random data generator
############################################################
import random
random.seed(12345)
data = [random.gauss(mu=0.0, sigma=1.0) for i in range(100)]
""")
os.close(fd)
schedule_for_cleanup(path)
code_entity = syn.store(File(path, parent=project['id']))
# Create a new Activity asserting that the Code Entity was 'used'
activity = Activity(name='random.gauss',
description='Generate some random numbers')
activity.used(code_entity, wasExecuted=True)
activity.used(
{
'name': 'Superhack',
'url': 'https://github.com/joe_coder/Superhack'
},
wasExecuted=True)
activity = syn.setProvenance(data_entity, activity)
# Retrieve and verify the saved Provenance record
retrieved_activity = syn.getProvenance(data_entity)
assert retrieved_activity == activity
# Test Activity update
new_description = 'Generate random numbers like a gangsta'
retrieved_activity['description'] = new_description
updated_activity = syn.updateActivity(retrieved_activity)
assert updated_activity['name'] == retrieved_activity['name']
assert updated_activity['description'] == new_description
# Test delete
syn.deleteProvenance(data_entity)
assert_raises(SynapseHTTPError, syn.getProvenance, data_entity['id'])
def test_md5_query():
# Add the same Entity several times
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
repeated = File(path, parent=project['id'], description='Same data over and over again')
# Retrieve the data via MD5
num = 5
stored = []
for i in range(num):
repeated.name = 'Repeated data %d.dat' % i
stored.append(syn.store(repeated).id)
# Although we expect num results, it is possible for the MD5 to be non-unique
results = syn.md5Query(utils.md5_for_file(path).hexdigest())
assert str(sorted(stored)) == str(sorted([res['id'] for res in results]))
assert len(results) == num
def test_md5_query():
# Add the same Entity several times
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
repeated = File(path, parent=project['id'], description='Same data over and over again')
# Retrieve the data via MD5
num = 5
stored = []
for i in range(num):
repeated.name = 'Repeated data %d.dat' % i
stored.append(syn.store(repeated).id)
# Although we expect num results, it is possible for the MD5 to be non-unique
results = syn.md5Query(utils.md5_for_file(path).hexdigest())
assert str(sorted(stored)) == str(sorted([res['id'] for res in results]))
assert len(results) == num
def test_parent_id_required():
xkcd1 = File('http://xkcd.com/1343/',
name='XKCD: Manuals',
parent='syn1000001',
synapseStore=False)
assert xkcd1.parentId == 'syn1000001'
xkcd2 = File('http://xkcd.com/1343/',
name='XKCD: Manuals',
parentId='syn1000002',
synapseStore=False)
assert xkcd2.parentId == 'syn1000002'
assert_raises(SynapseMalformedEntityError,
File,
'http://xkcd.com/1343/',
name='XKCD: Manuals',
synapseStore=False)
def test_md5_query():
# Create a project then add the same entity several times and retrieve them via MD5
project = create_project()
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
repeated = File(path, parent=project['id'], description='Same data over and over again')
num = 5
for i in range(num):
try:
repeated.name = 'Repeated data %d.dat' % i
syn.store(repeated)
except Exception as ex:
print ex
print ex.response.text
results = syn.md5Query(utils.md5_for_file(path).hexdigest())
print [res['id'] for res in results]
def test_entity_version():
# Make an Entity and make sure the version is one
entity = File(parent=project['id'])
entity['path'] = utils.make_bogus_data_file()
schedule_for_cleanup(entity['path'])
entity = syn.createEntity(entity)
syn.setAnnotations(entity, {'fizzbuzz':111222})
entity = syn.getEntity(entity)
assert entity.versionNumber == 1
# Update the Entity and make sure the version is incremented
entity.foo = 998877
entity['name'] = 'foobarbat'
entity['description'] = 'This is a test entity...'
entity = syn.updateEntity(entity, incrementVersion=True, versionLabel="Prada remix")
assert entity.versionNumber == 2
# Get the older data and verify the random stuff is still there
annotations = syn.getAnnotations(entity, version=1)
assert annotations['fizzbuzz'][0] == 111222
returnEntity = syn.getEntity(entity, version=1)
assert returnEntity.versionNumber == 1
assert returnEntity['fizzbuzz'][0] == 111222
assert 'foo' not in returnEntity
# Try the newer Entity
returnEntity = syn.getEntity(entity)
assert returnEntity.versionNumber == 2
assert returnEntity['foo'][0] == 998877
assert returnEntity['name'] == 'foobarbat'
assert returnEntity['description'] == 'This is a test entity...'
assert returnEntity['versionLabel'] == 'Prada remix'
# Try the older Entity again
returnEntity = syn.downloadEntity(entity, version=1)
assert returnEntity.versionNumber == 1
assert returnEntity['fizzbuzz'][0] == 111222
assert 'foo' not in returnEntity
# Delete version 2
syn.delete(entity, version=2)
returnEntity = syn.getEntity(entity)
assert returnEntity.versionNumber == 1
def test_keys():
f = File('foo.xyz', parent='syn1234', foo='bar')
iter_keys = []
for key in f:
iter_keys.append(key)
assert 'parentId' in iter_keys
assert 'name' in iter_keys
assert 'foo' in iter_keys
assert 'concreteType' in iter_keys
def test_ExternalObjectStore_roundtrip():
endpoint = "https://s3.amazonaws.com"
bucket = "test-client-auth-s3"
profile_name = syn._get_client_authenticated_s3_profile(endpoint, bucket)
if profile_name != 'client-auth-s3-test':
raise SkipTest(
"This test only works on travis because it requires AWS credentials to a specific S3 bucket"
)
proj = syn.store(
Project(name=str(uuid.uuid4()) + "ExternalObjStoreProject"))
schedule_for_cleanup(proj)
storage_location = syn.createStorageLocationSetting(
"ExternalObjectStorage", endpointUrl=endpoint, bucket=bucket)
syn.setStorageLocation(proj, storage_location['storageLocationId'])
file_path = utils.make_bogus_data_file()
file_entity = File(file_path, name="TestName", parent=proj)
file_entity = syn.store(file_entity)
syn.cache.purge(time.time())
assert_is_none(syn.cache.get(file_entity['dataFileHandleId']))
# verify key is in s3
import boto3
boto_session = boto3.session.Session(profile_name=profile_name)
s3 = boto_session.resource('s3', endpoint_url=endpoint)
try:
s3_file = s3.Object(file_entity._file_handle.bucket,
file_entity._file_handle.fileKey)
s3_file.load()
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == "404":
raise Exception("The file was not uploaded to S3")
file_entity_downloaded = syn.get(file_entity['id'])
file_handle = file_entity_downloaded['_file_handle']
# verify file_handle metadata
assert_equals(endpoint, file_handle['endpointUrl'])
assert_equals(bucket, file_handle['bucket'])
assert_equals(
utils.md5_for_file(file_path).hexdigest(), file_handle['contentMd5'])
assert_equals(os.stat(file_path).st_size, file_handle['contentSize'])
assert_equals('text/plain', file_handle['contentType'])
assert_not_equal(utils.normalize_path(file_path),
utils.normalize_path(file_entity_downloaded['path']))
assert_true(filecmp.cmp(file_path, file_entity_downloaded['path']))
# clean up
s3_file.delete()
def test_entity_constructors():
project = Project(name=str(uuid.uuid4()), description='Testing 123')
folder = Folder(name='Musicians',
parent=project,
genre='Jazz',
datatype='personnel')
personnel_file = File(fname,
parentId=folder.id,
group='Miles Davis Quintet',
album='Stockholm 1960 Complete')
def test_md5_query():
# Create a project then add the same entity several times and retrieve them via MD5
project = create_project()
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
repeated = File(path,
parent=project['id'],
description='Same data over and over again')
num = 5
for i in range(num):
try:
repeated.name = 'Repeated data %d.dat' % i
syn.store(repeated)
except Exception as ex:
print ex
print ex.response.text
results = syn.md5Query(utils.md5_for_file(path).hexdigest())
print[res['id'] for res in results]
## Not sure how to make this assertion more accurate
## Although we expect num results, it is possible for the MD5 to be non-unique
assert len(results) == num
def test_entity_constructors():
project = Project('TestProject', id='syn1001', foo='bar')
assert project.name == 'TestProject'
assert project['foo'] == 'bar'
folder = Folder('MyFolder', parent=project, foo='bat', id='syn1002')
assert folder.name == 'MyFolder'
assert folder.foo == 'bat'
assert folder.parentId == 'syn1001'
a_file = File('/path/to/fabulous_things.zzz', parent=folder, foo='biz')
#assert a_file.name == 'fabulous_things.zzz'
assert a_file.entityType == 'org.sagebionetworks.repo.model.FileEntity'
assert a_file.path == '/path/to/fabulous_things.zzz'
assert a_file.foo == 'biz'
assert a_file.parentId == 'syn1002'
def test_external_s3_upload():
#skip if not on the synapse-test user
if syn.username != 'synapse-test':
raise SkipTest(
"This test is configured to work on synapse's TravisCI. If you wish to run this locally, please create an external S3 bucket that your Synapse username can access (http://docs.synapse.org/articles/custom_storage_location.html) and modify the EXTERNAL_S3_BUCKET variable"
)
#setup
project_id, storage_location_id = _set_up_external_s3_project()
# create a temporary file for upload
temp_file_path = utils.make_bogus_data_file()
expected_md5 = utils.md5_for_file(temp_file_path).hexdigest()
schedule_for_cleanup(temp_file_path)
#upload the file
uploaded_syn_file = syn.store(File(path=temp_file_path, parent=project_id))
#get file_handle of the uploaded file
file_handle = syn.restGET('/entity/%s/filehandles' %
uploaded_syn_file.id)['list'][0]
#Verify correct file handle type
assert_equals(file_handle['concreteType'],
'org.sagebionetworks.repo.model.file.S3FileHandle')
# Verify storage location id to make sure it's using external S3
assert_equals(storage_location_id, file_handle['storageLocationId'])
#Verify md5 of upload
assert_equals(expected_md5, file_handle['contentMd5'])
# clear the cache and download the file
syn.cache.purge(time.time())
downloaded_syn_file = syn.get(uploaded_syn_file.id)
#verify the correct file was downloaded
assert_equals(os.path.basename(downloaded_syn_file['path']),
os.path.basename(temp_file_path))
assert_not_equal(os.path.normpath(temp_file_path),
os.path.normpath(downloaded_syn_file['path']))
assert filecmp.cmp(temp_file_path, downloaded_syn_file['path'])
def test_entity_constructors():
project = Project('TestProject', id='syn1001', foo='bar')
assert_equals(project.name, 'TestProject')
assert_equals(project['foo'], 'bar')
folder = Folder('MyFolder', parent=project, foo='bat', id='syn1002')
assert_equals(folder.name, 'MyFolder')
assert_equals(folder.foo, 'bat')
assert_equals(folder.parentId, 'syn1001')
a_file = File('/path/to/fabulous_things.zzz',
parent=folder,
foo='biz',
contentType='application/cattywampus')
assert_equals(a_file.concreteType,
'org.sagebionetworks.repo.model.FileEntity')
assert_equals(a_file.path, '/path/to/fabulous_things.zzz')
assert_equals(a_file.foo, 'biz')
assert_equals(a_file.parentId, 'syn1002')
assert_equals(a_file.contentType, 'application/cattywampus')
assert_in('contentType', a_file._file_handle)
def test_entity_constructors():
project = Project('TestProject', id='syn1001', foo='bar')
assert project.name == 'TestProject'
assert project['foo'] == 'bar'
folder = Folder('MyFolder', parent=project, foo='bat', id='syn1002')
assert folder.name == 'MyFolder'
assert folder.foo == 'bat'
assert folder.parentId == 'syn1001'
a_file = File('/path/to/fabulous_things.zzz',
parent=folder,
foo='biz',
contentType='application/cattywampus')
#assert a_file.name == 'fabulous_things.zzz'
assert a_file.concreteType == 'org.sagebionetworks.repo.model.FileEntity'
assert a_file.path == '/path/to/fabulous_things.zzz'
assert a_file.foo == 'biz'
assert a_file.parentId == 'syn1002'
assert a_file.contentType == 'application/cattywampus'
assert 'contentType' in a_file.__dict__
def test_is_container():
## result from a Synapse entity annotation query
## Note: prefix may be capitalized or not, depending on the from clause of the query
result = {
'entity.versionNumber': 1,
'entity.nodeType': 'project',
'entity.concreteType': ['org.sagebionetworks.repo.model.Project'],
'entity.createdOn': 1451512703905,
'entity.id': 'syn5570912',
'entity.name': 'blah'
}
assert is_container(result)
result = {
'Entity.nodeType': 'project',
'Entity.id': 'syn5570912',
'Entity.name': 'blah'
}
assert is_container(result)
result = {
'entity.concreteType': ['org.sagebionetworks.repo.model.Folder'],
'entity.id': 'syn5570914',
'entity.name': 'flapdoodle'
}
assert is_container(result)
result = {
'File.concreteType': ['org.sagebionetworks.repo.model.FileEntity'],
'File.id': 'syn5570914',
'File.name': 'flapdoodle'
}
assert not is_container(result)
assert is_container(Folder("Stuff", parentId="syn12345"))
assert is_container(Project("My Project", parentId="syn12345"))
assert not is_container(File("asdf.png", parentId="syn12345"))
def test_external_s3_upload():
# setup
project_id, storage_location_id = _set_up_external_s3_project()
# create a temporary file for upload
temp_file_path = utils.make_bogus_data_file()
expected_md5 = utils.md5_for_file(temp_file_path).hexdigest()
schedule_for_cleanup(temp_file_path)
# upload the file
uploaded_syn_file = syn.store(File(path=temp_file_path, parent=project_id))
# get file_handle of the uploaded file
file_handle = syn.restGET('/entity/%s/filehandles' %
uploaded_syn_file.id)['list'][0]
# Verify correct file handle type
assert_equals(file_handle['concreteType'],
'org.sagebionetworks.repo.model.file.S3FileHandle')
# Verify storage location id to make sure it's using external S3
assert_equals(storage_location_id, file_handle['storageLocationId'])
# Verify md5 of upload
assert_equals(expected_md5, file_handle['contentMd5'])
# clear the cache and download the file
syn.cache.purge(time.time())
downloaded_syn_file = syn.get(uploaded_syn_file.id)
# verify the correct file was downloaded
assert_equals(os.path.basename(downloaded_syn_file['path']),
os.path.basename(temp_file_path))
assert_not_equal(os.path.normpath(temp_file_path),
os.path.normpath(downloaded_syn_file['path']))
assert_true(filecmp.cmp(temp_file_path, downloaded_syn_file['path']))
def test_evaluations():
## create a new project
project = create_project()
name = 'Test Evaluation %s' % (str(uuid.uuid4()), )
try:
#Create evaluation
ev = Evaluation(name=name,
description='Evaluation for testing',
contentSource=project['id'],
status='CLOSED')
ev = syn.store(ev)
#Update evaluation
ev['status'] = 'OPEN'
ev = syn.store(ev, createOrUpdate=True)
assert ev.status == 'OPEN'
## add the current user as a participant
user = syn.getUserProfile()
syn.addEvaluationParticipant(ev, user['ownerId'])
## test getSubmissions with no submissions (SYNR-453)
submissions = syn.getSubmissions(ev)
assert len(list(submissions)) == 0
## increase this to fully test paging by getEvaluationSubmissions
num_of_submissions = 3
## create a bunch of entities and submit them for evaluation
sys.stdout.write('\ncreating evaluation submissions')
for i in range(num_of_submissions):
try:
(fd, filename) = tempfile.mkstemp()
with os.fdopen(fd, 'w') as f:
f.write(str(random.gauss(0, 1)))
f.write('\n')
f = File(filename,
parentId=project.id,
name='entry-%02d' % i,
description='An entry for testing evaluation')
entity = syn.store(f)
syn.submit(ev, entity)
finally:
os.remove(filename)
sys.stdout.write('.')
sys.stdout.flush()
## score the submissions
submissions = syn.getSubmissions(ev)
sys.stdout.write('\nscoring submissions')
for submission in submissions:
status = syn.getSubmissionStatus(submission)
status.score = random.random()
status.status = 'SCORED'
status.report = 'a fabulous effort!'
syn.store(status)
sys.stdout.write('.')
sys.stdout.flush()
sys.stdout.write('\n')
finally:
syn.delete(ev)
## make sure it's deleted
try:
ev = syn.getEvaluation(ev)
except Exception as e:
print e
assert e.response.status_code == 404
def test_evaluations():
# Create an Evaluation
name = 'Test Evaluation %s' % str(uuid.uuid4())
ev = Evaluation(name=name,
description='Evaluation for testing',
contentSource=project['id'],
status='CLOSED')
ev = syn.store(ev)
try:
# -- Get the Evaluation by name
evalNamed = syn.getEvaluationByName(name)
assert_equals(ev['contentSource'], evalNamed['contentSource'])
assert_equals(ev['createdOn'], evalNamed['createdOn'])
assert_equals(ev['description'], evalNamed['description'])
assert_equals(ev['etag'], evalNamed['etag'])
assert_equals(ev['id'], evalNamed['id'])
assert_equals(ev['name'], evalNamed['name'])
assert_equals(ev['ownerId'], evalNamed['ownerId'])
assert_equals(ev['status'], evalNamed['status'])
# -- Get the Evaluation by project
evalProj = syn.getEvaluationByContentSource(project)
evalProj = next(evalProj)
assert_equals(ev['contentSource'], evalProj['contentSource'])
assert_equals(ev['createdOn'], evalProj['createdOn'])
assert_equals(ev['description'], evalProj['description'])
assert_equals(ev['etag'], evalProj['etag'])
assert_equals(ev['id'], evalProj['id'])
assert_equals(ev['name'], evalProj['name'])
assert_equals(ev['ownerId'], evalProj['ownerId'])
assert_equals(ev['status'], evalProj['status'])
# Update the Evaluation
ev['status'] = 'OPEN'
ev = syn.store(ev, createOrUpdate=True)
assert_equals(ev.status, 'OPEN')
# Add the current user as a participant
myOwnerId = int(syn.getUserProfile()['ownerId'])
syn._allowParticipation(ev, myOwnerId)
# AUTHENTICATED_USERS = 273948
# PUBLIC = 273949
syn.setPermissions(ev, 273948, accessType=['READ'])
syn.setPermissions(ev, 273949, accessType=['READ'])
# test getPermissions
permissions = syn.getPermissions(ev, 273949)
assert_equals(['READ'], permissions)
permissions = syn.getPermissions(ev, syn.getUserProfile()['ownerId'])
for p in [
'READ', 'CREATE', 'DELETE', 'UPDATE', 'CHANGE_PERMISSIONS',
'READ_PRIVATE_SUBMISSION'
]:
assert_in(p, permissions)
# Test getSubmissions with no Submissions (SYNR-453)
submissions = syn.getSubmissions(ev)
assert_equals(len(list(submissions)), 0)
# Increase this to fully test paging by getEvaluationSubmissions
# not to be less than 2
num_of_submissions = 2
# Create a bunch of Entities and submit them for scoring
for i in range(num_of_submissions):
with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
filename = f.name
f.write(str(random.gauss(0, 1)) + '\n')
f = File(filename,
parentId=project.id,
name='entry-%02d' % i,
description='An entry for testing evaluation')
entity = syn.store(f)
syn.submit(ev,
entity,
name='Submission %02d' % i,
submitterAlias='My Team')
# Score the submissions
submissions = syn.getSubmissions(ev, limit=num_of_submissions - 1)
for submission in submissions:
assert_true(re.match('Submission \d+', submission['name']))
status = syn.getSubmissionStatus(submission)
status.score = random.random()
if submission['name'] == 'Submission 01':
status.status = 'INVALID'
status.report = 'Uh-oh, something went wrong!'
else:
status.status = 'SCORED'
status.report = 'a fabulous effort!'
syn.store(status)
# Annotate the submissions
bogosity = {}
submissions = syn.getSubmissions(ev)
b = 123
for submission, status in syn.getSubmissionBundles(ev):
bogosity[submission.id] = b
a = dict(foo='bar', bogosity=b)
b += 123
status['annotations'] = to_submission_status_annotations(a)
set_privacy(status['annotations'],
key='bogosity',
is_private=False)
syn.store(status)
# Test that the annotations stuck
for submission, status in syn.getSubmissionBundles(ev):
a = from_submission_status_annotations(status.annotations)
assert_equals(a['foo'], 'bar')
assert_equals(a['bogosity'], bogosity[submission.id])
for kvp in status.annotations['longAnnos']:
if kvp['key'] == 'bogosity':
assert_false(kvp['isPrivate'])
# test query by submission annotations
# These queries run against an eventually consistent index table which is
# populated by an asynchronous worker. Thus, the queries may remain out
# of sync for some unbounded, but assumed to be short time.
attempts = 2
while attempts > 0:
try:
results = syn.restGET(
"/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s"
% ev.id)
assert_equals(len(results['rows']), num_of_submissions + 1)
results = syn.restGET(
"/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s where bogosity > 200"
% ev.id)
assert_equals(len(results['rows']), num_of_submissions)
except AssertionError as ex1:
attempts -= 1
time.sleep(2)
else:
attempts = 0
# Test that we can retrieve submissions with a specific status
invalid_submissions = list(syn.getSubmissions(ev, status='INVALID'))
assert_equals(len(invalid_submissions), 1, len(invalid_submissions))
assert_equals(invalid_submissions[0]['name'], 'Submission 01')
finally:
# Clean up
syn.delete(ev)
if 'testSyn' in locals():
if 'other_project' in locals():
# Clean up, since the current user can't access this project
# This also removes references to the submitted object :)
testSyn.delete(other_project)
if 'team' in locals():
# remove team
testSyn.delete(team)
# Just deleted it. Shouldn't be able to get it.
assert_raises(SynapseHTTPError, syn.getEvaluation, ev)
def test_evaluations():
# Create an Evaluation
name = 'Test Evaluation %s' % str(uuid.uuid4())
ev = Evaluation(name=name, description='Evaluation for testing',
contentSource=project['id'], status='CLOSED')
ev = syn.store(ev)
try:
# -- Get the Evaluation by name
evalNamed = syn.getEvaluationByName(name)
assert ev['contentSource'] == evalNamed['contentSource']
assert ev['createdOn'] == evalNamed['createdOn']
assert ev['description'] == evalNamed['description']
assert ev['etag'] == evalNamed['etag']
assert ev['id'] == evalNamed['id']
assert ev['name'] == evalNamed['name']
assert ev['ownerId'] == evalNamed['ownerId']
assert ev['status'] == evalNamed['status']
# -- Get the Evaluation by project
evalProj = syn.getEvaluationByContentSource(project)
evalProj = next(evalProj)
assert ev['contentSource'] == evalProj['contentSource']
assert ev['createdOn'] == evalProj['createdOn']
assert ev['description'] == evalProj['description']
assert ev['etag'] == evalProj['etag']
assert ev['id'] == evalProj['id']
assert ev['name'] == evalProj['name']
assert ev['ownerId'] == evalProj['ownerId']
assert ev['status'] == evalProj['status']
# Update the Evaluation
ev['status'] = 'OPEN'
ev = syn.store(ev, createOrUpdate=True)
assert ev.status == 'OPEN'
# # Add the current user as a participant
myOwnerId = int(syn.getUserProfile()['ownerId'])
syn._allowParticipation(ev, myOwnerId)
# AUTHENTICATED_USERS = 273948
# PUBLIC = 273949
syn.setPermissions(ev, 273948, accessType=['READ'])
syn.setPermissions(ev, 273949, accessType=['READ'])
# test getPermissions
permissions = syn.getPermissions(ev, 273949)
assert ['READ'] == permissions
permissions = syn.getPermissions(ev, syn.getUserProfile()['ownerId'])
assert [p in permissions for p in ['READ', 'CREATE', 'DELETE', 'UPDATE', 'CHANGE_PERMISSIONS', 'READ_PRIVATE_SUBMISSION']]
# Test getSubmissions with no Submissions (SYNR-453)
submissions = syn.getSubmissions(ev)
assert len(list(submissions)) == 0
# -- Get a Submission attachment belonging to another user (SYNR-541) --
# See if the configuration contains test authentication
try:
config = configparser.ConfigParser()
config.read(client.CONFIG_FILE)
other_user = {}
other_user['username'] = config.get('test-authentication', 'username')
other_user['password'] = config.get('test-authentication', 'password')
print("Testing SYNR-541")
# Login as the test user
testSyn = client.Synapse(skip_checks=True)
testSyn.login(email=other_user['username'], password=other_user['password'])
testOwnerId = int(testSyn.getUserProfile()['ownerId'])
# Make a project
other_project = Project(name=str(uuid.uuid4()))
other_project = testSyn.createEntity(other_project)
# Give the test user permission to read and join the evaluation
syn._allowParticipation(ev, testOwnerId)
# Make a file to submit
with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
filename = f.name
f.write(str(random.gauss(0,1)) + '\n')
f = File(filename, parentId=other_project.id,
name='Submission 999',
description ="Haha! I'm inaccessible...")
entity = testSyn.store(f)
## test submission by evaluation ID
submission = testSyn.submit(ev.id, entity, submitterAlias="My Nickname")
# Mess up the cached file so that syn._getWithEntityBundle must download again
os.utime(filename, (0, 0))
# Grab the Submission as the original user
fetched = syn.getSubmission(submission['id'])
assert os.path.exists(fetched['filePath'])
# make sure the fetched file is the same as the original (PLFM-2666)
assert filecmp.cmp(filename, fetched['filePath'])
except configparser.Error:
print('Skipping test for SYNR-541: No [test-authentication] in %s' % client.CONFIG_FILE)
# Increase this to fully test paging by getEvaluationSubmissions
# not to be less than 2
num_of_submissions = 2
# Create a bunch of Entities and submit them for scoring
print("Creating Submissions")
for i in range(num_of_submissions):
with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
filename = f.name
f.write(str(random.gauss(0,1)) + '\n')
f = File(filename, parentId=project.id, name='entry-%02d' % i,
description='An entry for testing evaluation')
entity=syn.store(f)
syn.submit(ev, entity, name='Submission %02d' % i, submitterAlias='My Team')
# Score the submissions
submissions = syn.getSubmissions(ev, limit=num_of_submissions-1)
print("Scoring Submissions")
for submission in submissions:
assert re.match('Submission \d+', submission['name'])
status = syn.getSubmissionStatus(submission)
status.score = random.random()
if submission['name'] == 'Submission 01':
status.status = 'INVALID'
status.report = 'Uh-oh, something went wrong!'
else:
status.status = 'SCORED'
status.report = 'a fabulous effort!'
syn.store(status)
# Annotate the submissions
print("Annotating Submissions")
bogosity = {}
submissions = syn.getSubmissions(ev)
b = 123
for submission, status in syn.getSubmissionBundles(ev):
bogosity[submission.id] = b
a = dict(foo='bar', bogosity=b)
b += 123
status['annotations'] = to_submission_status_annotations(a)
set_privacy(status['annotations'], key='bogosity', is_private=False)
syn.store(status)
# Test that the annotations stuck
for submission, status in syn.getSubmissionBundles(ev):
a = from_submission_status_annotations(status.annotations)
assert a['foo'] == 'bar'
assert a['bogosity'] == bogosity[submission.id]
for kvp in status.annotations['longAnnos']:
if kvp['key'] == 'bogosity':
assert kvp['isPrivate'] == False
# test query by submission annotations
# These queries run against an eventually consistent index table which is
# populated by an asynchronous worker. Thus, the queries may remain out
# of sync for some unbounded, but assumed to be short time.
attempts = 2
while attempts > 0:
try:
print("Querying for submissions")
results = syn.restGET("/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s" % ev.id)
print(results)
assert results[u'totalNumberOfResults'] == num_of_submissions+1
results = syn.restGET("/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s where bogosity > 200" % ev.id)
print(results)
assert results[u'totalNumberOfResults'] == num_of_submissions
except AssertionError as ex1:
print("failed query: ", ex1)
attempts -= 1
if attempts > 0: print("retrying...")
time.sleep(2)
else:
attempts = 0
## Test that we can retrieve submissions with a specific status
invalid_submissions = list(syn.getSubmissions(ev, status='INVALID'))
assert len(invalid_submissions) == 1, len(invalid_submissions)
assert invalid_submissions[0]['name'] == 'Submission 01'
finally:
# Clean up
syn.delete(ev)
if 'testSyn' in locals():
if 'other_project' in locals():
# Clean up, since the current user can't access this project
# This also removes references to the submitted object :)
testSyn.delete(other_project)
if 'team' in locals():
## remove team
testSyn.delete(team)
## Just deleted it. Shouldn't be able to get it.
assert_raises(SynapseHTTPError, syn.getEvaluation, ev)
def test_evaluations():
# Create an Evaluation
name = 'Test Evaluation %s' % str(uuid.uuid4())
ev = Evaluation(name=name,
description='Evaluation for testing',
contentSource=project['id'],
status='CLOSED')
ev = syn.store(ev)
try:
# -- Get the Evaluation by name
evalNamed = syn.getEvaluationByName(name)
assert ev['contentSource'] == evalNamed['contentSource']
assert ev['createdOn'] == evalNamed['createdOn']
assert ev['description'] == evalNamed['description']
assert ev['etag'] == evalNamed['etag']
assert ev['id'] == evalNamed['id']
assert ev['name'] == evalNamed['name']
assert ev['ownerId'] == evalNamed['ownerId']
assert ev['status'] == evalNamed['status']
# -- Get the Evaluation by project
evalProj = syn.getEvaluationByContentSource(project)
evalProj = next(evalProj)
assert ev['contentSource'] == evalProj['contentSource']
assert ev['createdOn'] == evalProj['createdOn']
assert ev['description'] == evalProj['description']
assert ev['etag'] == evalProj['etag']
assert ev['id'] == evalProj['id']
assert ev['name'] == evalProj['name']
assert ev['ownerId'] == evalProj['ownerId']
assert ev['status'] == evalProj['status']
# Update the Evaluation
ev['status'] = 'OPEN'
ev = syn.store(ev, createOrUpdate=True)
assert ev.status == 'OPEN'
# # Add the current user as a participant
myOwnerId = int(syn.getUserProfile()['ownerId'])
syn._allowParticipation(ev, myOwnerId)
# AUTHENTICATED_USERS = 273948
# PUBLIC = 273949
syn.setPermissions(ev, 273948, accessType=['READ'])
syn.setPermissions(ev, 273949, accessType=['READ'])
# test getPermissions
permissions = syn.getPermissions(ev, 273949)
assert ['READ'] == permissions
permissions = syn.getPermissions(ev, syn.getUserProfile()['ownerId'])
assert [
p in permissions for p in [
'READ', 'CREATE', 'DELETE', 'UPDATE', 'CHANGE_PERMISSIONS',
'READ_PRIVATE_SUBMISSION'
]
]
# Test getSubmissions with no Submissions (SYNR-453)
submissions = syn.getSubmissions(ev)
assert len(list(submissions)) == 0
# -- Get a Submission attachment belonging to another user (SYNR-541) --
# See if the configuration contains test authentication
if other_user['username']:
print("Testing SYNR-541")
# Login as the test user
testSyn = client.Synapse(skip_checks=True)
testSyn.login(email=other_user['username'],
password=other_user['password'])
testOwnerId = int(testSyn.getUserProfile()['ownerId'])
# Make a project
other_project = Project(name=str(uuid.uuid4()))
other_project = testSyn.createEntity(other_project)
# Give the test user permission to read and join the evaluation
syn._allowParticipation(ev, testOwnerId)
# Make a file to submit
with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
filename = f.name
f.write(str(random.gauss(0, 1)) + '\n')
f = File(filename,
parentId=other_project.id,
name='Submission 999',
description="Haha! I'm inaccessible...")
entity = testSyn.store(f)
## test submission by evaluation ID
submission = testSyn.submit(ev.id,
entity,
submitterAlias="My Nickname")
# Mess up the cached file so that syn._getWithEntityBundle must download again
os.utime(filename, (0, 0))
# Grab the Submission as the original user
fetched = syn.getSubmission(submission['id'])
assert os.path.exists(fetched['filePath'])
# make sure the fetched file is the same as the original (PLFM-2666)
assert filecmp.cmp(filename, fetched['filePath'])
else:
print(
'Skipping test for SYNR-541: No [test-authentication] in %s' %
client.CONFIG_FILE)
# Increase this to fully test paging by getEvaluationSubmissions
# not to be less than 2
num_of_submissions = 2
# Create a bunch of Entities and submit them for scoring
print("Creating Submissions")
for i in range(num_of_submissions):
with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
filename = f.name
f.write(str(random.gauss(0, 1)) + '\n')
f = File(filename,
parentId=project.id,
name='entry-%02d' % i,
description='An entry for testing evaluation')
entity = syn.store(f)
syn.submit(ev,
entity,
name='Submission %02d' % i,
submitterAlias='My Team')
# Score the submissions
submissions = syn.getSubmissions(ev, limit=num_of_submissions - 1)
print("Scoring Submissions")
for submission in submissions:
assert re.match('Submission \d+', submission['name'])
status = syn.getSubmissionStatus(submission)
status.score = random.random()
if submission['name'] == 'Submission 01':
status.status = 'INVALID'
status.report = 'Uh-oh, something went wrong!'
else:
status.status = 'SCORED'
status.report = 'a fabulous effort!'
syn.store(status)
# Annotate the submissions
print("Annotating Submissions")
bogosity = {}
submissions = syn.getSubmissions(ev)
b = 123
for submission, status in syn.getSubmissionBundles(ev):
bogosity[submission.id] = b
a = dict(foo='bar', bogosity=b)
b += 123
status['annotations'] = to_submission_status_annotations(a)
set_privacy(status['annotations'],
key='bogosity',
is_private=False)
syn.store(status)
# Test that the annotations stuck
for submission, status in syn.getSubmissionBundles(ev):
a = from_submission_status_annotations(status.annotations)
assert a['foo'] == 'bar'
assert a['bogosity'] == bogosity[submission.id]
for kvp in status.annotations['longAnnos']:
if kvp['key'] == 'bogosity':
assert kvp['isPrivate'] == False
# test query by submission annotations
# These queries run against an eventually consistent index table which is
# populated by an asynchronous worker. Thus, the queries may remain out
# of sync for some unbounded, but assumed to be short time.
attempts = 2
while attempts > 0:
try:
print("Querying for submissions")
results = syn.restGET(
"/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s"
% ev.id)
print(results)
assert len(results['rows']) == num_of_submissions + 1
results = syn.restGET(
"/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s where bogosity > 200"
% ev.id)
print(results)
assert len(results['rows']) == num_of_submissions
except AssertionError as ex1:
print("failed query: ", ex1)
attempts -= 1
if attempts > 0: print("retrying...")
time.sleep(2)
else:
attempts = 0
## Test that we can retrieve submissions with a specific status
invalid_submissions = list(syn.getSubmissions(ev, status='INVALID'))
assert len(invalid_submissions) == 1, len(invalid_submissions)
assert invalid_submissions[0]['name'] == 'Submission 01'
finally:
# Clean up
syn.delete(ev)
if 'testSyn' in locals():
if 'other_project' in locals():
# Clean up, since the current user can't access this project
# This also removes references to the submitted object :)
testSyn.delete(other_project)
if 'team' in locals():
## remove team
testSyn.delete(team)
## Just deleted it. Shouldn't be able to get it.
assert_raises(SynapseHTTPError, syn.getEvaluation, ev)
def test_concrete_type():
f1 = File('http://en.wikipedia.org/wiki/File:Nettlebed_cave.jpg',
name='Nettlebed Cave',
parent='syn1234567',
synapseStore=False)
assert f1.concreteType == 'org.sagebionetworks.repo.model.FileEntity'
def test_attrs():
f = File('foo.xyz', parent='syn1234', foo='bar')
assert hasattr(f, 'parentId')
assert hasattr(f, 'foo')
assert hasattr(f, 'path')
