def testMoveProject():
proj1 = syn.store(Project(name=str(uuid.uuid4()) +
"testMoveProject-child"))
proj2 = syn.store(
Project(name=str(uuid.uuid4()) + "testMoveProject-newParent"))
assert_raises(SynapseHTTPError, syn.move, proj1, proj2)
schedule_for_cleanup(proj1)
schedule_for_cleanup(proj2)
def _set_up_external_s3_project():
"""
creates a project and links it to an external s3 storage
:return: synapse id of the created project, and storageLocationId of the project
"""
EXTERNAL_S3_BUCKET = 'python-client-integration-test.sagebase.org'
project_ext_s3 = syn.store(Project(name=str(uuid.uuid4())))
destination = {
'uploadType': 'S3',
'concreteType':
'org.sagebionetworks.repo.model.project.ExternalS3StorageLocationSetting',
'bucket': EXTERNAL_S3_BUCKET
}
destination = syn.restPOST('/storageLocation',
body=json.dumps(destination))
project_destination = {
'concreteType':
'org.sagebionetworks.repo.model.project.UploadDestinationListSetting',
'settingsType': 'upload',
'locations': [destination['storageLocationId']],
'projectId': project_ext_s3.id
}
syn.restPOST('/projectSettings', body=json.dumps(project_destination))
schedule_for_cleanup(project_ext_s3)
return project_ext_s3.id, destination['storageLocationId']
def test_getChildren():
# setup a hierarchy for folders
# PROJECT
# | \
# File Folder
# |
# File
project_name = str(uuid.uuid1())
test_project = syn.store(Project(name=project_name))
folder = syn.store(Folder(name="firstFolder", parent=test_project))
syn.store(
File(path="~/doesntMatter.txt",
name="file inside folders",
parent=folder,
synapseStore=False))
project_file = syn.store(
File(path="~/doesntMatterAgain.txt",
name="file inside project",
parent=test_project,
synapseStore=False))
schedule_for_cleanup(test_project)
expected_id_set = {project_file.id, folder.id}
children_id_set = {x['id'] for x in syn.getChildren(test_project.id)}
assert_equals(expected_id_set, children_id_set)
def test_ExternalObjectStore_roundtrip():
endpoint = "https://s3.amazonaws.com"
bucket = "test-client-auth-s3"
profile_name = syn._get_client_authenticated_s3_profile(endpoint, bucket)
if profile_name != 'client-auth-s3-test':
raise SkipTest(
"This test only works on travis because it requires AWS credentials to a specific S3 bucket"
)
proj = syn.store(
Project(name=str(uuid.uuid4()) + "ExternalObjStoreProject"))
schedule_for_cleanup(proj)
storage_location = syn.createStorageLocationSetting(
"ExternalObjectStorage", endpointUrl=endpoint, bucket=bucket)
syn.setStorageLocation(proj, storage_location['storageLocationId'])
file_path = utils.make_bogus_data_file()
file_entity = File(file_path, name="TestName", parent=proj)
file_entity = syn.store(file_entity)
syn.cache.purge(time.time())
assert_is_none(syn.cache.get(file_entity['dataFileHandleId']))
# verify key is in s3
import boto3
boto_session = boto3.session.Session(profile_name=profile_name)
s3 = boto_session.resource('s3', endpoint_url=endpoint)
try:
s3_file = s3.Object(file_entity._file_handle.bucket,
file_entity._file_handle.fileKey)
s3_file.load()
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == "404":
raise Exception("The file was not uploaded to S3")
file_entity_downloaded = syn.get(file_entity['id'])
file_handle = file_entity_downloaded['_file_handle']
# verify file_handle metadata
assert_equals(endpoint, file_handle['endpointUrl'])
assert_equals(bucket, file_handle['bucket'])
assert_equals(
utils.md5_for_file(file_path).hexdigest(), file_handle['contentMd5'])
assert_equals(os.stat(file_path).st_size, file_handle['contentSize'])
assert_equals('text/plain', file_handle['contentType'])
assert_not_equal(utils.normalize_path(file_path),
utils.normalize_path(file_entity_downloaded['path']))
assert_true(filecmp.cmp(file_path, file_entity_downloaded['path']))
# clean up
s3_file.delete()
def test_entity_constructors():
project = Project(name=str(uuid.uuid4()), description='Testing 123')
folder = Folder(name='Musicians',
parent=project,
genre='Jazz',
datatype='personnel')
personnel_file = File(fname,
parentId=folder.id,
group='Miles Davis Quintet',
album='Stockholm 1960 Complete')
def testSetStorageLocation():
proj = syn.store(
Project(name=str(uuid.uuid4()) +
"testSetStorageLocation__existing_storage_location"))
schedule_for_cleanup(proj)
endpoint = "https://url.doesnt.matter.com"
bucket = "fake-bucket-name"
storage_location = syn.createStorageLocationSetting(
"ExternalObjectStorage", endpointUrl=endpoint, bucket=bucket)
storage_setting = syn.setStorageLocation(
proj, storage_location['storageLocationId'])
retrieved_setting = syn.getProjectSetting(proj, 'upload')
assert_equals(storage_setting, retrieved_setting)
def test_entity_constructors():
project = Project('TestProject', id='syn1001', foo='bar')
assert project.name == 'TestProject'
assert project['foo'] == 'bar'
folder = Folder('MyFolder', parent=project, foo='bat', id='syn1002')
assert folder.name == 'MyFolder'
assert folder.foo == 'bat'
assert folder.parentId == 'syn1001'
a_file = File('/path/to/fabulous_things.zzz', parent=folder, foo='biz')
#assert a_file.name == 'fabulous_things.zzz'
assert a_file.entityType == 'org.sagebionetworks.repo.model.FileEntity'
assert a_file.path == '/path/to/fabulous_things.zzz'
assert a_file.foo == 'biz'
assert a_file.parentId == 'syn1002'
def test_entity_constructors():
project = Project('TestProject', id='syn1001', foo='bar')
assert project.name == 'TestProject'
assert project['foo'] == 'bar'
folder = Folder('MyFolder', parent=project, foo='bat', id='syn1002')
assert folder.name == 'MyFolder'
assert folder.foo == 'bat'
assert folder.parentId == 'syn1001'
a_file = File('/path/to/fabulous_things.zzz',
parent=folder,
foo='biz',
contentType='application/cattywampus')
#assert a_file.name == 'fabulous_things.zzz'
assert a_file.concreteType == 'org.sagebionetworks.repo.model.FileEntity'
assert a_file.path == '/path/to/fabulous_things.zzz'
assert a_file.foo == 'biz'
assert a_file.parentId == 'syn1002'
assert a_file.contentType == 'application/cattywampus'
assert 'contentType' in a_file.__dict__
def test_entity_constructors():
project = Project('TestProject', id='syn1001', foo='bar')
assert_equals(project.name, 'TestProject')
assert_equals(project['foo'], 'bar')
folder = Folder('MyFolder', parent=project, foo='bat', id='syn1002')
assert_equals(folder.name, 'MyFolder')
assert_equals(folder.foo, 'bat')
assert_equals(folder.parentId, 'syn1001')
a_file = File('/path/to/fabulous_things.zzz',
parent=folder,
foo='biz',
contentType='application/cattywampus')
assert_equals(a_file.concreteType,
'org.sagebionetworks.repo.model.FileEntity')
assert_equals(a_file.path, '/path/to/fabulous_things.zzz')
assert_equals(a_file.foo, 'biz')
assert_equals(a_file.parentId, 'syn1002')
assert_equals(a_file.contentType, 'application/cattywampus')
assert_in('contentType', a_file._file_handle)
def test_is_container():
## result from a Synapse entity annotation query
## Note: prefix may be capitalized or not, depending on the from clause of the query
result = {
'entity.versionNumber': 1,
'entity.nodeType': 'project',
'entity.concreteType': ['org.sagebionetworks.repo.model.Project'],
'entity.createdOn': 1451512703905,
'entity.id': 'syn5570912',
'entity.name': 'blah'
}
assert is_container(result)
result = {
'Entity.nodeType': 'project',
'Entity.id': 'syn5570912',
'Entity.name': 'blah'
}
assert is_container(result)
result = {
'entity.concreteType': ['org.sagebionetworks.repo.model.Folder'],
'entity.id': 'syn5570914',
'entity.name': 'flapdoodle'
}
assert is_container(result)
result = {
'File.concreteType': ['org.sagebionetworks.repo.model.FileEntity'],
'File.id': 'syn5570914',
'File.name': 'flapdoodle'
}
assert not is_container(result)
assert is_container(Folder("Stuff", parentId="syn12345"))
assert is_container(Project("My Project", parentId="syn12345"))
assert not is_container(File("asdf.png", parentId="syn12345"))
def test_evaluations():
# Create an Evaluation
name = 'Test Evaluation %s' % str(uuid.uuid4())
ev = Evaluation(name=name,
description='Evaluation for testing',
contentSource=project['id'],
status='CLOSED')
ev = syn.store(ev)
try:
# -- Get the Evaluation by name
evalNamed = syn.getEvaluationByName(name)
assert ev['contentSource'] == evalNamed['contentSource']
assert ev['createdOn'] == evalNamed['createdOn']
assert ev['description'] == evalNamed['description']
assert ev['etag'] == evalNamed['etag']
assert ev['id'] == evalNamed['id']
assert ev['name'] == evalNamed['name']
assert ev['ownerId'] == evalNamed['ownerId']
assert ev['status'] == evalNamed['status']
# -- Get the Evaluation by project
evalProj = syn.getEvaluationByContentSource(project)
evalProj = next(evalProj)
assert ev['contentSource'] == evalProj['contentSource']
assert ev['createdOn'] == evalProj['createdOn']
assert ev['description'] == evalProj['description']
assert ev['etag'] == evalProj['etag']
assert ev['id'] == evalProj['id']
assert ev['name'] == evalProj['name']
assert ev['ownerId'] == evalProj['ownerId']
assert ev['status'] == evalProj['status']
# Update the Evaluation
ev['status'] = 'OPEN'
ev = syn.store(ev, createOrUpdate=True)
assert ev.status == 'OPEN'
# # Add the current user as a participant
myOwnerId = int(syn.getUserProfile()['ownerId'])
syn._allowParticipation(ev, myOwnerId)
# AUTHENTICATED_USERS = 273948
# PUBLIC = 273949
syn.setPermissions(ev, 273948, accessType=['READ'])
syn.setPermissions(ev, 273949, accessType=['READ'])
# test getPermissions
permissions = syn.getPermissions(ev, 273949)
assert ['READ'] == permissions
permissions = syn.getPermissions(ev, syn.getUserProfile()['ownerId'])
assert [
p in permissions for p in [
'READ', 'CREATE', 'DELETE', 'UPDATE', 'CHANGE_PERMISSIONS',
'READ_PRIVATE_SUBMISSION'
]
]
# Test getSubmissions with no Submissions (SYNR-453)
submissions = syn.getSubmissions(ev)
assert len(list(submissions)) == 0
# -- Get a Submission attachment belonging to another user (SYNR-541) --
# See if the configuration contains test authentication
if other_user['username']:
print("Testing SYNR-541")
# Login as the test user
testSyn = client.Synapse(skip_checks=True)
testSyn.login(email=other_user['username'],
password=other_user['password'])
testOwnerId = int(testSyn.getUserProfile()['ownerId'])
# Make a project
other_project = Project(name=str(uuid.uuid4()))
other_project = testSyn.createEntity(other_project)
# Give the test user permission to read and join the evaluation
syn._allowParticipation(ev, testOwnerId)
# Make a file to submit
with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
filename = f.name
f.write(str(random.gauss(0, 1)) + '\n')
f = File(filename,
parentId=other_project.id,
name='Submission 999',
description="Haha! I'm inaccessible...")
entity = testSyn.store(f)
## test submission by evaluation ID
submission = testSyn.submit(ev.id,
entity,
submitterAlias="My Nickname")
# Mess up the cached file so that syn._getWithEntityBundle must download again
os.utime(filename, (0, 0))
# Grab the Submission as the original user
fetched = syn.getSubmission(submission['id'])
assert os.path.exists(fetched['filePath'])
# make sure the fetched file is the same as the original (PLFM-2666)
assert filecmp.cmp(filename, fetched['filePath'])
else:
print(
'Skipping test for SYNR-541: No [test-authentication] in %s' %
client.CONFIG_FILE)
# Increase this to fully test paging by getEvaluationSubmissions
# not to be less than 2
num_of_submissions = 2
# Create a bunch of Entities and submit them for scoring
print("Creating Submissions")
for i in range(num_of_submissions):
with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
filename = f.name
f.write(str(random.gauss(0, 1)) + '\n')
f = File(filename,
parentId=project.id,
name='entry-%02d' % i,
description='An entry for testing evaluation')
entity = syn.store(f)
syn.submit(ev,
entity,
name='Submission %02d' % i,
submitterAlias='My Team')
# Score the submissions
submissions = syn.getSubmissions(ev, limit=num_of_submissions - 1)
print("Scoring Submissions")
for submission in submissions:
assert re.match('Submission \d+', submission['name'])
status = syn.getSubmissionStatus(submission)
status.score = random.random()
if submission['name'] == 'Submission 01':
status.status = 'INVALID'
status.report = 'Uh-oh, something went wrong!'
else:
status.status = 'SCORED'
status.report = 'a fabulous effort!'
syn.store(status)
# Annotate the submissions
print("Annotating Submissions")
bogosity = {}
submissions = syn.getSubmissions(ev)
b = 123
for submission, status in syn.getSubmissionBundles(ev):
bogosity[submission.id] = b
a = dict(foo='bar', bogosity=b)
b += 123
status['annotations'] = to_submission_status_annotations(a)
set_privacy(status['annotations'],
key='bogosity',
is_private=False)
syn.store(status)
# Test that the annotations stuck
for submission, status in syn.getSubmissionBundles(ev):
a = from_submission_status_annotations(status.annotations)
assert a['foo'] == 'bar'
assert a['bogosity'] == bogosity[submission.id]
for kvp in status.annotations['longAnnos']:
if kvp['key'] == 'bogosity':
assert kvp['isPrivate'] == False
# test query by submission annotations
# These queries run against an eventually consistent index table which is
# populated by an asynchronous worker. Thus, the queries may remain out
# of sync for some unbounded, but assumed to be short time.
attempts = 2
while attempts > 0:
try:
print("Querying for submissions")
results = syn.restGET(
"/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s"
% ev.id)
print(results)
assert len(results['rows']) == num_of_submissions + 1
results = syn.restGET(
"/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s where bogosity > 200"
% ev.id)
print(results)
assert len(results['rows']) == num_of_submissions
except AssertionError as ex1:
print("failed query: ", ex1)
attempts -= 1
if attempts > 0: print("retrying...")
time.sleep(2)
else:
attempts = 0
## Test that we can retrieve submissions with a specific status
invalid_submissions = list(syn.getSubmissions(ev, status='INVALID'))
assert len(invalid_submissions) == 1, len(invalid_submissions)
assert invalid_submissions[0]['name'] == 'Submission 01'
finally:
# Clean up
syn.delete(ev)
if 'testSyn' in locals():
if 'other_project' in locals():
# Clean up, since the current user can't access this project
# This also removes references to the submitted object :)
testSyn.delete(other_project)
if 'team' in locals():
## remove team
testSyn.delete(team)
## Just deleted it. Shouldn't be able to get it.
assert_raises(SynapseHTTPError, syn.getEvaluation, ev)
def test_findEntityIdByNameAndParent():
project_name = str(uuid.uuid1())
project_id = syn.store(Project(name=project_name))['id']
assert_equals(project_id, syn.findEntityId(project_name))