def runTest(self):
self.make_mount(fuse.ProjectDirectory,
project_object=self.api.users().current().execute())
d1 = llfuse.listdir(self.mounttmp)
self.assertIn('Unrestricted public data', d1)
d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
public_project = run_test_server.fixture('groups')[
'anonymously_accessible_project']
found_in = 0
found_not_in = 0
for name, item in run_test_server.fixture('collections').iteritems():
if 'name' not in item:
pass
elif item['owner_uuid'] == public_project['uuid']:
self.assertIn(item['name'], d2)
found_in += 1
else:
# Artificial assumption here: there is no public
# collection fixture with the same name as a
# non-public collection.
self.assertNotIn(item['name'], d2)
found_not_in += 1
self.assertNotEqual(0, found_in)
self.assertNotEqual(0, found_not_in)
d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
def runTest(self):
self.make_mount(fuse.ProjectDirectory,
project_object=self.api.users().current().execute())
d1 = llfuse.listdir(self.mounttmp)
self.assertIn('Unrestricted public data', d1)
d2 = llfuse.listdir(
os.path.join(self.mounttmp, 'Unrestricted public data'))
public_project = run_test_server.fixture(
'groups')['anonymously_accessible_project']
found_in = 0
found_not_in = 0
for name, item in run_test_server.fixture('collections').iteritems():
if 'name' not in item:
pass
elif item['owner_uuid'] == public_project['uuid']:
self.assertIn(item['name'], d2)
found_in += 1
else:
# Artificial assumption here: there is no public
# collection fixture with the same name as a
# non-public collection.
self.assertNotIn(item['name'], d2)
found_not_in += 1
self.assertNotEqual(0, found_in)
self.assertNotEqual(0, found_not_in)
d3 = llfuse.listdir(
os.path.join(self.mounttmp, 'Unrestricted public data',
'GNU General Public License, version 3'))
self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
def test_mutually_exclusive_args(self):
cid = run_test_server.fixture('collections')['public_text_file']['uuid']
gid = run_test_server.fixture('groups')['aproject']['uuid']
for badargs in [
['--mount-tmp', 'foo', '--collection', cid],
['--mount-tmp', 'foo', '--project', gid],
['--collection', cid, '--project', gid],
['--by-id', '--project', gid],
['--mount-tmp', 'foo', '--by-id'],
]:
with nostderr():
with self.assertRaises(SystemExit):
args = arvados_fuse.command.ArgumentParser().parse_args(
badargs + ['--foreground', self.mntdir])
arvados_fuse.command.Mount(args)
def test_project(self):
uuid = run_test_server.fixture('groups')['aproject']['uuid']
args = arvados_fuse.command.ArgumentParser().parse_args(
['--project', uuid, '--foreground', self.mntdir])
self.mnt = arvados_fuse.command.Mount(args)
e = self.check_ent_type(arvados_fuse.ProjectDirectory)
self.assertEqual(e.project_object['uuid'], uuid)
def setUp(self, api=None):
super(FuseMagicTest, self).setUp(api=api)
self.test_project = run_test_server.fixture('groups')['aproject']['uuid']
self.non_project_group = run_test_server.fixture('groups')['public']['uuid']
self.collection_in_test_project = run_test_server.fixture('collections')['foo_collection_in_aproject']['name']
cw = arvados.CollectionWriter()
cw.start_new_file('thing1.txt')
cw.write("data 1")
self.testcollection = cw.finish()
self.test_manifest = cw.manifest_text()
coll = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
self.test_manifest_pdh = coll['portable_data_hash']
def setUp(self, api=None):
super(FuseMagicTest, self).setUp(api=api)
self.test_project = run_test_server.fixture('groups')['aproject']['uuid']
self.non_project_group = run_test_server.fixture('groups')['public']['uuid']
self.collection_in_test_project = run_test_server.fixture('collections')['foo_collection_in_aproject']['name']
cw = arvados.CollectionWriter()
cw.start_new_file('thing1.txt')
cw.write("data 1")
self.testcollection = cw.finish()
self.test_manifest = cw.manifest_text()
coll = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
self.test_manifest_pdh = coll['portable_data_hash']
class CollectionTestMixin(object):
PROXY_RESPONSE = {
'items_available': 1,
'items': [{
'uuid': 'zzzzz-bi6l4-mockproxy012345',
'owner_uuid': 'zzzzz-tpzed-mockowner012345',
'service_host': tutil.TEST_HOST,
'service_port': 65535,
'service_ssl_flag': True,
'service_type': 'proxy',
}]}
API_COLLECTIONS = run_test_server.fixture('collections')
DEFAULT_COLLECTION = API_COLLECTIONS['foo_file']
DEFAULT_DATA_HASH = DEFAULT_COLLECTION['portable_data_hash']
DEFAULT_MANIFEST = DEFAULT_COLLECTION['manifest_text']
DEFAULT_UUID = DEFAULT_COLLECTION['uuid']
def _mock_api_call(self, mock_method, code, body):
mock_method = mock_method().execute
if code == 200:
mock_method.return_value = body
else:
mock_method.side_effect = arvados.errors.ApiError(
tutil.fake_httplib2_response(code), "{}")
def mock_keep_services(self, api_mock, code, body):
self._mock_api_call(api_mock.keep_services().accessible, code, body)
def api_client_mock(self, code=200):
client = mock.MagicMock(name='api_client')
self.mock_keep_services(client, code, self.PROXY_RESPONSE)
return client
class StreamRetryTestMixin(object):
# Define reader_for(coll_name, **kwargs)
# and read_for_test(reader, size, **kwargs).
API_COLLECTIONS = run_test_server.fixture('collections')
def keep_client(self):
return arvados.KeepClient(proxy='http://[%s]:1' % (tutil.TEST_HOST, ),
local_store='')
def manifest_for(self, coll_name):
return self.API_COLLECTIONS[coll_name]['manifest_text']
@tutil.skip_sleep
def test_success_without_retries(self):
reader = self.reader_for('bar_file')
with tutil.mock_responses('bar', 200):
self.assertEqual('bar', self.read_for_test(reader, 3))
@tutil.skip_sleep
def test_read_no_default_retry(self):
reader = self.reader_for('user_agreement')
with tutil.mock_responses('', 500):
with self.assertRaises(arvados.errors.KeepReadError):
self.read_for_test(reader, 10)
@tutil.skip_sleep
def test_read_with_instance_retries(self):
reader = self.reader_for('foo_file', num_retries=3)
with tutil.mock_responses('foo', 500, 200):
self.assertEqual('foo', self.read_for_test(reader, 3))
@tutil.skip_sleep
def test_read_with_method_retries(self):
reader = self.reader_for('foo_file')
with tutil.mock_responses('foo', 500, 200):
self.assertEqual('foo', self.read_for_test(reader,
3,
num_retries=3))
@tutil.skip_sleep
def test_read_instance_retries_exhausted(self):
reader = self.reader_for('bar_file', num_retries=3)
with tutil.mock_responses('bar', 500, 500, 500, 500, 200):
with self.assertRaises(arvados.errors.KeepReadError):
self.read_for_test(reader, 3)
@tutil.skip_sleep
def test_read_method_retries_exhausted(self):
reader = self.reader_for('bar_file')
with tutil.mock_responses('bar', 500, 500, 500, 500, 200):
with self.assertRaises(arvados.errors.KeepReadError):
self.read_for_test(reader, 3, num_retries=3)
@tutil.skip_sleep
def test_method_retries_take_precedence(self):
reader = self.reader_for('user_agreement', num_retries=10)
with tutil.mock_responses('', 500, 500, 500, 200):
with self.assertRaises(arvados.errors.KeepReadError):
self.read_for_test(reader, 10, num_retries=1)
def test_project(self):
uuid = run_test_server.fixture('groups')['aproject']['uuid']
args = arvados_fuse.command.ArgumentParser().parse_args([
'--project', uuid,
'--foreground', self.mntdir])
self.mnt = arvados_fuse.command.Mount(args)
e = self.check_ent_type(arvados_fuse.ProjectDirectory)
self.assertEqual(e.project_object['uuid'], uuid)
def test_collection(self, id_type='uuid'):
c = run_test_server.fixture('collections')['public_text_file']
cid = c[id_type]
args = arvados_fuse.command.ArgumentParser().parse_args(
['--collection', cid, '--foreground', self.mntdir])
self.mnt = arvados_fuse.command.Mount(args)
e = self.check_ent_type(arvados_fuse.CollectionDirectory)
self.assertEqual(e.collection_locator, cid)
self.assertEqual(id_type == 'uuid', self.mnt.listen_for_events)
def test_shared(self):
args = arvados_fuse.command.ArgumentParser().parse_args(
['--shared', '--foreground', self.mntdir])
self.assertEqual(args.mode, 'shared')
self.mnt = arvados_fuse.command.Mount(args)
e = self.check_ent_type(arvados_fuse.SharedDirectory)
self.assertEqual(e.current_user['uuid'],
run_test_server.fixture('users')['active']['uuid'])
self.assertEqual(True, self.mnt.listen_for_events)
def test_home(self):
args = arvados_fuse.command.ArgumentParser().parse_args(
['--home', '--foreground', self.mntdir])
self.assertEqual(args.mode, 'home')
self.mnt = arvados_fuse.command.Mount(args)
e = self.check_ent_type(arvados_fuse.ProjectDirectory)
self.assertEqual(e.project_object['uuid'],
run_test_server.fixture('users')['active']['uuid'])
self.assertEqual(True, self.mnt.listen_for_events)
def test_shared(self):
args = arvados_fuse.command.ArgumentParser().parse_args([
'--shared',
'--foreground', self.mntdir])
self.assertEqual(args.mode, 'shared')
self.mnt = arvados_fuse.command.Mount(args)
e = self.check_ent_type(arvados_fuse.SharedDirectory)
self.assertEqual(e.current_user['uuid'],
run_test_server.fixture('users')['active']['uuid'])
def test_home(self):
args = arvados_fuse.command.ArgumentParser().parse_args([
'--home',
'--foreground', self.mntdir])
self.assertEqual(args.mode, 'home')
self.mnt = arvados_fuse.command.Mount(args)
e = self.check_ent_type(arvados_fuse.ProjectDirectory)
self.assertEqual(e.project_object['uuid'],
run_test_server.fixture('users')['active']['uuid'])
def test_collection(self, id_type='uuid'):
c = run_test_server.fixture('collections')['public_text_file']
cid = c[id_type]
args = arvados_fuse.command.ArgumentParser().parse_args([
'--collection', cid,
'--foreground', self.mntdir])
self.mnt = arvados_fuse.command.Mount(args)
e = self.check_ent_type(arvados_fuse.CollectionDirectory)
self.assertEqual(e.collection_locator, cid)
def runTest(self):
self.make_mount(fuse.TagsDirectory, poll_time=1)
self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
self.tag_collection(bar_uuid, 'fuse_test_tag')
time.sleep(1)
self.assertIn('fuse_test_tag', llfuse.listdir(self.mounttmp))
self.assertDirContents('fuse_test_tag', [bar_uuid])
baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
l = self.tag_collection(baz_uuid, 'fuse_test_tag')
time.sleep(1)
self.assertDirContents('fuse_test_tag', [bar_uuid, baz_uuid])
self.api.links().delete(uuid=l['uuid']).execute()
time.sleep(1)
self.assertDirContents('fuse_test_tag', [bar_uuid])
def runTest(self):
self.make_mount(fuse.TagsDirectory, poll_time=1)
self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
self.tag_collection(bar_uuid, 'fuse_test_tag')
for attempt in AssertWithTimeout(10):
attempt(self.assertIn, 'fuse_test_tag', llfuse.listdir(self.mounttmp))
self.assertDirContents('fuse_test_tag', [bar_uuid])
baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
l = self.tag_collection(baz_uuid, 'fuse_test_tag')
for attempt in AssertWithTimeout(10):
attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid, baz_uuid])
self.api.links().delete(uuid=l['uuid']).execute()
for attempt in AssertWithTimeout(10):
attempt(self.assertDirContents, 'fuse_test_tag', [bar_uuid])
def test_custom(self):
args = arvados_fuse.command.ArgumentParser().parse_args([
'--mount-tmp', 'foo',
'--mount-tmp', 'bar',
'--mount-home', 'my_home',
'--foreground', self.mntdir])
self.assertEqual(args.mode, None)
self.mnt = arvados_fuse.command.Mount(args)
self.check_ent_type(arvados_fuse.Directory)
self.check_ent_type(arvados_fuse.TmpCollectionDirectory, 'foo')
self.check_ent_type(arvados_fuse.TmpCollectionDirectory, 'bar')
e = self.check_ent_type(arvados_fuse.ProjectDirectory, 'my_home')
self.assertEqual(e.project_object['uuid'],
run_test_server.fixture('users')['active']['uuid'])
def test_custom(self, mock_subscribe):
args = arvados_fuse.command.ArgumentParser().parse_args([
'--mount-tmp', 'foo', '--mount-tmp', 'bar', '--mount-home',
'my_home', '--foreground', self.mntdir
])
self.assertEqual(args.mode, None)
self.mnt = arvados_fuse.command.Mount(args)
self.check_ent_type(arvados_fuse.Directory)
self.check_ent_type(arvados_fuse.TmpCollectionDirectory, 'foo')
self.check_ent_type(arvados_fuse.TmpCollectionDirectory, 'bar')
e = self.check_ent_type(arvados_fuse.ProjectDirectory, 'my_home')
self.assertEqual(e.project_object['uuid'],
run_test_server.fixture('users')['active']['uuid'])
self.assertEqual(True, self.mnt.listen_for_events)
with self.mnt:
pass
self.assertEqual(1, mock_subscribe.call_count)
def test_default_all(self):
args = arvados_fuse.command.ArgumentParser().parse_args([
'--foreground', self.mntdir])
self.assertEqual(args.mode, None)
self.mnt = arvados_fuse.command.Mount(args)
e = self.check_ent_type(arvados_fuse.ProjectDirectory, 'home')
self.assertEqual(e.project_object['uuid'],
run_test_server.fixture('users')['active']['uuid'])
e = self.check_ent_type(arvados_fuse.MagicDirectory, 'by_id')
e = self.check_ent_type(arvados_fuse.StringFile, 'README')
readme = e.readfrom(0, -1)
self.assertRegexpMatches(readme, r'active-user@arvados\.local')
self.assertRegexpMatches(readme, r'\n$')
e = self.check_ent_type(arvados_fuse.StringFile, 'by_id', 'README')
txt = e.readfrom(0, -1)
self.assertRegexpMatches(txt, r'portable data hash')
self.assertRegexpMatches(txt, r'\n$')
class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
ArvadosBaseTestCase):
def _getKeepServerConfig():
for config_file, mandatory in [
['application.yml', True], ['application.default.yml', False]]:
path = os.path.join(run_test_server.SERVICES_SRC_DIR,
"api", "config", config_file)
if not mandatory and not os.path.exists(path):
continue
with open(path) as f:
rails_config = yaml.load(f.read())
for config_section in ['test', 'common']:
try:
key = rails_config[config_section]["blob_signing_key"]
except (KeyError, TypeError):
pass
else:
return {'blob_signing_key': key,
'enforce_permissions': True}
return {'blog_signing_key': None, 'enforce_permissions': False}
MAIN_SERVER = {}
KEEP_SERVER = _getKeepServerConfig()
PROJECT_UUID = run_test_server.fixture('groups')['aproject']['uuid']
@classmethod
def setUpClass(cls):
super(ArvPutIntegrationTest, cls).setUpClass()
cls.ENVIRON = os.environ.copy()
cls.ENVIRON['PYTHONPATH'] = ':'.join(sys.path)
def setUp(self):
super(ArvPutIntegrationTest, self).setUp()
arv_put.api_client = None
def authorize_with(self, token_name):
run_test_server.authorize_with(token_name)
for v in ["ARVADOS_API_HOST",
"ARVADOS_API_HOST_INSECURE",
"ARVADOS_API_TOKEN"]:
self.ENVIRON[v] = arvados.config.settings()[v]
arv_put.api_client = arvados.api('v1')
def current_user(self):
return arv_put.api_client.users().current().execute()
def test_check_real_project_found(self):
self.authorize_with('active')
self.assertTrue(arv_put.desired_project_uuid(arv_put.api_client, self.PROJECT_UUID, 0),
"did not correctly find test fixture project")
def test_check_error_finding_nonexistent_uuid(self):
BAD_UUID = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
self.authorize_with('active')
try:
result = arv_put.desired_project_uuid(arv_put.api_client, BAD_UUID,
0)
except ValueError as error:
self.assertIn(BAD_UUID, error.message)
else:
self.assertFalse(result, "incorrectly found nonexistent project")
def test_check_error_finding_nonexistent_project(self):
BAD_UUID = 'zzzzz-tpzed-zzzzzzzzzzzzzzz'
self.authorize_with('active')
with self.assertRaises(apiclient.errors.HttpError):
result = arv_put.desired_project_uuid(arv_put.api_client, BAD_UUID,
0)
def test_short_put_from_stdin(self):
# Have to run this as an integration test since arv-put can't
# read from the tests' stdin.
# arv-put usually can't stat(os.path.realpath('/dev/stdin')) in this
# case, because the /proc entry is already gone by the time it tries.
pipe = subprocess.Popen(
[sys.executable, arv_put.__file__, '--stream'],
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, env=self.ENVIRON)
pipe.stdin.write('stdin test\n')
pipe.stdin.close()
deadline = time.time() + 5
while (pipe.poll() is None) and (time.time() < deadline):
time.sleep(.1)
returncode = pipe.poll()
if returncode is None:
pipe.terminate()
self.fail("arv-put did not PUT from stdin within 5 seconds")
elif returncode != 0:
sys.stdout.write(pipe.stdout.read())
self.fail("arv-put returned exit code {}".format(returncode))
self.assertIn('4a9c8b735dce4b5fa3acf221a0b13628+11', pipe.stdout.read())
def test_ArvPutSignedManifest(self):
# ArvPutSignedManifest runs "arv-put foo" and then attempts to get
# the newly created manifest from the API server, testing to confirm
# that the block locators in the returned manifest are signed.
self.authorize_with('active')
# Before doing anything, demonstrate that the collection
# we're about to create is not present in our test fixture.
manifest_uuid = "00b4e9f40ac4dd432ef89749f1c01e74+47"
with self.assertRaises(apiclient.errors.HttpError):
notfound = arv_put.api_client.collections().get(
uuid=manifest_uuid).execute()
datadir = self.make_tmpdir()
with open(os.path.join(datadir, "foo"), "w") as f:
f.write("The quick brown fox jumped over the lazy dog")
p = subprocess.Popen([sys.executable, arv_put.__file__, datadir],
stdout=subprocess.PIPE, env=self.ENVIRON)
(arvout, arverr) = p.communicate()
self.assertEqual(arverr, None)
self.assertEqual(p.returncode, 0)
# The manifest text stored in the API server under the same
# manifest UUID must use signed locators.
c = arv_put.api_client.collections().get(uuid=manifest_uuid).execute()
self.assertRegexpMatches(
c['manifest_text'],
r'^\. 08a008a01d498c404b0c30852b39d3b8\+44\+A[0-9a-f]+@[0-9a-f]+ 0:44:foo\n')
os.remove(os.path.join(datadir, "foo"))
os.rmdir(datadir)
def run_and_find_collection(self, text, extra_args=[]):
self.authorize_with('active')
pipe = subprocess.Popen(
[sys.executable, arv_put.__file__] + extra_args,
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, env=self.ENVIRON)
stdout, stderr = pipe.communicate(text)
search_key = ('portable_data_hash'
if '--portable-data-hash' in extra_args else 'uuid')
collection_list = arvados.api('v1').collections().list(
filters=[[search_key, '=', stdout.strip()]]).execute().get('items', [])
self.assertEqual(1, len(collection_list))
return collection_list[0]
def test_put_collection_with_high_redundancy(self):
# Write empty data: we're not testing CollectionWriter, just
# making sure collections.create tells the API server what our
# desired replication level is.
collection = self.run_and_find_collection("", ['--replication', '4'])
self.assertEqual(4, collection['replication_desired'])
def test_put_collection_with_default_redundancy(self):
collection = self.run_and_find_collection("")
self.assertEqual(None, collection['replication_desired'])
def test_put_collection_with_unnamed_project_link(self):
link = self.run_and_find_collection(
"Test unnamed collection",
['--portable-data-hash', '--project-uuid', self.PROJECT_UUID])
username = pwd.getpwuid(os.getuid()).pw_name
self.assertRegexpMatches(
link['name'],
r'^Saved at .* by {}@'.format(re.escape(username)))
def test_put_collection_with_name_and_no_project(self):
link_name = 'Test Collection Link in home project'
collection = self.run_and_find_collection(
"Test named collection in home project",
['--portable-data-hash', '--name', link_name])
self.assertEqual(link_name, collection['name'])
my_user_uuid = self.current_user()['uuid']
self.assertEqual(my_user_uuid, collection['owner_uuid'])
def test_put_collection_with_named_project_link(self):
link_name = 'Test auto Collection Link'
collection = self.run_and_find_collection("Test named collection",
['--portable-data-hash',
'--name', link_name,
'--project-uuid', self.PROJECT_UUID])
self.assertEqual(link_name, collection['name'])
