def setUp(self):
super(ArvPutUploadJobTest, self).setUp()
run_test_server.authorize_with('active')
# Temp files creation
self.tempdir = tempfile.mkdtemp()
subdir = os.path.join(self.tempdir, 'subdir')
os.mkdir(subdir)
data = "x" * 1024 # 1 KB
for i in range(1, 5):
with open(os.path.join(self.tempdir, str(i)), 'w') as f:
f.write(data * i)
with open(os.path.join(subdir, 'otherfile'), 'w') as f:
f.write(data * 5)
# Large temp file for resume test
_, self.large_file_name = tempfile.mkstemp()
fileobj = open(self.large_file_name, 'w')
# Make sure to write just a little more than one block
for _ in range((arvados.config.KEEP_BLOCK_SIZE / (1024 * 1024)) + 1):
data = random.choice(['x', 'y', 'z']) * 1024 * 1024 # 1 MB
fileobj.write(data)
fileobj.close()
# Temp dir containing small files to be repacked
self.small_files_dir = tempfile.mkdtemp()
data = 'y' * 1024 * 1024 # 1 MB
for i in range(1, 70):
with open(os.path.join(self.small_files_dir, str(i)), 'w') as f:
f.write(data + str(i))
self.arvfile_write = getattr(arvados.arvfile.ArvadosFileWriter,
'write')
def test_KeepBasicRWTest(self):
run_test_server.authorize_with('active')
keep_client = arvados.KeepClient()
foo_locator = keep_client.put('foo')
self.assertRegexpMatches(
foo_locator,
r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
'invalid locator from Keep.put("foo"): ' + foo_locator)
self.assertEqual(keep_client.get(foo_locator), 'foo',
'wrong content from Keep.get(md5("foo"))')
# GET with an unsigned locator => NotFound
bar_locator = keep_client.put('bar')
unsigned_bar_locator = "37b51d194a7513e45b56f6524f2d51f2+3"
self.assertRegexpMatches(
bar_locator,
r'^37b51d194a7513e45b56f6524f2d51f2\+3\+A[a-f0-9]+@[a-f0-9]+$',
'invalid locator from Keep.put("bar"): ' + bar_locator)
self.assertRaises(arvados.errors.NotFoundError, keep_client.get,
unsigned_bar_locator)
# GET from a different user => NotFound
run_test_server.authorize_with('spectator')
self.assertRaises(arvados.errors.NotFoundError, arvados.Keep.get,
bar_locator)
# Unauthenticated GET for a signed locator => NotFound
# Unauthenticated GET for an unsigned locator => NotFound
keep_client.api_token = ''
self.assertRaises(arvados.errors.NotFoundError, keep_client.get,
bar_locator)
self.assertRaises(arvados.errors.NotFoundError, keep_client.get,
unsigned_bar_locator)
def authorize_with(self, token_name):
run_test_server.authorize_with(token_name)
for v in ["ARVADOS_API_HOST",
"ARVADOS_API_HOST_INSECURE",
"ARVADOS_API_TOKEN"]:
self.ENVIRON[v] = arvados.config.settings()[v]
arv_put.api_client = arvados.api('v1')
def setUpClass(cls):
super(KeepTestCase, cls).setUpClass()
run_test_server.authorize_with("admin")
cls.api_client = arvados.api('v1')
cls.keep_client = arvados.KeepClient(api_client=cls.api_client,
proxy='',
local_store='')
def setUp(self):
self.keeptmp = tempfile.mkdtemp()
os.environ['KEEP_LOCAL_STORE'] = self.keeptmp
self.mounttmp = tempfile.mkdtemp()
run_test_server.run(False)
run_test_server.authorize_with("admin")
self.api = api = fuse.SafeApi(arvados.config)
def setUp(self):
self.keeptmp = tempfile.mkdtemp()
os.environ['KEEP_LOCAL_STORE'] = self.keeptmp
self.mounttmp = tempfile.mkdtemp()
run_test_server.run(False)
run_test_server.authorize_with("admin")
self.api = api = fuse.SafeApi(arvados.config)
def runTest(self):
run_test_server.authorize_with("admin")
api = arvados.api('v1', cache=False)
operations = fuse.Operations(os.getuid(), os.getgid())
e = operations.inodes.add_entry(fuse.TagsDirectory(llfuse.ROOT_INODE, operations.inodes, api))
llfuse.init(operations, self.mounttmp, [])
t = threading.Thread(None, lambda: llfuse.main())
t.start()
# wait until the driver is finished initializing
operations.initlock.wait()
d1 = os.listdir(self.mounttmp)
d1.sort()
self.assertEqual(['foo_tag'], d1)
d2 = os.listdir(os.path.join(self.mounttmp, 'foo_tag'))
d2.sort()
self.assertEqual(['1f4b0bc7583c2a7f9102c395f4ffc5e3+45'], d2)
d3 = os.listdir(os.path.join(self.mounttmp, 'foo_tag', '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'))
d3.sort()
self.assertEqual(['foo'], d3)
files = {}
files[os.path.join(self.mounttmp, 'foo_tag', '1f4b0bc7583c2a7f9102c395f4ffc5e3+45', 'foo')] = 'foo'
for k, v in files.items():
with open(os.path.join(self.mounttmp, k)) as f:
self.assertEqual(v, f.read())
def authorize_with(self, token_name):
run_test_server.authorize_with(token_name)
for v in ["ARVADOS_API_HOST",
"ARVADOS_API_HOST_INSECURE",
"ARVADOS_API_TOKEN"]:
self.ENVIRON[v] = arvados.config.settings()[v]
arv_put.api_client = arvados.api('v1')
def _test_websocket_reconnect(self, close_unexpected):
run_test_server.authorize_with('active')
events = Queue.Queue(100)
logstream = io.BytesIO()
rootLogger = logging.getLogger()
streamHandler = logging.StreamHandler(logstream)
rootLogger.addHandler(streamHandler)
filters = [['object_uuid', 'is_a', 'arvados#human']]
filters.append(['created_at', '>=', self.localiso(self.TIME_PAST)])
self.ws = arvados.events.subscribe(
arvados.api('v1'), filters,
events.put_nowait,
poll_fallback=False,
last_log_id=None)
self.assertIsInstance(self.ws, arvados.events.EventClient)
self.assertEqual(200, events.get(True, 5)['status'])
# create obj
human = arvados.api('v1').humans().create(body={}).execute()
# expect an event
self.assertIn(human['uuid'], events.get(True, 5)['object_uuid'])
with self.assertRaises(Queue.Empty):
self.assertEqual(events.get(True, 2), None)
# close (im)properly
if close_unexpected:
self.ws.ec.close_connection()
else:
self.ws.close()
# create one more obj
human2 = arvados.api('v1').humans().create(body={}).execute()
# (un)expect the object creation event
if close_unexpected:
log_object_uuids = []
for i in range(0, 2):
event = events.get(True, 5)
if event.get('object_uuid') != None:
log_object_uuids.append(event['object_uuid'])
with self.assertRaises(Queue.Empty):
self.assertEqual(events.get(True, 2), None)
self.assertNotIn(human['uuid'], log_object_uuids)
self.assertIn(human2['uuid'], log_object_uuids)
else:
with self.assertRaises(Queue.Empty):
self.assertEqual(events.get(True, 2), None)
# verify log message to ensure that an (un)expected close
log_messages = logstream.getvalue()
closeLogFound = log_messages.find("Unexpected close. Reconnecting.")
retryLogFound = log_messages.find("Error during websocket reconnect. Will retry")
if close_unexpected:
self.assertNotEqual(closeLogFound, -1)
else:
self.assertEqual(closeLogFound, -1)
rootLogger.removeHandler(streamHandler)
def test_websocket_reconnect_retry(self, event_client_connect):
event_client_connect.side_effect = [None, Exception('EventClient.connect error'), None]
logstream = io.BytesIO()
rootLogger = logging.getLogger()
streamHandler = logging.StreamHandler(logstream)
rootLogger.addHandler(streamHandler)
run_test_server.authorize_with('active')
events = Queue.Queue(100)
filters = [['object_uuid', 'is_a', 'arvados#human']]
self.ws = arvados.events.subscribe(
arvados.api('v1'), filters,
events.put_nowait,
poll_fallback=False,
last_log_id=None)
self.assertIsInstance(self.ws, arvados.events.EventClient)
# simulate improper close
self.ws.on_closed()
# verify log messages to ensure retry happened
log_messages = logstream.getvalue()
found = log_messages.find("Error 'EventClient.connect error' during websocket reconnect.")
self.assertNotEqual(found, -1)
rootLogger.removeHandler(streamHandler)
def setUp(self):
self.keeptmp = tempfile.mkdtemp()
os.environ['KEEP_LOCAL_STORE'] = self.keeptmp
self.mounttmp = tempfile.mkdtemp()
run_test_server.run()
run_test_server.authorize_with("admin")
self.api = arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
def runTest(self):
self.state = 1
run_test_server.authorize_with("admin")
api = arvados.api('v1', cache=False)
arvados.events.subscribe(api, [['object_uuid', 'is_a', 'arvados#human']], lambda ev: self.on_event(ev))
time.sleep(1)
self.h = api.humans().create(body={}).execute()
time.sleep(1)
def test_KeepAuthenticatedUnsignedTest(self):
run_test_server.authorize_with('active')
signed_locator = arvados.Keep.put('foo')
self.assertRegexpMatches(
signed_locator,
r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
'invalid locator from Keep.put("foo"): ' + signed_locator)
self.assertEqual(arvados.Keep.get("acbd18db4cc2f85cedef654fccc4a4d8"),
'foo',
'wrong content from Keep.get(md5("foo"))')
def runTest(self):
self.state = 1
run_test_server.authorize_with("admin")
api = arvados.api('v1', cache=False)
arvados.events.subscribe(api,
[['object_uuid', 'is_a', 'arvados#human']],
lambda ev: self.on_event(ev))
time.sleep(1)
self.h = api.humans().create(body={}).execute()
time.sleep(1)
def _test_subscribe(self, poll_fallback, expect_type):
run_test_server.authorize_with('active')
events = Queue.Queue(3)
self.ws = arvados.events.subscribe(
arvados.api('v1'), [['object_uuid', 'is_a', 'arvados#human']],
events.put, poll_fallback=poll_fallback)
self.assertIsInstance(self.ws, expect_type)
self.assertEqual(200, events.get(True, 10)['status'])
human = arvados.api('v1').humans().create(body={}).execute()
self.assertEqual(human['uuid'], events.get(True, 10)['object_uuid'])
self.assertTrue(events.empty(), "got more events than expected")
def _test_subscribe(self, poll_fallback, expect_type):
run_test_server.authorize_with('active')
events = Queue.Queue(3)
self.ws = arvados.events.subscribe(
arvados.api('v1'), [['object_uuid', 'is_a', 'arvados#human']],
events.put,
poll_fallback=poll_fallback)
self.assertIsInstance(self.ws, expect_type)
self.assertEqual(200, events.get(True, 10)['status'])
human = arvados.api('v1').humans().create(body={}).execute()
self.assertEqual(human['uuid'], events.get(True, 10)['object_uuid'])
self.assertTrue(events.empty(), "got more events than expected")
def runRealTest(self):
run_test_server.authorize_with("admin")
api = arvados.api('v1', cache=False)
operations = fuse.Operations(os.getuid(), os.getgid())
e = operations.inodes.add_entry(fuse.TagsDirectory(llfuse.ROOT_INODE, operations.inodes, api, poll_time=1))
llfuse.init(operations, self.mounttmp, [])
t = threading.Thread(None, lambda: llfuse.main())
t.start()
# wait until the driver is finished initializing
operations.initlock.wait()
d1 = os.listdir(self.mounttmp)
d1.sort()
self.assertEqual(['foo_tag'], d1)
api.links().create(body={'link': {
'head_uuid': 'fa7aeb5140e2848d39b416daeef4ffc5+45',
'link_class': 'tag',
'name': 'bar_tag'
}}).execute()
time.sleep(1)
d2 = os.listdir(self.mounttmp)
d2.sort()
self.assertEqual(['bar_tag', 'foo_tag'], d2)
d3 = os.listdir(os.path.join(self.mounttmp, 'bar_tag'))
d3.sort()
self.assertEqual(['fa7aeb5140e2848d39b416daeef4ffc5+45'], d3)
l = api.links().create(body={'link': {
'head_uuid': 'ea10d51bcf88862dbcc36eb292017dfd+45',
'link_class': 'tag',
'name': 'bar_tag'
}}).execute()
time.sleep(1)
d4 = os.listdir(os.path.join(self.mounttmp, 'bar_tag'))
d4.sort()
self.assertEqual(['ea10d51bcf88862dbcc36eb292017dfd+45', 'fa7aeb5140e2848d39b416daeef4ffc5+45'], d4)
api.links().delete(uuid=l['uuid']).execute()
time.sleep(1)
d5 = os.listdir(os.path.join(self.mounttmp, 'bar_tag'))
d5.sort()
self.assertEqual(['fa7aeb5140e2848d39b416daeef4ffc5+45'], d5)
def test_KeepUnauthenticatedUnsignedTest(self):
# Since --enforce-permissions is not in effect, GET requests
# need not be authenticated.
run_test_server.authorize_with('active')
signed_locator = arvados.Keep.put('foo')
self.assertRegexpMatches(
signed_locator,
r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
'invalid locator from Keep.put("foo"): ' + signed_locator)
del arvados.config.settings()["ARVADOS_API_TOKEN"]
self.assertEqual(arvados.Keep.get("acbd18db4cc2f85cedef654fccc4a4d8"),
'foo',
'wrong content from Keep.get(md5("foo"))')
def put(files):
os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3000"
run_test_server.authorize_with('active')
for v in ["ARVADOS_API_HOST",
"ARVADOS_API_HOST_INSECURE",
"ARVADOS_API_TOKEN"]:
os.environ[v] = arvados.config.settings()[v]
if not os.environ.has_key('PYTHONPATH'):
os.environ['PYTHONPATH'] = ''
os.environ['PYTHONPATH'] = "{}:{}:{}".format(
PYSDK_DIR, PYTEST_DIR, os.environ['PYTHONPATH'])
for c in files:
manifest_uuid = arv_cmd(ARV_PUT_PATH, c)
def put(files):
os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
run_test_server.authorize_with('active')
for v in ["ARVADOS_API_HOST",
"ARVADOS_API_HOST_INSECURE",
"ARVADOS_API_TOKEN"]:
os.environ[v] = arvados.config.settings()[v]
if not os.environ.has_key('PYTHONPATH'):
os.environ['PYTHONPATH'] = ''
os.environ['PYTHONPATH'] = "{}:{}:{}".format(
PYSDK_DIR, PYTEST_DIR, os.environ['PYTHONPATH'])
for c in files:
manifest_uuid = arv_cmd(ARV_PUT_PATH, c)
def setUp(self):
# The underlying C implementation of open() makes a fstat() syscall
# with the GIL still held. When the GETATTR message comes back to
# llfuse (which in these tests is in the same interpreter process) it
# can't acquire the GIL, so it can't service the fstat() call, so it
# deadlocks. The workaround is to run some of our test code in a
# separate process. Forturnately the multiprocessing module makes this
# relatively easy.
self.pool = multiprocessing.Pool(1)
self.keeptmp = tempfile.mkdtemp()
os.environ['KEEP_LOCAL_STORE'] = self.keeptmp
self.mounttmp = tempfile.mkdtemp()
run_test_server.run()
run_test_server.authorize_with("admin")
self.api = arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
def setUp(self, api=None):
# The underlying C implementation of open() makes a fstat() syscall
# with the GIL still held. When the GETATTR message comes back to
# llfuse (which in these tests is in the same interpreter process) it
# can't acquire the GIL, so it can't service the fstat() call, so it
# deadlocks. The workaround is to run some of our test code in a
# separate process. Forturnately the multiprocessing module makes this
# relatively easy.
self.pool = multiprocessing.Pool(1)
self.keeptmp = tempfile.mkdtemp()
os.environ['KEEP_LOCAL_STORE'] = self.keeptmp
self.mounttmp = tempfile.mkdtemp()
run_test_server.run()
run_test_server.authorize_with("admin")
self.api = api if api else arvados.safeapi.ThreadSafeApiCache(
arvados.config.settings())
def get(blocks):
os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3000"
run_test_server.authorize_with('active')
for v in ["ARVADOS_API_HOST",
"ARVADOS_API_HOST_INSECURE",
"ARVADOS_API_TOKEN"]:
os.environ[v] = arvados.config.settings()[v]
nqueries = 0
while True:
b = random.choice(blocks)
print "GET /" + b
body = arv_cmd(ARV_GET_PATH, b)
print "got {} bytes".format(len(body))
time.sleep(SECONDS_BETWEEN_GETS)
nqueries = nqueries + 1
def setUpClass(cls):
super(KeepTestCase, cls).setUpClass()
try:
del os.environ['KEEP_LOCAL_STORE']
except KeyError:
pass
# Make sure these are clear, we want to talk to the Keep servers
# directly.
os.environ["ARVADOS_KEEP_PROXY"] = ""
os.environ["ARVADOS_EXTERNAL_CLIENT"] = ""
run_test_server.run()
run_test_server.run_keep()
arvados.keep.global_client_object = None
arvados.config._settings = None
run_test_server.authorize_with("admin")
def get(blocks):
os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
run_test_server.authorize_with('active')
for v in ["ARVADOS_API_HOST",
"ARVADOS_API_HOST_INSECURE",
"ARVADOS_API_TOKEN"]:
os.environ[v] = arvados.config.settings()[v]
nqueries = 0
while True:
b = random.choice(blocks)
print "GET /" + b
body = arv_cmd(ARV_GET_PATH, b)
print "got {} bytes".format(len(body))
time.sleep(SECONDS_BETWEEN_GETS)
nqueries = nqueries + 1
def runTest(self):
run_test_server.authorize_with("admin")
pt_uuid = arvados.api('v1').pipeline_templates().create(
body={'name':__file__}
).execute()['uuid']
self.assertEqual(len(pt_uuid), 27,
'Unexpected format of pipeline template UUID ("%s")'
% pt_uuid)
components = {
'x': 'x',
'-x-': [1,2,{'foo':'bar'}],
'Boggis': {'Bunce': '[\'Bean\']'},
'SpassBox': True,
'spass_box': False,
'spass-box': [True, 'Maybe', False]
}
update_response = arvados.api('v1').pipeline_templates().update(
uuid=pt_uuid,
body={'components':components}
).execute()
self.assertEqual('uuid' in update_response, True,
'update() response did not include a uuid')
self.assertEqual(update_response['uuid'], pt_uuid,
'update() response has a different uuid (%s, not %s)'
% (update_response['uuid'], pt_uuid))
self.assertEqual(update_response['name'], __file__,
'update() response has a different name (%s, not %s)'
% (update_response['name'], __file__))
get_response = arvados.api('v1').pipeline_templates().get(
uuid=pt_uuid
).execute()
self.assertEqual(get_response['components'], components,
'components got munged by server (%s -> %s)'
% (components, update_response['components']))
delete_response = arvados.api('v1').pipeline_templates().delete(
uuid=pt_uuid
).execute()
self.assertEqual(delete_response['uuid'], pt_uuid,
'delete() response has wrong uuid (%s, not %s)'
% (delete_response['uuid'], pt_uuid))
with self.assertRaises(apiclient.errors.HttpError):
geterror_response = arvados.api('v1').pipeline_templates().get(
uuid=pt_uuid
).execute()
def _test_subscribe(self,
poll_fallback,
expect_type,
start_time=None,
expected=1):
run_test_server.authorize_with('active')
events = Queue.Queue(100)
# Create ancestor before subscribing.
# When listening with start_time in the past, this should also be retrieved.
# However, when start_time is omitted in subscribe, this should not be fetched.
ancestor = arvados.api('v1').humans().create(body={}).execute()
filters = [['object_uuid', 'is_a', 'arvados#human']]
if start_time:
filters.append(['created_at', '>=', start_time])
self.ws = arvados.events.subscribe(
arvados.api('v1'),
filters,
events.put_nowait,
poll_fallback=poll_fallback,
last_log_id=(1 if start_time else None))
self.assertIsInstance(self.ws, expect_type)
self.assertEqual(200, events.get(True, 5)['status'])
human = arvados.api('v1').humans().create(body={}).execute()
want_uuids = []
if expected > 0:
want_uuids.append(human['uuid'])
if expected > 1:
want_uuids.append(ancestor['uuid'])
log_object_uuids = []
while set(want_uuids) - set(log_object_uuids):
log_object_uuids.append(events.get(True, 5)['object_uuid'])
if expected < 2:
with self.assertRaises(Queue.Empty):
# assertEqual just serves to show us what unexpected
# thing comes out of the queue when the assertRaises
# fails; when the test passes, this assertEqual
# doesn't get called.
self.assertEqual(events.get(True, 2), None)
def runTest(self):
run_test_server.authorize_with("admin")
api = arvados.api('v1', cache=False)
operations = fuse.Operations(os.getuid(), os.getgid())
e = operations.inodes.add_entry(fuse.GroupsDirectory(llfuse.ROOT_INODE, operations.inodes, api))
llfuse.init(operations, self.mounttmp, [])
t = threading.Thread(None, lambda: llfuse.main())
t.start()
# wait until the driver is finished initializing
operations.initlock.wait()
d1 = os.listdir(self.mounttmp)
d1.sort()
self.assertIn('zzzzz-j7d0g-v955i6s2oi1cbso', d1)
d2 = os.listdir(os.path.join(self.mounttmp, 'zzzzz-j7d0g-v955i6s2oi1cbso'))
d2.sort()
self.assertEqual(['1f4b0bc7583c2a7f9102c395f4ffc5e3+45 added sometime',
"I'm a job in a project",
"I'm a template in a project",
"zzzzz-j58dm-5gid26432uujf79",
"zzzzz-j58dm-7r18rnd5nzhg5yk",
"zzzzz-j58dm-ypsjlol9dofwijz",
"zzzzz-j7d0g-axqo7eu9pwvna1x"
], d2)
d3 = os.listdir(os.path.join(self.mounttmp, 'zzzzz-j7d0g-v955i6s2oi1cbso', 'zzzzz-j7d0g-axqo7eu9pwvna1x'))
d3.sort()
self.assertEqual(["I'm in a subproject, too",
"ea10d51bcf88862dbcc36eb292017dfd+45 added sometime",
"zzzzz-j58dm-c40lddwcqqr1ffs"
], d3)
with open(os.path.join(self.mounttmp, 'zzzzz-j7d0g-v955i6s2oi1cbso', "I'm a template in a project")) as f:
j = json.load(f)
self.assertEqual("Two Part Pipeline Template", j['name'])
def test_KeepBasicRWTest(self):
run_test_server.authorize_with('active')
keep_client = arvados.KeepClient()
foo_locator = keep_client.put('foo')
self.assertRegexpMatches(
foo_locator,
r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
'invalid locator from Keep.put("foo"): ' + foo_locator)
self.assertEqual(keep_client.get(foo_locator),
'foo',
'wrong content from Keep.get(md5("foo"))')
# GET with an unsigned locator => NotFound
bar_locator = keep_client.put('bar')
unsigned_bar_locator = "37b51d194a7513e45b56f6524f2d51f2+3"
self.assertRegexpMatches(
bar_locator,
r'^37b51d194a7513e45b56f6524f2d51f2\+3\+A[a-f0-9]+@[a-f0-9]+$',
'invalid locator from Keep.put("bar"): ' + bar_locator)
self.assertRaises(arvados.errors.NotFoundError,
keep_client.get,
unsigned_bar_locator)
# GET from a different user => NotFound
run_test_server.authorize_with('spectator')
self.assertRaises(arvados.errors.NotFoundError,
arvados.Keep.get,
bar_locator)
# Unauthenticated GET for a signed locator => NotFound
# Unauthenticated GET for an unsigned locator => NotFound
keep_client.api_token = ''
self.assertRaises(arvados.errors.NotFoundError,
keep_client.get,
bar_locator)
self.assertRaises(arvados.errors.NotFoundError,
keep_client.get,
unsigned_bar_locator)
def _test_subscribe(self, poll_fallback, expect_type, start_time=None, expected=1):
run_test_server.authorize_with('active')
events = Queue.Queue(100)
# Create ancestor before subscribing.
# When listening with start_time in the past, this should also be retrieved.
# However, when start_time is omitted in subscribe, this should not be fetched.
ancestor = arvados.api('v1').humans().create(body={}).execute()
filters = [['object_uuid', 'is_a', 'arvados#human']]
if start_time:
filters.append(['created_at', '>=', start_time])
self.ws = arvados.events.subscribe(
arvados.api('v1'), filters,
events.put_nowait,
poll_fallback=poll_fallback,
last_log_id=(1 if start_time else None))
self.assertIsInstance(self.ws, expect_type)
self.assertEqual(200, events.get(True, 5)['status'])
human = arvados.api('v1').humans().create(body={}).execute()
log_object_uuids = []
for i in range(0, expected):
log_object_uuids.append(events.get(True, 5)['object_uuid'])
if expected > 0:
self.assertIn(human['uuid'], log_object_uuids)
if expected > 1:
self.assertIn(ancestor['uuid'], log_object_uuids)
with self.assertRaises(Queue.Empty):
# assertEqual just serves to show us what unexpected thing
# comes out of the queue when the assertRaises fails; when
# the test passes, this assertEqual doesn't get called.
self.assertEqual(events.get(True, 2), None)
def test_ArvPutSignedManifest(self):
# ArvPutSignedManifest runs "arv-put foo" and then attempts to get
# the newly created manifest from the API server, testing to confirm
# that the block locators in the returned manifest are signed.
run_test_server.authorize_with('active')
for v in ["ARVADOS_API_HOST",
"ARVADOS_API_HOST_INSECURE",
"ARVADOS_API_TOKEN"]:
os.environ[v] = arvados.config.settings()[v]
# Before doing anything, demonstrate that the collection
# we're about to create is not present in our test fixture.
api = arvados.api('v1', cache=False)
manifest_uuid = "00b4e9f40ac4dd432ef89749f1c01e74+47"
with self.assertRaises(apiclient.errors.HttpError):
notfound = api.collections().get(uuid=manifest_uuid).execute()
datadir = tempfile.mkdtemp()
with open(os.path.join(datadir, "foo"), "w") as f:
f.write("The quick brown fox jumped over the lazy dog")
p = subprocess.Popen([sys.executable, arv_put.__file__, datadir],
stdout=subprocess.PIPE)
(arvout, arverr) = p.communicate()
self.assertEqual(p.returncode, 0)
self.assertEqual(arverr, None)
self.assertEqual(arvout.strip(), manifest_uuid)
# The manifest text stored in the API server under the same
# manifest UUID must use signed locators.
c = api.collections().get(uuid=manifest_uuid).execute()
self.assertRegexpMatches(
c['manifest_text'],
r'^\. 08a008a01d498c404b0c30852b39d3b8\+44\+A[0-9a-f]+@[0-9a-f]+ 0:44:foo\n')
os.remove(os.path.join(datadir, "foo"))
os.rmdir(datadir)
def setUpClass(cls):
run_test_server.run()
run_test_server.run_keep(enforce_permissions=True, num_servers=2)
run_test_server.authorize_with('active')
def setUpClass(cls):
super(KeepOptionalPermission, cls).setUpClass()
run_test_server.authorize_with("admin")
cls.api_client = arvados.api('v1')
def setUp(self):
self.mnt = tempfile.mkdtemp()
run_test_server.authorize_with('active')
self.api = arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
def setUp(self):
self.mnt = tempfile.mkdtemp()
run_test_server.authorize_with('active')
def setUp(self):
self.mnt = tempfile.mkdtemp()
run_test_server.authorize_with('active')
def setUpClass(cls):
super(KeepProxyTestCase, cls).setUpClass()
run_test_server.authorize_with('active')
cls.api_client = arvados.api('v1')
def setUpClass(cls):
super(CollectionBenchmark, cls).setUpClass()
run_test_server.authorize_with('active')
cls.api_client = arvados.api('v1')
cls.keep_client = arvados.KeepClient(api_client=cls.api_client,
local_store=cls.local_store)
def setUpClass(cls):
super(KeepOptionalPermission, cls).setUpClass()
run_test_server.authorize_with("admin")
cls.api_client = arvados.api('v1')
def setUpClass(cls):
super(KeepTestCase, cls).setUpClass()
run_test_server.authorize_with("admin")
cls.api_client = arvados.api('v1')
cls.keep_client = arvados.KeepClient(api_client=cls.api_client,
proxy='', local_store='')
def setUpClass(cls):
super(KeepProxyTestCase, cls).setUpClass()
run_test_server.authorize_with('active')
cls.api_client = arvados.api('v1')
def setUp(self):
super(ArvadosPutTest, self).setUp()
run_test_server.authorize_with('active')
arv_put.api_client = None
def authorize_with(self, token_name):
run_test_server.authorize_with(token_name)
for v in ["ARVADOS_API_HOST",
"ARVADOS_API_HOST_INSECURE",
"ARVADOS_API_TOKEN"]:
os.environ[v] = arvados.config.settings()[v]
def setUp(self):
super(ArvadosPutCollectionWriterTest, self).setUp()
run_test_server.authorize_with('active')
with tempfile.NamedTemporaryFile(delete=False) as cachefile:
self.cache = arv_put.ResumeCache(cachefile.name)
self.cache_filename = cachefile.name
def setUp(self):
super(ArvadosPutTest, self).setUp()
run_test_server.authorize_with('active')
arv_put.api_client = None
def setUp(self):
super(ArvadosPutCollectionWriterTest, self).setUp()
run_test_server.authorize_with('active')
with tempfile.NamedTemporaryFile(delete=False) as cachefile:
self.cache = arv_put.ResumeCache(cachefile.name)
self.cache_filename = cachefile.name
def setUp(self):
self.mntdir = tempfile.mkdtemp()
run_test_server.run()
run_test_server.authorize_with("active")
self.logger = logging.getLogger("null")
self.logger.setLevel(logging.CRITICAL+1)
def setUp(self):
self.mnt = tempfile.mkdtemp()
run_test_server.authorize_with('active')
self.api = arvados.safeapi.ThreadSafeApiCache(
arvados.config.settings())
def setUpClass(cls):
super(CollectionBenchmark, cls).setUpClass()
run_test_server.authorize_with('active')
cls.api_client = arvados.api('v1')
cls.keep_client = arvados.KeepClient(api_client=cls.api_client,
local_store=cls.local_store)
