def run_push_and_fetch_test(self, namespace):
storage = isolateserver.get_storage(self.server.url, namespace)
# Upload items.
items = [isolateserver.BufferItem('item %d' % i) for i in xrange(10)]
uploaded = storage.upload_items(items)
self.assertEqual(set(items), set(uploaded))
# Fetch them all back into local memory cache.
cache = isolateserver.MemoryCache()
queue = isolateserver.FetchQueue(storage, cache)
# Start fetching.
pending = set()
for item in items:
pending.add(item.digest)
queue.add(item.digest)
# Wait for fetch to complete.
while pending:
fetched = queue.wait(pending)
pending.discard(fetched)
# Ensure fetched same data as was pushed.
self.assertEqual([i.buffer for i in items],
[cache.read(i.digest) for i in items])
def _run_tha_test(self, isolated_hash=None, files=None, command=None):
files = files or {}
make_tree_call = []
def add(i, _):
make_tree_call.append(i)
for i in ('make_tree_read_only', 'make_tree_files_read_only',
'make_tree_deleteable', 'make_tree_writeable'):
self.mock(file_path, i, functools.partial(add, i))
ret = run_isolated.run_tha_test(
command or [],
isolated_hash,
StorageFake(files),
isolateserver.MemoryCache(),
None,
init_named_caches_stub,
False,
None,
None,
None,
None,
None,
run_isolated.noop_install_packages,
False)
self.assertEqual(0, ret)
return make_tree_call
def _run_tha_test(self, isolated_hash=None, files=None, command=None):
files = files or {}
make_tree_call = []
def add(i, _):
make_tree_call.append(i)
for i in ('make_tree_read_only', 'make_tree_files_read_only',
'make_tree_deleteable', 'make_tree_writeable'):
self.mock(file_path, i, functools.partial(add, i))
data = run_isolated.TaskData(
command=command or [],
relative_cwd=None,
extra_args=[],
isolated_hash=isolated_hash,
storage=StorageFake(files),
isolate_cache=isolateserver.MemoryCache(),
outputs=None,
install_named_caches=init_named_caches_stub,
leak_temp_dir=False,
root_dir=None,
hard_timeout=60,
grace_period=30,
bot_file=None,
switch_to_account=False,
install_packages_fn=run_isolated.noop_install_packages,
use_symlinks=False,
env={},
env_prefix={})
ret = run_isolated.run_tha_test(data, None)
self.assertEqual(0, ret)
return make_tree_call
class RunIsolatedTestOutputFiles(RunIsolatedTestBase):
def _run_test(self, isolated, command):
# Starts a full isolate server mock and have run_tha_test() uploads results
# back after the task completed.
server = isolateserver_mock.MockIsolateServer()
try:
script = ('import sys\n'
'open(sys.argv[1], "w").write("bar")\n'
'open(sys.argv[2], "w").write("baz")\n')
script_hash = isolateserver_mock.hash_content(script)
isolated['files']['cmd.py'] = {
'h': script_hash,
'm': 0700,
's': len(script),
}
if sys.platform == 'win32':
isolated['files']['cmd.py'].pop('m')
isolated_data = json_dumps(isolated)
isolated_hash = isolateserver_mock.hash_content(isolated_data)
server.add_content('default-store', script)
server.add_content('default-store', isolated_data)
store = isolateserver.get_storage(server.url, 'default-store')
self.mock(sys, 'stdout', StringIO.StringIO())
ret = run_isolated.run_tha_test(command, isolated_hash, store,
isolateserver.MemoryCache(),
['foo', 'foodir/foo2'],
init_named_caches_stub, False,
None, None, None, None, None,
run_isolated.noop_install_packages,
False)
self.assertEqual(0, ret)
# It uploaded back. Assert the store has a new item containing foo.
hashes = {isolated_hash, script_hash}
foo_output_hash = isolateserver_mock.hash_content('bar')
foo2_output_hash = isolateserver_mock.hash_content('baz')
hashes.add(foo_output_hash)
hashes.add(foo2_output_hash)
isolated = {
'algo': 'sha-1',
'files': {
'foo': {
'h': foo_output_hash,
# TODO(maruel): Handle umask.
'm': 0640,
's': 3,
},
'foodir/foo2': {
'h': foo2_output_hash,
# TODO(maruel): Handle umask.
'm': 0640,
's': 3,
},
},
'version': isolated_format.ISOLATED_FILE_VERSION,
}
def _run_tha_test(self, isolated_hash, files):
make_tree_call = []
def add(i, _):
make_tree_call.append(i)
for i in ('make_tree_read_only', 'make_tree_files_read_only',
'make_tree_deleteable', 'make_tree_writeable'):
self.mock(file_path, i, functools.partial(add, i))
ret = run_isolated.run_tha_test(isolated_hash, StorageFake(files),
isolateserver.MemoryCache(), False,
None, None, None, None, [])
self.assertEqual(0, ret)
return make_tree_call
def send_and_receive(random_pool, storage, progress, size):
"""Sends a random file and gets it back.
# TODO(maruel): Add a batching argument of value [1, 500] to batch requests.
Returns (delay, size)
"""
# Create a file out of the pool.
start = time.time()
batch = 1
items = [
isolateserver.BufferItem(random_pool.gen(size), False)
for _ in xrange(batch)
]
try:
# len(_uploaded) may be < len(items) happen if the items is not random
# enough or value of --mid-size is very low compared to --items.
_uploaded = storage.upload_items(items)
start = time.time()
cache = isolateserver.MemoryCache()
queue = isolateserver.FetchQueue(storage, cache)
for i in items:
queue.add(i.digest, i.size)
waiting = [i.digest for i in items]
while waiting:
waiting.remove(queue.wait(waiting))
expected = {i.digest: ''.join(i.content()) for i in items}
for d in cache.cached_set():
actual = cache.read(d)
assert expected.pop(d) == actual
assert not expected, expected
duration = max(0, time.time() - start)
except isolateserver.MappingError as e:
duration = str(e)
if isinstance(duration, float):
progress.update_item('', index=1, data=size)
else:
progress.update_item('', index=1)
return (duration, size)
def process_shard_result(self, result):
"""Stores results of a single task shard, fetches output files if necessary.
Called concurrently from multiple threads.
"""
# We are going to put |shard_index| into a file path. Make sure it is int.
shard_index = result['config_instance_index']
if not isinstance(shard_index, int):
raise ValueError('Shard index should be an int: %r' %
(shard_index, ))
# Sanity check index is in expected range.
if shard_index < 0 or shard_index >= self.shard_count:
logging.warning(
'Shard index %d is outside of expected range: [0; %d]',
shard_index, self.shard_count - 1)
return
# Store result dict of that shard, ignore results we've already seen.
with self._lock:
if shard_index in self._per_shard_results:
logging.warning('Ignoring duplicate shard index %d',
shard_index)
return
self._per_shard_results[shard_index] = result
# Fetch output files if necessary.
isolated_files_location = extract_output_files_location(
result['output'])
if isolated_files_location:
isolate_server, namespace, isolated_hash = isolated_files_location
storage = self._get_storage(isolate_server, namespace)
if storage:
# Output files are supposed to be small and they are not reused across
# tasks. So use MemoryCache for them instead of on-disk cache. Make
# files writable, so that calling script can delete them.
isolateserver.fetch_isolated(
isolated_hash, storage,
isolateserver.MemoryCache(file_mode_mask=0700),
os.path.join(self.task_output_dir, str(shard_index)),
False)
class RunIsolatedTestOutputFiles(RunIsolatedTestBase):
def _run_test(self, isolated, command):
# Starts a full isolate server mock and have run_tha_test() uploads results
# back after the task completed.
server = isolateserver_mock.MockIsolateServer()
try:
# Output two files. If we're on Linux, we'll try to make one of them a
# symlink to ensure that we correctly follow symlinks. Note that this only
# tests file symlinks, not directory symlinks.
# TODO(aludwin): follow directory symlinks
script = (
'import os\n'
'import sys\n'
'open(sys.argv[1], "w").write("bar")\n'
'if sys.platform.startswith("linux"):\n'
' realpath = os.path.abspath("contents_of_symlink")\n'
' open(realpath, "w").write("baz")\n'
' os.symlink(realpath, sys.argv[2])\n'
'else:\n'
' open(sys.argv[2], "w").write("baz")\n')
script_hash = isolateserver_mock.hash_content(script)
isolated['files']['cmd.py'] = {
'h': script_hash,
'm': 0700,
's': len(script),
}
if sys.platform == 'win32':
isolated['files']['cmd.py'].pop('m')
isolated_data = json_dumps(isolated)
isolated_hash = isolateserver_mock.hash_content(isolated_data)
server.add_content('default-store', script)
server.add_content('default-store', isolated_data)
store = isolateserver.get_storage(server.url, 'default-store')
self.mock(sys, 'stdout', StringIO.StringIO())
ret = run_isolated.run_tha_test(
command,
isolated_hash,
store,
isolateserver.MemoryCache(),
['foo', 'foodir/foo2'],
init_named_caches_stub,
False,
None,
None,
None,
None,
None,
run_isolated.noop_install_packages,
False)
self.assertEqual(0, ret)
# It uploaded back. Assert the store has a new item containing foo.
hashes = {isolated_hash, script_hash}
foo_output_hash = isolateserver_mock.hash_content('bar')
foo2_output_hash = isolateserver_mock.hash_content('baz')
hashes.add(foo_output_hash)
hashes.add(foo2_output_hash)
isolated = {
'algo': 'sha-1',
'files': {
'foo': {
'h': foo_output_hash,
# TODO(maruel): Handle umask.
'm': 0640,
's': 3,
},
'foodir/foo2': {
'h': foo2_output_hash,
# TODO(maruel): Handle umask.
'm': 0640,
's': 3,
},
},
'version': isolated_format.ISOLATED_FILE_VERSION,
}
class RunIsolatedTestOutputFiles(RunIsolatedTestBase):
# Like RunIsolatedTestRun, but ensures that specific output files
# (as opposed to anything in $(ISOLATED_OUTDIR)) are returned.
def _run_test(self, isolated, command, extra_args):
# Starts a full isolate server mock and have run_tha_test() uploads results
# back after the task completed.
server = isolateserver_mock.MockIsolateServer()
try:
# Output two files. If we're on Linux, we'll try to make one of them a
# symlink to ensure that we correctly follow symlinks. Note that this only
# tests file symlinks, not directory symlinks.
# TODO(aludwin): follow directory symlinks
script = (
'import os\n'
'import sys\n'
'open(sys.argv[1], "w").write("bar")\n'
'if sys.platform.startswith("linux"):\n'
' realpath = os.path.abspath("contents_of_symlink")\n'
' open(realpath, "w").write("baz")\n'
' os.symlink(realpath, sys.argv[2])\n'
'else:\n'
' open(sys.argv[2], "w").write("baz")\n')
script_hash = isolateserver_mock.hash_content(script)
isolated['files']['cmd.py'] = {
'h': script_hash,
'm': 0700,
's': len(script),
}
if sys.platform == 'win32':
isolated['files']['cmd.py'].pop('m')
isolated_data = json_dumps(isolated)
isolated_hash = isolateserver_mock.hash_content(isolated_data)
server.add_content('default-store', script)
server.add_content('default-store', isolated_data)
store = isolateserver.get_storage(server.url, 'default-store')
self.mock(sys, 'stdout', StringIO.StringIO())
data = run_isolated.TaskData(
command=command,
relative_cwd=None,
extra_args=extra_args,
isolated_hash=isolated_hash,
storage=store,
isolate_cache=isolateserver.MemoryCache(),
outputs=['foo', 'foodir/foo2'],
install_named_caches=init_named_caches_stub,
leak_temp_dir=False,
root_dir=None,
hard_timeout=60,
grace_period=30,
bot_file=None,
switch_to_account=False,
install_packages_fn=run_isolated.noop_install_packages,
use_symlinks=False,
env={},
env_prefix={})
ret = run_isolated.run_tha_test(data, None)
self.assertEqual(0, ret)
# It uploaded back. Assert the store has a new item containing foo.
hashes = {isolated_hash, script_hash}
foo_output_hash = isolateserver_mock.hash_content('bar')
foo2_output_hash = isolateserver_mock.hash_content('baz')
hashes.add(foo_output_hash)
hashes.add(foo2_output_hash)
isolated = {
u'algo': u'sha-1',
u'files': {
u'foo': {
u'h': foo_output_hash,
# TODO(maruel): Handle umask.
u'm': 0640,
u's': 3,
},
u'foodir/foo2': {
u'h': foo2_output_hash,
# TODO(maruel): Handle umask.
u'm': 0640,
u's': 3,
},
},
u'version': isolated_format.ISOLATED_FILE_VERSION,
}
class RunIsolatedTestRun(RunIsolatedTestBase):
# Runs the actual command requested.
def test_output(self):
# Starts a full isolate server mock and have run_tha_test() uploads results
# back after the task completed.
server = isolateserver_mock.MockIsolateServer()
try:
script = (
'import sys\n'
'open(sys.argv[1], "w").write("bar")\n')
script_hash = isolateserver_mock.hash_content(script)
isolated = {
u'algo': u'sha-1',
u'command': [u'cmd.py', u'${ISOLATED_OUTDIR}/foo'],
u'files': {
u'cmd.py': {
u'h': script_hash,
u'm': 0700,
u's': len(script),
},
},
u'version': isolated_format.ISOLATED_FILE_VERSION,
}
if sys.platform == 'win32':
isolated[u'files'][u'cmd.py'].pop(u'm')
isolated_data = json_dumps(isolated)
isolated_hash = isolateserver_mock.hash_content(isolated_data)
server.add_content('default-store', script)
server.add_content('default-store', isolated_data)
store = isolateserver.get_storage(server.url, 'default-store')
self.mock(sys, 'stdout', StringIO.StringIO())
data = run_isolated.TaskData(
command=[],
relative_cwd=None,
extra_args=[],
isolated_hash=isolated_hash,
storage=store,
isolate_cache=isolateserver.MemoryCache(),
outputs=None,
install_named_caches=init_named_caches_stub,
leak_temp_dir=False,
root_dir=None,
hard_timeout=60,
grace_period=30,
bot_file=None,
switch_to_account=False,
install_packages_fn=run_isolated.noop_install_packages,
use_symlinks=False,
env={},
env_prefix={})
ret = run_isolated.run_tha_test(data, None)
self.assertEqual(0, ret)
# It uploaded back. Assert the store has a new item containing foo.
hashes = {isolated_hash, script_hash}
output_hash = isolateserver_mock.hash_content('bar')
hashes.add(output_hash)
isolated = {
u'algo': u'sha-1',
u'files': {
u'foo': {
u'h': output_hash,
# TODO(maruel): Handle umask.
u'm': 0640,
u's': 3,
},
},
u'version': isolated_format.ISOLATED_FILE_VERSION,
}
