def test_archive_no_server(self):
with self.assertRaises(SystemExit):
isolateserver.main(['archive', '.'])
prog = self.get_isolateserver_prog()
self.checkOutput(
'',
'Usage: %(prog)s archive [options] <file1..fileN> or - to read '
'from stdin\n\n'
'%(prog)s: error: --isolate-server is required.\n' % {'prog': prog})
def test_archive_no_server(self):
with self.assertRaises(SystemExit):
isolateserver.main(['archive', '.'])
prog = os.path.basename(sys.modules[isolateserver.__name__].__file__)
self.checkOutput(
'',
'Usage: %(prog)s archive [options] <file1..fileN> or - to read '
'from stdin\n\n'
'%(prog)s: error: --isolate-server is required.\n' % {'prog': prog})
def test_archive_no_server(self):
with self.assertRaises(SystemExit):
isolateserver.main(['archive', '.'])
prog = self.get_isolateserver_prog()
self.checkOutput(
'',
'Usage: %(prog)s archive [options] <file1..fileN> or - to read '
'from stdin\n\n'
'%(prog)s: error: --isolate-server is required.\n' % {'prog': prog})
def test_archive_files(self):
self.mock(isolateserver, 'get_storage', get_storage)
self.make_tree(CONTENTS)
f = ['empty_file.txt', 'small_file.txt']
os.chdir(self.tempdir)
isolateserver.main(
['archive', '--isolate-server', 'https://localhost:1'] + f)
self.checkOutput(
'da39a3ee5e6b4b0d3255bfef95601890afd80709 empty_file.txt\n'
'0491bd1da8087ad10fcdd7c9634e308804b72158 small_file.txt\n', '')
def test_archive_files(self):
self.mock(isolateserver, 'get_storage', get_storage)
self.make_tree(CONTENTS)
f = ['empty_file.txt', 'small_file.txt']
os.chdir(self.tempdir)
isolateserver.main(
['archive', '--isolate-server', 'https://localhost:1'] + f)
self.checkOutput(
'da39a3ee5e6b4b0d3255bfef95601890afd80709 empty_file.txt\n'
'0491bd1da8087ad10fcdd7c9634e308804b72158 small_file.txt\n',
'')
def test_archive_duplicates(self):
with self.assertRaises(SystemExit):
isolateserver.main(
[
'archive', '--isolate-server', 'https://localhost:1',
# Effective dupes.
'.', os.getcwd(),
])
prog = self.get_isolateserver_prog()
self.checkOutput(
'',
'Usage: %(prog)s archive [options] <file1..fileN> or - to read '
'from stdin\n\n'
'%(prog)s: error: Duplicate entries found.\n' % {'prog': prog})
def test_archive_files(self):
old_cwd = os.getcwd()
try:
os.chdir(os.path.join(TEST_DIR, 'isolateserver'))
self.mock(isolateserver, 'get_storage', get_storage)
f = ['empty_file.txt', 'small_file.txt']
isolateserver.main(
['archive', '--isolate-server', 'https://localhost:1'] + f)
self.checkOutput(
'da39a3ee5e6b4b0d3255bfef95601890afd80709 empty_file.txt\n'
'0491bd1da8087ad10fcdd7c9634e308804b72158 small_file.txt\n',
'')
finally:
os.chdir(old_cwd)
def test_archive_duplicates(self):
with self.assertRaises(SystemExit):
isolateserver.main(
[
'archive', '--isolate-server', 'https://localhost:1',
# Effective dupes.
'.', os.getcwd(),
])
prog = self.get_isolateserver_prog()
self.checkOutput(
'',
'Usage: %(prog)s archive [options] <file1..fileN> or - to read '
'from stdin\n\n'
'%(prog)s: error: Duplicate entries found.\n' % {'prog': prog})
def test_archive_files(self):
old_cwd = os.getcwd()
try:
os.chdir(os.path.join(TEST_DIR, 'isolateserver'))
self.mock(isolateserver, 'get_storage', get_storage)
f = ['empty_file.txt', 'small_file.txt']
isolateserver.main(
['archive', '--isolate-server', 'https://localhost:1'] + f)
self.checkOutput(
'da39a3ee5e6b4b0d3255bfef95601890afd80709 empty_file.txt\n'
'0491bd1da8087ad10fcdd7c9634e308804b72158 small_file.txt\n',
'')
finally:
os.chdir(old_cwd)
def test_download_isolated(self):
# Test downloading an isolated tree.
self.tempdir = tempfile.mkdtemp(prefix='isolateserver')
actual = {}
def file_write_mock(key, generator):
actual[key] = ''.join(generator)
self.mock(isolateserver, 'file_write', file_write_mock)
self.mock(os, 'makedirs', lambda _: None)
stdout = StringIO.StringIO()
self.mock(sys, 'stdout', stdout)
server = 'http://example.com'
files = {
'a/foo': 'Content',
'b': 'More content',
}
isolated = {
'command': ['Absurb', 'command'],
'relative_cwd':
'a',
'files':
dict((k, {
'h': ALGO(v).hexdigest(),
's': len(v)
}) for k, v in files.iteritems()),
}
isolated_data = json.dumps(isolated,
sort_keys=True,
separators=(',', ':'))
isolated_hash = ALGO(isolated_data).hexdigest()
requests = [(v['h'], files[k])
for k, v in isolated['files'].iteritems()]
requests.append((isolated_hash, isolated_data))
self._requests = [(
server + '/content-gs/retrieve/default-gzip/' + h,
{
'read_timeout': isolateserver.DOWNLOAD_READ_TIMEOUT,
'retry_404': True,
},
zlib.compress(v),
) for h, v in requests]
cmd = [
'download',
'--isolate-server',
server,
'--target',
self.tempdir,
'--isolated',
isolated_hash,
]
self.assertEqual(0, isolateserver.main(cmd))
expected = dict(
(os.path.join(self.tempdir, k), v) for k, v in files.iteritems())
self.assertEqual(expected, actual)
expected_stdout = (
'To run this test please run from the directory %s:\n Absurb command\n'
% os.path.join(self.tempdir, 'a'))
self.assertEqual(expected_stdout, stdout.getvalue())
def test_download_two_files(self):
# Test downloading two files.
actual = {}
def out(key, generator):
actual[key] = ''.join(generator)
self.mock(isolateserver, 'file_write', out)
server = 'http://example.com'
self._requests = [
(
server + '/content-gs/retrieve/default-gzip/sha-1',
{'read_timeout': 60, 'retry_404': True, 'headers': None},
zlib.compress('Coucou'),
None,
),
(
server + '/content-gs/retrieve/default-gzip/sha-2',
{'read_timeout': 60, 'retry_404': True, 'headers': None},
zlib.compress('Bye Bye'),
None,
),
]
cmd = [
'download',
'--isolate-server', server,
'--target', ROOT_DIR,
'--file', 'sha-1', 'path/to/a',
'--file', 'sha-2', 'path/to/b',
]
self.assertEqual(0, isolateserver.main(cmd))
expected = {
os.path.join(ROOT_DIR, 'path/to/a'): 'Coucou',
os.path.join(ROOT_DIR, 'path/to/b'): 'Bye Bye',
}
self.assertEqual(expected, actual)
def help_test_archive(self, cmd_line_prefix):
old_cwd = os.getcwd()
try:
os.chdir(ROOT_DIR)
self.mock(isolateserver, 'get_storage', get_storage)
p = os.path.join(TEST_DIR, 'isolateserver')
isolateserver.main(cmd_line_prefix + [p])
# TODO(maruel): The problem here is that the test depends on the file mode
# of the files in this directory.
# Fix is to copy the files in a temporary directory with known file modes.
#
# If you modify isolateserver.ISOLATED_FILE_VERSION, you'll have to update
# the hash below. Sorry about that.
self.checkOutput(
'1501166255279df1509408567340798d1cf089e7 %s\n' % p, '')
finally:
os.chdir(old_cwd)
def test_download_isolated_simple(self):
# Test downloading an isolated tree.
actual = {}
def putfile_mock(
srcfileobj, dstpath, file_mode=None, size=-1, use_symlink=False):
actual[dstpath] = srcfileobj.read()
self.mock(isolateserver, 'putfile', putfile_mock)
self.mock(os, 'makedirs', lambda _: None)
server = 'http://example.com'
files = {
os.path.join('a', 'foo'): 'Content',
'b': 'More content',
}
isolated = {
'command': ['Absurb', 'command'],
'relative_cwd': 'a',
'files': dict(
(k, {'h': isolateserver_mock.hash_content(v), 's': len(v)})
for k, v in files.iteritems()),
'version': isolated_format.ISOLATED_FILE_VERSION,
}
isolated_data = json.dumps(isolated, sort_keys=True, separators=(',',':'))
isolated_hash = isolateserver_mock.hash_content(isolated_data)
requests = [(v['h'], files[k]) for k, v in isolated['files'].iteritems()]
requests.append((isolated_hash, isolated_data))
requests = [
(
server + '/api/isolateservice/v1/retrieve',
{
'data': {
'digest': h.encode('utf-8'),
'namespace': {
'namespace': 'default-gzip',
'digest_hash': 'sha-1',
'compression': 'flate',
},
'offset': 0,
},
'read_timeout': 60,
},
{'content': base64.b64encode(zlib.compress(v))},
) for h, v in requests
]
cmd = [
'download',
'--isolate-server', server,
'--target', self.tempdir,
'--isolated', isolated_hash,
]
self.expected_requests(requests)
self.assertEqual(0, isolateserver.main(cmd))
expected = dict(
(os.path.join(self.tempdir, k), v) for k, v in files.iteritems())
self.assertEqual(expected, actual)
expected_stdout = (
'To run this test please run from the directory %s:\n Absurb command\n'
% os.path.join(self.tempdir, 'a'))
self.checkOutput(expected_stdout, '')
def help_test_archive(self, cmd_line_prefix):
old_cwd = os.getcwd()
try:
os.chdir(net_utils.ROOT_DIR)
self.mock(isolateserver, 'get_storage', get_storage)
p = os.path.join(net_utils.TEST_DIR, 'isolateserver')
isolateserver.main(cmd_line_prefix + [p])
# TODO(maruel): The problem here is that the test depends on the file mode
# of the files in this directory.
# Fix is to copy the files in a temporary directory with known file modes.
#
# If you modify isolateserver.ISOLATED_FILE_VERSION, you'll have to update
# the hash below. Sorry about that.
self.checkOutput(
'1501166255279df1509408567340798d1cf089e7 %s\n' % p,
'')
finally:
os.chdir(old_cwd)
def help_test_archive(self, cmd_line_prefix):
old_cwd = os.getcwd()
try:
os.chdir(ROOT_DIR)
self.mock(isolateserver, 'get_storage', get_storage)
p = os.path.join(TEST_DIR, 'isolateserver')
isolateserver.main(cmd_line_prefix + [p])
# TODO(maruel): The problem here is that the test depends on the file mode
# of the files in this directory.
# Fix is to copy the files in a temporary directory with known file modes.
#
# If you modify isolateserver.ISOLATED_FILE_VERSION, you'll have to update
# the hash below. Sorry about that.
self.checkOutput(
'189dbab83102b8ebcff92c1332a25fe26c1a5d7d %s\n' % p,
'')
finally:
os.chdir(old_cwd)
def test_download_isolated(self):
# Test downloading an isolated tree.
actual = {}
def file_write_mock(key, generator):
actual[key] = ''.join(generator)
self.mock(isolateserver, 'file_write', file_write_mock)
self.mock(os, 'makedirs', lambda _: None)
server = 'http://example.com'
files = {
os.path.join('a', 'foo'): 'Content',
'b': 'More content',
}
isolated = {
'command': ['Absurb', 'command'],
'relative_cwd': 'a',
'files': dict(
(k, {'h': isolateserver_mock.hash_content(v), 's': len(v)})
for k, v in files.iteritems()),
'version': isolated_format.ISOLATED_FILE_VERSION,
}
isolated_data = json.dumps(isolated, sort_keys=True, separators=(',',':'))
isolated_hash = isolateserver_mock.hash_content(isolated_data)
requests = [(v['h'], files[k]) for k, v in isolated['files'].iteritems()]
requests.append((isolated_hash, isolated_data))
requests = [
(
server + '/_ah/api/isolateservice/v1/retrieve',
{
'data': {
'digest': h.encode('utf-8'),
'namespace': {
'namespace': 'default-gzip',
'digest_hash': 'sha-1',
'compression': 'flate',
},
'offset': 0,
},
'read_timeout': 60,
},
{'content': base64.b64encode(zlib.compress(v))},
) for h, v in requests
]
cmd = [
'download',
'--isolate-server', server,
'--target', self.tempdir,
'--isolated', isolated_hash,
]
self.expected_requests(requests)
self.assertEqual(0, isolateserver.main(cmd))
expected = dict(
(os.path.join(self.tempdir, k), v) for k, v in files.iteritems())
self.assertEqual(expected, actual)
expected_stdout = (
'To run this test please run from the directory %s:\n Absurb command\n'
% os.path.join(self.tempdir, 'a'))
self.checkOutput(expected_stdout, '')
def help_test_archive(self, cmd_line_prefix):
self.mock(isolateserver, 'get_storage', get_storage)
self.make_tree(CONTENTS)
isolateserver.main(cmd_line_prefix + [self.tempdir])
isolated = {
'algo': 'sha-1',
'files': {},
'version': isolated_format.ISOLATED_FILE_VERSION,
}
for k, v in CONTENTS.items():
isolated['files'][k] = {
'h': isolateserver_fake.hash_content(v),
's': len(v),
}
if sys.platform != 'win32':
isolated['files'][k]['m'] = 0o600
isolated_data = json.dumps(isolated, sort_keys=True, separators=(',', ':'))
isolated_hash = isolateserver_fake.hash_content(isolated_data)
self.checkOutput(
'%s %s\n' % (isolated_hash, self.tempdir),
'')
def test_download_isolated(self):
# Test downloading an isolated tree.
self.tempdir = tempfile.mkdtemp(prefix='isolateserver')
actual = {}
def file_write_mock(key, generator):
actual[key] = ''.join(generator)
self.mock(isolateserver, 'file_write', file_write_mock)
self.mock(os, 'makedirs', lambda _: None)
server = 'http://example.com'
files = {
os.path.join('a', 'foo'): 'Content',
'b': 'More content',
}
isolated = {
'command': ['Absurb', 'command'],
'relative_cwd': 'a',
'files': dict(
(k, {'h': ALGO(v).hexdigest(), 's': len(v)})
for k, v in files.iteritems()),
'version': isolateserver.ISOLATED_FILE_VERSION,
}
isolated_data = json.dumps(isolated, sort_keys=True, separators=(',',':'))
isolated_hash = ALGO(isolated_data).hexdigest()
requests = [(v['h'], files[k]) for k, v in isolated['files'].iteritems()]
requests.append((isolated_hash, isolated_data))
requests = [
(
server + '/content-gs/retrieve/default-gzip/' + h,
{
'read_timeout': isolateserver.DOWNLOAD_READ_TIMEOUT,
'headers': None,
},
zlib.compress(v),
None,
) for h, v in requests
]
cmd = [
'download',
'--isolate-server', server,
'--target', self.tempdir,
'--isolated', isolated_hash,
]
self.expected_requests(requests)
self.assertEqual(0, isolateserver.main(cmd))
expected = dict(
(os.path.join(self.tempdir, k), v) for k, v in files.iteritems())
self.assertEqual(expected, actual)
expected_stdout = (
'To run this test please run from the directory %s:\n Absurb command\n'
% os.path.join(self.tempdir, 'a'))
self.checkOutput(expected_stdout, '')
def test_download_two_files(self):
# Test downloading two files.
actual = {}
def out(key, generator):
actual[key] = ''.join(generator)
self.mock(local_caching, 'file_write', out)
server = 'http://example.com'
coucou_sha1 = hashlib.sha1('Coucou').hexdigest()
byebye_sha1 = hashlib.sha1('Bye Bye').hexdigest()
requests = [(
server + '/_ah/api/isolateservice/v1/retrieve',
{
'data': {
'digest': h.encode('utf-8'),
'namespace': {
'namespace': 'default-gzip',
'digest_hash': 'sha-1',
'compression': 'flate',
},
'offset': 0,
},
'read_timeout': 60,
},
{
'content': base64.b64encode(zlib.compress(v))
},
) for h, v in [(coucou_sha1, 'Coucou'), (byebye_sha1, 'Bye Bye')]]
self.expected_requests(requests)
cmd = [
'download',
'--isolate-server',
server,
'--target',
net_utils.ROOT_DIR,
'--file',
coucou_sha1,
'path/to/a',
'--file',
byebye_sha1,
'path/to/b',
]
self.assertEqual(0, isolateserver.main(cmd))
expected = {
os.path.join(net_utils.ROOT_DIR, 'path/to/a'): 'Coucou',
os.path.join(net_utils.ROOT_DIR, 'path/to/b'): 'Bye Bye',
}
self.assertEqual(expected, actual)
def help_test_archive(self, cmd_line_prefix):
self.mock(isolateserver, 'get_storage', get_storage)
self.make_tree(CONTENTS)
isolateserver.main(cmd_line_prefix + [self.tempdir])
# If you modify isolated_format.ISOLATED_FILE_VERSION, you'll have to update
# the hash below. Sorry about that but this ensures the .isolated format is
# stable.
isolated = {
'algo': 'sha-1',
'files': {},
'version': isolated_format.ISOLATED_FILE_VERSION,
}
for k, v in CONTENTS.iteritems():
isolated['files'][k] = {
'h': isolateserver_mock.hash_content(v),
's': len(v),
}
if sys.platform != 'win32':
isolated['files'][k]['m'] = 0600
isolated_data = json.dumps(isolated,
sort_keys=True,
separators=(',', ':'))
isolated_hash = isolateserver_mock.hash_content(isolated_data)
self.checkOutput('%s %s\n' % (isolated_hash, self.tempdir), '')
def help_test_archive(self, cmd_line_prefix):
self.mock(isolateserver, 'get_storage', get_storage)
self.make_tree(CONTENTS)
isolateserver.main(cmd_line_prefix + [self.tempdir])
# If you modify isolated_format.ISOLATED_FILE_VERSION, you'll have to update
# the hash below. Sorry about that but this ensures the .isolated format is
# stable.
isolated = {
'algo': 'sha-1',
'files': {},
'version': isolated_format.ISOLATED_FILE_VERSION,
}
for k, v in CONTENTS.iteritems():
isolated['files'][k] = {
'h': isolateserver_mock.hash_content(v),
's': len(v),
}
if sys.platform != 'win32':
isolated['files'][k]['m'] = 0600
isolated_data = json.dumps(isolated, sort_keys=True, separators=(',',':'))
isolated_hash = isolateserver_mock.hash_content(isolated_data)
self.checkOutput(
'%s %s\n' % (isolated_hash, self.tempdir),
'')
def test_download_two_files(self):
# Test downloading two files.
actual = {}
def out(key, generator):
actual[key] = ''.join(generator)
self.mock(isolateserver, 'file_write', out)
server = 'http://example.com'
self._requests = [
(
server + '/content-gs/retrieve/default-gzip/sha-1',
{
'read_timeout': 60,
'headers': None
},
zlib.compress('Coucou'),
None,
),
(
server + '/content-gs/retrieve/default-gzip/sha-2',
{
'read_timeout': 60,
'headers': None
},
zlib.compress('Bye Bye'),
None,
),
]
cmd = [
'download',
'--isolate-server',
server,
'--target',
ROOT_DIR,
'--file',
'sha-1',
'path/to/a',
'--file',
'sha-2',
'path/to/b',
]
self.assertEqual(0, isolateserver.main(cmd))
expected = {
os.path.join(ROOT_DIR, 'path/to/a'): 'Coucou',
os.path.join(ROOT_DIR, 'path/to/b'): 'Bye Bye',
}
self.assertEqual(expected, actual)
def test_download_two_files(self):
# Test downloading two files.
# It doesn't touch disk, 'file_write' is mocked.
# It doesn't touch network, url_open() is mocked.
actual = {}
def out(key, generator):
actual[key] = ''.join(generator)
self.mock(local_caching, 'file_write', out)
server_ref = isolate_storage.ServerRef('http://example.com', 'default-gzip')
coucou_sha1 = hashlib.sha1('Coucou').hexdigest()
byebye_sha1 = hashlib.sha1('Bye Bye').hexdigest()
requests = [
(
'%s/_ah/api/isolateservice/v1/retrieve' % server_ref.url,
{
'data': {
'digest': h.encode('utf-8'),
'namespace': {
'namespace': 'default-gzip',
'digest_hash': 'sha-1',
'compression': 'flate',
},
'offset': 0,
},
'read_timeout': 60,
},
{'content': base64.b64encode(zlib.compress(v))},
) for h, v in [(coucou_sha1, 'Coucou'), (byebye_sha1, 'Bye Bye')]
]
self.expected_requests(requests)
cmd = [
'download',
'--isolate-server', server_ref.url,
'--namespace', server_ref.namespace,
'--target', test_env.CLIENT_DIR,
'--file', coucou_sha1, 'path/to/a',
'--file', byebye_sha1, 'path/to/b',
# Even if everything is mocked, the cache directory will still be created.
'--cache', self.tempdir,
]
self.assertEqual(0, isolateserver.main(cmd))
expected = {
os.path.join(test_env.CLIENT_DIR, 'path/to/a'): 'Coucou',
os.path.join(test_env.CLIENT_DIR, 'path/to/b'): 'Bye Bye',
}
self.assertEqual(expected, actual)
def test_download_two_files(self):
# Test downloading two files.
actual = {}
def out(key, generator):
actual[key] = ''.join(generator)
self.mock(isolateserver, 'file_write', out)
server = 'http://example.com'
requests = [
(
server + '/_ah/api/isolateservice/v1/retrieve',
{
'data': {
'digest': h.encode('utf-8'),
'namespace': {
'namespace': 'default-gzip',
'digest_hash': 'sha-1',
'compression': 'flate',
},
'offset': 0,
},
'read_timeout': 60,
},
{'content': base64.b64encode(zlib.compress(v))},
) for h, v in [('sha-1', 'Coucou'), ('sha-2', 'Bye Bye')]
]
self.expected_requests(requests)
cmd = [
'download',
'--isolate-server', server,
'--target', net_utils.ROOT_DIR,
'--file', 'sha-1', 'path/to/a',
'--file', 'sha-2', 'path/to/b',
]
self.assertEqual(0, isolateserver.main(cmd))
expected = {
os.path.join(net_utils.ROOT_DIR, 'path/to/a'): 'Coucou',
os.path.join(net_utils.ROOT_DIR, 'path/to/b'): 'Bye Bye',
}
self.assertEqual(expected, actual)
def test_download_isolated_simple(self):
# Test downloading an isolated tree.
# It writes files to disk for real.
server_ref = isolate_storage.ServerRef('http://example.com', 'default-gzip')
files = {
os.path.join('a', 'foo'): 'Content',
'b': 'More content',
}
isolated = {
'command': ['Absurb', 'command'],
'relative_cwd': 'a',
'files': {
os.path.join('a', 'foo'): {
'h': isolateserver_fake.hash_content('Content'),
's': len('Content'),
'm': 0o700,
},
'b': {
'h': isolateserver_fake.hash_content('More content'),
's': len('More content'),
'm': 0o600,
},
'c': {
'l': 'a/foo',
},
},
'read_only': 1,
'version': isolated_format.ISOLATED_FILE_VERSION,
}
isolated_data = json.dumps(isolated, sort_keys=True, separators=(',', ':'))
isolated_hash = isolateserver_fake.hash_content(isolated_data)
requests = [
(v['h'], files[k]) for k, v in isolated['files'].items()
if 'h' in v
]
requests.append((isolated_hash, isolated_data))
requests = [
(
'%s/_ah/api/isolateservice/v1/retrieve' % server_ref.url,
{
'data': {
'digest': h.encode('utf-8'),
'namespace': {
'namespace': 'default-gzip',
'digest_hash': 'sha-1',
'compression': 'flate',
},
'offset': 0,
},
'read_timeout': 60,
},
{'content': base64.b64encode(zlib.compress(v))},
) for h, v in requests
]
cmd = [
'download',
'--isolate-server', server_ref.url,
'--namespace', server_ref.namespace,
'--target', os.path.join(self.tempdir, 'target'),
'--isolated', isolated_hash,
'--cache', os.path.join(self.tempdir, 'cache'),
]
self.expected_requests(requests)
self.assertEqual(0, isolateserver.main(cmd))
expected = {
os.path.join(self.tempdir, 'target', 'a', 'foo'): ('Content', 0o500),
os.path.join(self.tempdir, 'target', 'b'): ('More content', 0o400),
os.path.join(self.tempdir, 'target', 'c'): (u'a/foo', 0),
}
actual = self._get_actual()
self.assertEqual(expected, actual)
expected_stdout = (
'To run this test please run from the directory %s:\n Absurb command\n'
% os.path.join(self.tempdir, 'target', 'a'))
self.checkOutput(expected_stdout, '')
def test_download_isolated_tar_archive(self):
# Test downloading an isolated tree.
actual = {}
def putfile_mock(
srcfileobj, dstpath, file_mode=None, size=-1, use_symlink=False):
actual[dstpath] = srcfileobj.read(size)
self.mock(isolateserver, 'putfile', putfile_mock)
self.mock(os, 'makedirs', lambda _: None)
server = 'http://example.com'
files = {
os.path.join('a', 'foo'): 'Content',
'b': 'More content',
'c': 'Even more content!',
}
# Generate a tar archive
tf = io.BytesIO()
with tarfile.TarFile(mode='w', fileobj=tf) as tar:
f1 = tarfile.TarInfo()
f1.type = tarfile.REGTYPE
f1.name = 'a/foo'
f1.size = 7
tar.addfile(f1, io.BytesIO('Content'))
f2 = tarfile.TarInfo()
f2.type = tarfile.REGTYPE
f2.name = 'b'
f2.size = 12
tar.addfile(f2, io.BytesIO('More content'))
archive = tf.getvalue()
isolated = {
'command': ['Absurb', 'command'],
'relative_cwd': 'a',
'files': {
'archive1': {
'h': isolateserver_mock.hash_content(archive),
's': len(archive),
't': 'tar',
},
'c': {
'h': isolateserver_mock.hash_content(files['c']),
's': len(files['c']),
},
},
'version': isolated_format.ISOLATED_FILE_VERSION,
}
isolated_data = json.dumps(isolated, sort_keys=True, separators=(',',':'))
isolated_hash = isolateserver_mock.hash_content(isolated_data)
requests = [
(isolated['files']['archive1']['h'], archive),
(isolated['files']['c']['h'], files['c']),
]
requests.append((isolated_hash, isolated_data))
requests = [
(
server + '/api/isolateservice/v1/retrieve',
{
'data': {
'digest': h.encode('utf-8'),
'namespace': {
'namespace': 'default-gzip',
'digest_hash': 'sha-1',
'compression': 'flate',
},
'offset': 0,
},
'read_timeout': 60,
},
{'content': base64.b64encode(zlib.compress(v))},
) for h, v in requests
]
cmd = [
'download',
'--isolate-server', server,
'--target', self.tempdir,
'--isolated', isolated_hash,
]
self.expected_requests(requests)
self.assertEqual(0, isolateserver.main(cmd))
expected = dict(
(os.path.join(self.tempdir, k), v) for k, v in files.iteritems())
self.assertEqual(expected, actual)
expected_stdout = (
'To run this test please run from the directory %s:\n Absurb command\n'
% os.path.join(self.tempdir, 'a'))
self.checkOutput(expected_stdout, '')
},
{
'content': base64.b64encode(zlib.compress(v))
},
) for h, v in requests]
cmd = [
'download',
'--isolate-server',
server,
'--target',
self.tempdir,
'--isolated',
isolated_hash,
]
self.expected_requests(requests)
self.assertEqual(0, isolateserver.main(cmd))
expected = {
os.path.join(self.tempdir, 'a', 'foo'): ('Content', 0500),
os.path.join(self.tempdir, 'b'): ('More content', 0400),
os.path.join(self.tempdir, 'c'): (u'a/foo', 0),
}
actual = self._get_actual()
self.assertEqual(expected, actual)
expected_stdout = (
'To run this test please run from the directory %s:\n Absurb command\n'
% os.path.join(self.tempdir, 'a'))
self.checkOutput(expected_stdout, '')
def test_download_isolated_tar_archive(self):
# Test downloading an isolated tree.
server = 'http://example.com'
def test_download_isolated_ar_archive(self):
# Test downloading an isolated tree.
actual = {}
def putfile_mock(
srcfileobj, dstpath, file_mode=None, size=-1, use_symlink=False):
actual[dstpath] = srcfileobj.read(size)
self.mock(isolateserver, 'putfile', putfile_mock)
self.mock(os, 'makedirs', lambda _: None)
server = 'http://example.com'
files = {
os.path.join('a', 'foo'): 'Content',
'b': 'More content',
'c': 'Even more content!',
}
archive = (
# ar file header
'!<arch>\n'
# File 1 -------------------------
# (16 bytes) filename len
'#1/5 '
# file metadata
'1447140471 1000 1000 100640 '
# (10 bytes) Data size
'12 '
# (2 bytes) File magic
'\x60\n'
# (5 bytes) File name
'a/foo'
# (7 bytes) File data
'Content'
# File 2 -------------------------
# (16 bytes) filename
'b '
# file metadata
'1447140471 1000 1000 100640 '
# (12 bytes) Data size
'12 '
# (2 bytes) File magic
'\x60\n'
# (12 bytes) File data
'More content'
'')
isolated = {
'command': ['Absurb', 'command'],
'relative_cwd': 'a',
'files': {
'archive1': {
'h': isolateserver_mock.hash_content(archive),
's': len(archive),
't': 'ar',
},
'c': {
'h': isolateserver_mock.hash_content(files['c']),
's': len(files['c']),
},
},
'version': isolated_format.ISOLATED_FILE_VERSION,
}
isolated_data = json.dumps(isolated, sort_keys=True, separators=(',',':'))
isolated_hash = isolateserver_mock.hash_content(isolated_data)
requests = [
(isolated['files']['archive1']['h'], archive),
(isolated['files']['c']['h'], files['c']),
]
requests.append((isolated_hash, isolated_data))
requests = [
(
server + '/api/isolateservice/v1/retrieve',
{
'data': {
'digest': h.encode('utf-8'),
'namespace': {
'namespace': 'default-gzip',
'digest_hash': 'sha-1',
'compression': 'flate',
},
'offset': 0,
},
'read_timeout': 60,
},
{'content': base64.b64encode(zlib.compress(v))},
) for h, v in requests
]
cmd = [
'download',
'--isolate-server', server,
'--target', self.tempdir,
'--isolated', isolated_hash,
]
self.expected_requests(requests)
self.assertEqual(0, isolateserver.main(cmd))
expected = dict(
(os.path.join(self.tempdir, k), v) for k, v in files.iteritems())
self.assertEqual(expected, actual)
expected_stdout = (
'To run this test please run from the directory %s:\n Absurb command\n'
% os.path.join(self.tempdir, 'a'))
self.checkOutput(expected_stdout, '')
def test_download_isolated_tar_archive(self):
# Test downloading an isolated tree.
server_ref = isolate_storage.ServerRef('http://example.com', 'default-gzip')
files = {
os.path.join('a', 'foo'): ('Content', 0o500),
'b': ('More content', 0o400),
'c': ('Even more content!', 0o500),
}
# Generate a tar archive
tf = io.BytesIO()
with tarfile.TarFile(mode='w', fileobj=tf) as tar:
f1 = tarfile.TarInfo()
f1.type = tarfile.REGTYPE
f1.name = 'a/foo'
f1.size = 7
f1.mode = 0o570
tar.addfile(f1, io.BytesIO('Content'))
f2 = tarfile.TarInfo()
f2.type = tarfile.REGTYPE
f2.name = 'b'
f2.size = 12
f2.mode = 0o666
tar.addfile(f2, io.BytesIO('More content'))
archive = tf.getvalue()
isolated = {
'command': ['Absurb', 'command'],
'relative_cwd': 'a',
'files': {
'archive1': {
'h': isolateserver_fake.hash_content(archive),
's': len(archive),
't': 'tar',
},
'c': {
'h': isolateserver_fake.hash_content(files['c'][0]),
's': len(files['c'][0]),
},
},
'read_only': 1,
'version': isolated_format.ISOLATED_FILE_VERSION,
}
isolated_data = json.dumps(isolated, sort_keys=True, separators=(',', ':'))
isolated_hash = isolateserver_fake.hash_content(isolated_data)
requests = [
(isolated['files']['archive1']['h'], archive),
(isolated['files']['c']['h'], files['c'][0]),
]
requests.append((isolated_hash, isolated_data))
requests = [
(
'%s/_ah/api/isolateservice/v1/retrieve' % server_ref.url,
{
'data': {
'digest': h.encode('utf-8'),
'namespace': {
'namespace': 'default-gzip',
'digest_hash': 'sha-1',
'compression': 'flate',
},
'offset': 0,
},
'read_timeout': 60,
},
{'content': base64.b64encode(zlib.compress(v))},
) for h, v in requests
]
cmd = [
'download',
'--isolate-server', server_ref.url,
'--namespace', server_ref.namespace,
'--target', os.path.join(self.tempdir, 'target'),
'--isolated', isolated_hash,
'--cache', os.path.join(self.tempdir, 'cache'),
]
self.expected_requests(requests)
self.assertEqual(0, isolateserver.main(cmd))
expected = {
os.path.join(self.tempdir, 'target', k): v for k, v in files.items()
}
actual = self._get_actual()
self.assertEqual(expected, actual)
expected_stdout = (
'To run this test please run from the directory %s:\n Absurb command\n'
% os.path.join(self.tempdir, 'target', 'a'))
self.checkOutput(expected_stdout, '')
