def test_archive_no_server(self): with self.assertRaises(SystemExit): isolateserver.main(['archive', '.']) prog = self.get_isolateserver_prog() self.checkOutput( '', 'Usage: %(prog)s archive [options] <file1..fileN> or - to read ' 'from stdin\n\n' '%(prog)s: error: --isolate-server is required.\n' % {'prog': prog})
def test_archive_no_server(self): with self.assertRaises(SystemExit): isolateserver.main(['archive', '.']) prog = os.path.basename(sys.modules[isolateserver.__name__].__file__) self.checkOutput( '', 'Usage: %(prog)s archive [options] <file1..fileN> or - to read ' 'from stdin\n\n' '%(prog)s: error: --isolate-server is required.\n' % {'prog': prog})
def test_archive_files(self): self.mock(isolateserver, 'get_storage', get_storage) self.make_tree(CONTENTS) f = ['empty_file.txt', 'small_file.txt'] os.chdir(self.tempdir) isolateserver.main( ['archive', '--isolate-server', 'https://localhost:1'] + f) self.checkOutput( 'da39a3ee5e6b4b0d3255bfef95601890afd80709 empty_file.txt\n' '0491bd1da8087ad10fcdd7c9634e308804b72158 small_file.txt\n', '')
def test_archive_duplicates(self): with self.assertRaises(SystemExit): isolateserver.main( [ 'archive', '--isolate-server', 'https://localhost:1', # Effective dupes. '.', os.getcwd(), ]) prog = self.get_isolateserver_prog() self.checkOutput( '', 'Usage: %(prog)s archive [options] <file1..fileN> or - to read ' 'from stdin\n\n' '%(prog)s: error: Duplicate entries found.\n' % {'prog': prog})
def test_archive_files(self): old_cwd = os.getcwd() try: os.chdir(os.path.join(TEST_DIR, 'isolateserver')) self.mock(isolateserver, 'get_storage', get_storage) f = ['empty_file.txt', 'small_file.txt'] isolateserver.main( ['archive', '--isolate-server', 'https://localhost:1'] + f) self.checkOutput( 'da39a3ee5e6b4b0d3255bfef95601890afd80709 empty_file.txt\n' '0491bd1da8087ad10fcdd7c9634e308804b72158 small_file.txt\n', '') finally: os.chdir(old_cwd)
def test_download_isolated(self): # Test downloading an isolated tree. self.tempdir = tempfile.mkdtemp(prefix='isolateserver') actual = {} def file_write_mock(key, generator): actual[key] = ''.join(generator) self.mock(isolateserver, 'file_write', file_write_mock) self.mock(os, 'makedirs', lambda _: None) stdout = StringIO.StringIO() self.mock(sys, 'stdout', stdout) server = 'http://example.com' files = { 'a/foo': 'Content', 'b': 'More content', } isolated = { 'command': ['Absurb', 'command'], 'relative_cwd': 'a', 'files': dict((k, { 'h': ALGO(v).hexdigest(), 's': len(v) }) for k, v in files.iteritems()), } isolated_data = json.dumps(isolated, sort_keys=True, separators=(',', ':')) isolated_hash = ALGO(isolated_data).hexdigest() requests = [(v['h'], files[k]) for k, v in isolated['files'].iteritems()] requests.append((isolated_hash, isolated_data)) self._requests = [( server + '/content-gs/retrieve/default-gzip/' + h, { 'read_timeout': isolateserver.DOWNLOAD_READ_TIMEOUT, 'retry_404': True, }, zlib.compress(v), ) for h, v in requests] cmd = [ 'download', '--isolate-server', server, '--target', self.tempdir, '--isolated', isolated_hash, ] self.assertEqual(0, isolateserver.main(cmd)) expected = dict( (os.path.join(self.tempdir, k), v) for k, v in files.iteritems()) self.assertEqual(expected, actual) expected_stdout = ( 'To run this test please run from the directory %s:\n Absurb command\n' % os.path.join(self.tempdir, 'a')) self.assertEqual(expected_stdout, stdout.getvalue())
def test_download_two_files(self): # Test downloading two files. actual = {} def out(key, generator): actual[key] = ''.join(generator) self.mock(isolateserver, 'file_write', out) server = 'http://example.com' self._requests = [ ( server + '/content-gs/retrieve/default-gzip/sha-1', {'read_timeout': 60, 'retry_404': True, 'headers': None}, zlib.compress('Coucou'), None, ), ( server + '/content-gs/retrieve/default-gzip/sha-2', {'read_timeout': 60, 'retry_404': True, 'headers': None}, zlib.compress('Bye Bye'), None, ), ] cmd = [ 'download', '--isolate-server', server, '--target', ROOT_DIR, '--file', 'sha-1', 'path/to/a', '--file', 'sha-2', 'path/to/b', ] self.assertEqual(0, isolateserver.main(cmd)) expected = { os.path.join(ROOT_DIR, 'path/to/a'): 'Coucou', os.path.join(ROOT_DIR, 'path/to/b'): 'Bye Bye', } self.assertEqual(expected, actual)
def help_test_archive(self, cmd_line_prefix): old_cwd = os.getcwd() try: os.chdir(ROOT_DIR) self.mock(isolateserver, 'get_storage', get_storage) p = os.path.join(TEST_DIR, 'isolateserver') isolateserver.main(cmd_line_prefix + [p]) # TODO(maruel): The problem here is that the test depends on the file mode # of the files in this directory. # Fix is to copy the files in a temporary directory with known file modes. # # If you modify isolateserver.ISOLATED_FILE_VERSION, you'll have to update # the hash below. Sorry about that. self.checkOutput( '1501166255279df1509408567340798d1cf089e7 %s\n' % p, '') finally: os.chdir(old_cwd)
def test_download_isolated_simple(self): # Test downloading an isolated tree. actual = {} def putfile_mock( srcfileobj, dstpath, file_mode=None, size=-1, use_symlink=False): actual[dstpath] = srcfileobj.read() self.mock(isolateserver, 'putfile', putfile_mock) self.mock(os, 'makedirs', lambda _: None) server = 'http://example.com' files = { os.path.join('a', 'foo'): 'Content', 'b': 'More content', } isolated = { 'command': ['Absurb', 'command'], 'relative_cwd': 'a', 'files': dict( (k, {'h': isolateserver_mock.hash_content(v), 's': len(v)}) for k, v in files.iteritems()), 'version': isolated_format.ISOLATED_FILE_VERSION, } isolated_data = json.dumps(isolated, sort_keys=True, separators=(',',':')) isolated_hash = isolateserver_mock.hash_content(isolated_data) requests = [(v['h'], files[k]) for k, v in isolated['files'].iteritems()] requests.append((isolated_hash, isolated_data)) requests = [ ( server + '/api/isolateservice/v1/retrieve', { 'data': { 'digest': h.encode('utf-8'), 'namespace': { 'namespace': 'default-gzip', 'digest_hash': 'sha-1', 'compression': 'flate', }, 'offset': 0, }, 'read_timeout': 60, }, {'content': base64.b64encode(zlib.compress(v))}, ) for h, v in requests ] cmd = [ 'download', '--isolate-server', server, '--target', self.tempdir, '--isolated', isolated_hash, ] self.expected_requests(requests) self.assertEqual(0, isolateserver.main(cmd)) expected = dict( (os.path.join(self.tempdir, k), v) for k, v in files.iteritems()) self.assertEqual(expected, actual) expected_stdout = ( 'To run this test please run from the directory %s:\n Absurb command\n' % os.path.join(self.tempdir, 'a')) self.checkOutput(expected_stdout, '')
def help_test_archive(self, cmd_line_prefix): old_cwd = os.getcwd() try: os.chdir(net_utils.ROOT_DIR) self.mock(isolateserver, 'get_storage', get_storage) p = os.path.join(net_utils.TEST_DIR, 'isolateserver') isolateserver.main(cmd_line_prefix + [p]) # TODO(maruel): The problem here is that the test depends on the file mode # of the files in this directory. # Fix is to copy the files in a temporary directory with known file modes. # # If you modify isolateserver.ISOLATED_FILE_VERSION, you'll have to update # the hash below. Sorry about that. self.checkOutput( '1501166255279df1509408567340798d1cf089e7 %s\n' % p, '') finally: os.chdir(old_cwd)
def help_test_archive(self, cmd_line_prefix): old_cwd = os.getcwd() try: os.chdir(ROOT_DIR) self.mock(isolateserver, 'get_storage', get_storage) p = os.path.join(TEST_DIR, 'isolateserver') isolateserver.main(cmd_line_prefix + [p]) # TODO(maruel): The problem here is that the test depends on the file mode # of the files in this directory. # Fix is to copy the files in a temporary directory with known file modes. # # If you modify isolateserver.ISOLATED_FILE_VERSION, you'll have to update # the hash below. Sorry about that. self.checkOutput( '189dbab83102b8ebcff92c1332a25fe26c1a5d7d %s\n' % p, '') finally: os.chdir(old_cwd)
def test_download_isolated(self): # Test downloading an isolated tree. actual = {} def file_write_mock(key, generator): actual[key] = ''.join(generator) self.mock(isolateserver, 'file_write', file_write_mock) self.mock(os, 'makedirs', lambda _: None) server = 'http://example.com' files = { os.path.join('a', 'foo'): 'Content', 'b': 'More content', } isolated = { 'command': ['Absurb', 'command'], 'relative_cwd': 'a', 'files': dict( (k, {'h': isolateserver_mock.hash_content(v), 's': len(v)}) for k, v in files.iteritems()), 'version': isolated_format.ISOLATED_FILE_VERSION, } isolated_data = json.dumps(isolated, sort_keys=True, separators=(',',':')) isolated_hash = isolateserver_mock.hash_content(isolated_data) requests = [(v['h'], files[k]) for k, v in isolated['files'].iteritems()] requests.append((isolated_hash, isolated_data)) requests = [ ( server + '/_ah/api/isolateservice/v1/retrieve', { 'data': { 'digest': h.encode('utf-8'), 'namespace': { 'namespace': 'default-gzip', 'digest_hash': 'sha-1', 'compression': 'flate', }, 'offset': 0, }, 'read_timeout': 60, }, {'content': base64.b64encode(zlib.compress(v))}, ) for h, v in requests ] cmd = [ 'download', '--isolate-server', server, '--target', self.tempdir, '--isolated', isolated_hash, ] self.expected_requests(requests) self.assertEqual(0, isolateserver.main(cmd)) expected = dict( (os.path.join(self.tempdir, k), v) for k, v in files.iteritems()) self.assertEqual(expected, actual) expected_stdout = ( 'To run this test please run from the directory %s:\n Absurb command\n' % os.path.join(self.tempdir, 'a')) self.checkOutput(expected_stdout, '')
def help_test_archive(self, cmd_line_prefix): self.mock(isolateserver, 'get_storage', get_storage) self.make_tree(CONTENTS) isolateserver.main(cmd_line_prefix + [self.tempdir]) isolated = { 'algo': 'sha-1', 'files': {}, 'version': isolated_format.ISOLATED_FILE_VERSION, } for k, v in CONTENTS.items(): isolated['files'][k] = { 'h': isolateserver_fake.hash_content(v), 's': len(v), } if sys.platform != 'win32': isolated['files'][k]['m'] = 0o600 isolated_data = json.dumps(isolated, sort_keys=True, separators=(',', ':')) isolated_hash = isolateserver_fake.hash_content(isolated_data) self.checkOutput( '%s %s\n' % (isolated_hash, self.tempdir), '')
def test_download_isolated(self): # Test downloading an isolated tree. self.tempdir = tempfile.mkdtemp(prefix='isolateserver') actual = {} def file_write_mock(key, generator): actual[key] = ''.join(generator) self.mock(isolateserver, 'file_write', file_write_mock) self.mock(os, 'makedirs', lambda _: None) server = 'http://example.com' files = { os.path.join('a', 'foo'): 'Content', 'b': 'More content', } isolated = { 'command': ['Absurb', 'command'], 'relative_cwd': 'a', 'files': dict( (k, {'h': ALGO(v).hexdigest(), 's': len(v)}) for k, v in files.iteritems()), 'version': isolateserver.ISOLATED_FILE_VERSION, } isolated_data = json.dumps(isolated, sort_keys=True, separators=(',',':')) isolated_hash = ALGO(isolated_data).hexdigest() requests = [(v['h'], files[k]) for k, v in isolated['files'].iteritems()] requests.append((isolated_hash, isolated_data)) requests = [ ( server + '/content-gs/retrieve/default-gzip/' + h, { 'read_timeout': isolateserver.DOWNLOAD_READ_TIMEOUT, 'headers': None, }, zlib.compress(v), None, ) for h, v in requests ] cmd = [ 'download', '--isolate-server', server, '--target', self.tempdir, '--isolated', isolated_hash, ] self.expected_requests(requests) self.assertEqual(0, isolateserver.main(cmd)) expected = dict( (os.path.join(self.tempdir, k), v) for k, v in files.iteritems()) self.assertEqual(expected, actual) expected_stdout = ( 'To run this test please run from the directory %s:\n Absurb command\n' % os.path.join(self.tempdir, 'a')) self.checkOutput(expected_stdout, '')
def test_download_two_files(self): # Test downloading two files. actual = {} def out(key, generator): actual[key] = ''.join(generator) self.mock(local_caching, 'file_write', out) server = 'http://example.com' coucou_sha1 = hashlib.sha1('Coucou').hexdigest() byebye_sha1 = hashlib.sha1('Bye Bye').hexdigest() requests = [( server + '/_ah/api/isolateservice/v1/retrieve', { 'data': { 'digest': h.encode('utf-8'), 'namespace': { 'namespace': 'default-gzip', 'digest_hash': 'sha-1', 'compression': 'flate', }, 'offset': 0, }, 'read_timeout': 60, }, { 'content': base64.b64encode(zlib.compress(v)) }, ) for h, v in [(coucou_sha1, 'Coucou'), (byebye_sha1, 'Bye Bye')]] self.expected_requests(requests) cmd = [ 'download', '--isolate-server', server, '--target', net_utils.ROOT_DIR, '--file', coucou_sha1, 'path/to/a', '--file', byebye_sha1, 'path/to/b', ] self.assertEqual(0, isolateserver.main(cmd)) expected = { os.path.join(net_utils.ROOT_DIR, 'path/to/a'): 'Coucou', os.path.join(net_utils.ROOT_DIR, 'path/to/b'): 'Bye Bye', } self.assertEqual(expected, actual)
def help_test_archive(self, cmd_line_prefix): self.mock(isolateserver, 'get_storage', get_storage) self.make_tree(CONTENTS) isolateserver.main(cmd_line_prefix + [self.tempdir]) # If you modify isolated_format.ISOLATED_FILE_VERSION, you'll have to update # the hash below. Sorry about that but this ensures the .isolated format is # stable. isolated = { 'algo': 'sha-1', 'files': {}, 'version': isolated_format.ISOLATED_FILE_VERSION, } for k, v in CONTENTS.iteritems(): isolated['files'][k] = { 'h': isolateserver_mock.hash_content(v), 's': len(v), } if sys.platform != 'win32': isolated['files'][k]['m'] = 0600 isolated_data = json.dumps(isolated, sort_keys=True, separators=(',', ':')) isolated_hash = isolateserver_mock.hash_content(isolated_data) self.checkOutput('%s %s\n' % (isolated_hash, self.tempdir), '')
def help_test_archive(self, cmd_line_prefix): self.mock(isolateserver, 'get_storage', get_storage) self.make_tree(CONTENTS) isolateserver.main(cmd_line_prefix + [self.tempdir]) # If you modify isolated_format.ISOLATED_FILE_VERSION, you'll have to update # the hash below. Sorry about that but this ensures the .isolated format is # stable. isolated = { 'algo': 'sha-1', 'files': {}, 'version': isolated_format.ISOLATED_FILE_VERSION, } for k, v in CONTENTS.iteritems(): isolated['files'][k] = { 'h': isolateserver_mock.hash_content(v), 's': len(v), } if sys.platform != 'win32': isolated['files'][k]['m'] = 0600 isolated_data = json.dumps(isolated, sort_keys=True, separators=(',',':')) isolated_hash = isolateserver_mock.hash_content(isolated_data) self.checkOutput( '%s %s\n' % (isolated_hash, self.tempdir), '')
def test_download_two_files(self): # Test downloading two files. actual = {} def out(key, generator): actual[key] = ''.join(generator) self.mock(isolateserver, 'file_write', out) server = 'http://example.com' self._requests = [ ( server + '/content-gs/retrieve/default-gzip/sha-1', { 'read_timeout': 60, 'headers': None }, zlib.compress('Coucou'), None, ), ( server + '/content-gs/retrieve/default-gzip/sha-2', { 'read_timeout': 60, 'headers': None }, zlib.compress('Bye Bye'), None, ), ] cmd = [ 'download', '--isolate-server', server, '--target', ROOT_DIR, '--file', 'sha-1', 'path/to/a', '--file', 'sha-2', 'path/to/b', ] self.assertEqual(0, isolateserver.main(cmd)) expected = { os.path.join(ROOT_DIR, 'path/to/a'): 'Coucou', os.path.join(ROOT_DIR, 'path/to/b'): 'Bye Bye', } self.assertEqual(expected, actual)
def test_download_two_files(self): # Test downloading two files. # It doesn't touch disk, 'file_write' is mocked. # It doesn't touch network, url_open() is mocked. actual = {} def out(key, generator): actual[key] = ''.join(generator) self.mock(local_caching, 'file_write', out) server_ref = isolate_storage.ServerRef('http://example.com', 'default-gzip') coucou_sha1 = hashlib.sha1('Coucou').hexdigest() byebye_sha1 = hashlib.sha1('Bye Bye').hexdigest() requests = [ ( '%s/_ah/api/isolateservice/v1/retrieve' % server_ref.url, { 'data': { 'digest': h.encode('utf-8'), 'namespace': { 'namespace': 'default-gzip', 'digest_hash': 'sha-1', 'compression': 'flate', }, 'offset': 0, }, 'read_timeout': 60, }, {'content': base64.b64encode(zlib.compress(v))}, ) for h, v in [(coucou_sha1, 'Coucou'), (byebye_sha1, 'Bye Bye')] ] self.expected_requests(requests) cmd = [ 'download', '--isolate-server', server_ref.url, '--namespace', server_ref.namespace, '--target', test_env.CLIENT_DIR, '--file', coucou_sha1, 'path/to/a', '--file', byebye_sha1, 'path/to/b', # Even if everything is mocked, the cache directory will still be created. '--cache', self.tempdir, ] self.assertEqual(0, isolateserver.main(cmd)) expected = { os.path.join(test_env.CLIENT_DIR, 'path/to/a'): 'Coucou', os.path.join(test_env.CLIENT_DIR, 'path/to/b'): 'Bye Bye', } self.assertEqual(expected, actual)
def test_download_two_files(self): # Test downloading two files. actual = {} def out(key, generator): actual[key] = ''.join(generator) self.mock(isolateserver, 'file_write', out) server = 'http://example.com' requests = [ ( server + '/_ah/api/isolateservice/v1/retrieve', { 'data': { 'digest': h.encode('utf-8'), 'namespace': { 'namespace': 'default-gzip', 'digest_hash': 'sha-1', 'compression': 'flate', }, 'offset': 0, }, 'read_timeout': 60, }, {'content': base64.b64encode(zlib.compress(v))}, ) for h, v in [('sha-1', 'Coucou'), ('sha-2', 'Bye Bye')] ] self.expected_requests(requests) cmd = [ 'download', '--isolate-server', server, '--target', net_utils.ROOT_DIR, '--file', 'sha-1', 'path/to/a', '--file', 'sha-2', 'path/to/b', ] self.assertEqual(0, isolateserver.main(cmd)) expected = { os.path.join(net_utils.ROOT_DIR, 'path/to/a'): 'Coucou', os.path.join(net_utils.ROOT_DIR, 'path/to/b'): 'Bye Bye', } self.assertEqual(expected, actual)
def test_download_isolated_simple(self): # Test downloading an isolated tree. # It writes files to disk for real. server_ref = isolate_storage.ServerRef('http://example.com', 'default-gzip') files = { os.path.join('a', 'foo'): 'Content', 'b': 'More content', } isolated = { 'command': ['Absurb', 'command'], 'relative_cwd': 'a', 'files': { os.path.join('a', 'foo'): { 'h': isolateserver_fake.hash_content('Content'), 's': len('Content'), 'm': 0o700, }, 'b': { 'h': isolateserver_fake.hash_content('More content'), 's': len('More content'), 'm': 0o600, }, 'c': { 'l': 'a/foo', }, }, 'read_only': 1, 'version': isolated_format.ISOLATED_FILE_VERSION, } isolated_data = json.dumps(isolated, sort_keys=True, separators=(',', ':')) isolated_hash = isolateserver_fake.hash_content(isolated_data) requests = [ (v['h'], files[k]) for k, v in isolated['files'].items() if 'h' in v ] requests.append((isolated_hash, isolated_data)) requests = [ ( '%s/_ah/api/isolateservice/v1/retrieve' % server_ref.url, { 'data': { 'digest': h.encode('utf-8'), 'namespace': { 'namespace': 'default-gzip', 'digest_hash': 'sha-1', 'compression': 'flate', }, 'offset': 0, }, 'read_timeout': 60, }, {'content': base64.b64encode(zlib.compress(v))}, ) for h, v in requests ] cmd = [ 'download', '--isolate-server', server_ref.url, '--namespace', server_ref.namespace, '--target', os.path.join(self.tempdir, 'target'), '--isolated', isolated_hash, '--cache', os.path.join(self.tempdir, 'cache'), ] self.expected_requests(requests) self.assertEqual(0, isolateserver.main(cmd)) expected = { os.path.join(self.tempdir, 'target', 'a', 'foo'): ('Content', 0o500), os.path.join(self.tempdir, 'target', 'b'): ('More content', 0o400), os.path.join(self.tempdir, 'target', 'c'): (u'a/foo', 0), } actual = self._get_actual() self.assertEqual(expected, actual) expected_stdout = ( 'To run this test please run from the directory %s:\n Absurb command\n' % os.path.join(self.tempdir, 'target', 'a')) self.checkOutput(expected_stdout, '')
def test_download_isolated_tar_archive(self): # Test downloading an isolated tree. actual = {} def putfile_mock( srcfileobj, dstpath, file_mode=None, size=-1, use_symlink=False): actual[dstpath] = srcfileobj.read(size) self.mock(isolateserver, 'putfile', putfile_mock) self.mock(os, 'makedirs', lambda _: None) server = 'http://example.com' files = { os.path.join('a', 'foo'): 'Content', 'b': 'More content', 'c': 'Even more content!', } # Generate a tar archive tf = io.BytesIO() with tarfile.TarFile(mode='w', fileobj=tf) as tar: f1 = tarfile.TarInfo() f1.type = tarfile.REGTYPE f1.name = 'a/foo' f1.size = 7 tar.addfile(f1, io.BytesIO('Content')) f2 = tarfile.TarInfo() f2.type = tarfile.REGTYPE f2.name = 'b' f2.size = 12 tar.addfile(f2, io.BytesIO('More content')) archive = tf.getvalue() isolated = { 'command': ['Absurb', 'command'], 'relative_cwd': 'a', 'files': { 'archive1': { 'h': isolateserver_mock.hash_content(archive), 's': len(archive), 't': 'tar', }, 'c': { 'h': isolateserver_mock.hash_content(files['c']), 's': len(files['c']), }, }, 'version': isolated_format.ISOLATED_FILE_VERSION, } isolated_data = json.dumps(isolated, sort_keys=True, separators=(',',':')) isolated_hash = isolateserver_mock.hash_content(isolated_data) requests = [ (isolated['files']['archive1']['h'], archive), (isolated['files']['c']['h'], files['c']), ] requests.append((isolated_hash, isolated_data)) requests = [ ( server + '/api/isolateservice/v1/retrieve', { 'data': { 'digest': h.encode('utf-8'), 'namespace': { 'namespace': 'default-gzip', 'digest_hash': 'sha-1', 'compression': 'flate', }, 'offset': 0, }, 'read_timeout': 60, }, {'content': base64.b64encode(zlib.compress(v))}, ) for h, v in requests ] cmd = [ 'download', '--isolate-server', server, '--target', self.tempdir, '--isolated', isolated_hash, ] self.expected_requests(requests) self.assertEqual(0, isolateserver.main(cmd)) expected = dict( (os.path.join(self.tempdir, k), v) for k, v in files.iteritems()) self.assertEqual(expected, actual) expected_stdout = ( 'To run this test please run from the directory %s:\n Absurb command\n' % os.path.join(self.tempdir, 'a')) self.checkOutput(expected_stdout, '')
}, { 'content': base64.b64encode(zlib.compress(v)) }, ) for h, v in requests] cmd = [ 'download', '--isolate-server', server, '--target', self.tempdir, '--isolated', isolated_hash, ] self.expected_requests(requests) self.assertEqual(0, isolateserver.main(cmd)) expected = { os.path.join(self.tempdir, 'a', 'foo'): ('Content', 0500), os.path.join(self.tempdir, 'b'): ('More content', 0400), os.path.join(self.tempdir, 'c'): (u'a/foo', 0), } actual = self._get_actual() self.assertEqual(expected, actual) expected_stdout = ( 'To run this test please run from the directory %s:\n Absurb command\n' % os.path.join(self.tempdir, 'a')) self.checkOutput(expected_stdout, '') def test_download_isolated_tar_archive(self): # Test downloading an isolated tree. server = 'http://example.com'
def test_download_isolated_ar_archive(self): # Test downloading an isolated tree. actual = {} def putfile_mock( srcfileobj, dstpath, file_mode=None, size=-1, use_symlink=False): actual[dstpath] = srcfileobj.read(size) self.mock(isolateserver, 'putfile', putfile_mock) self.mock(os, 'makedirs', lambda _: None) server = 'http://example.com' files = { os.path.join('a', 'foo'): 'Content', 'b': 'More content', 'c': 'Even more content!', } archive = ( # ar file header '!<arch>\n' # File 1 ------------------------- # (16 bytes) filename len '#1/5 ' # file metadata '1447140471 1000 1000 100640 ' # (10 bytes) Data size '12 ' # (2 bytes) File magic '\x60\n' # (5 bytes) File name 'a/foo' # (7 bytes) File data 'Content' # File 2 ------------------------- # (16 bytes) filename 'b ' # file metadata '1447140471 1000 1000 100640 ' # (12 bytes) Data size '12 ' # (2 bytes) File magic '\x60\n' # (12 bytes) File data 'More content' '') isolated = { 'command': ['Absurb', 'command'], 'relative_cwd': 'a', 'files': { 'archive1': { 'h': isolateserver_mock.hash_content(archive), 's': len(archive), 't': 'ar', }, 'c': { 'h': isolateserver_mock.hash_content(files['c']), 's': len(files['c']), }, }, 'version': isolated_format.ISOLATED_FILE_VERSION, } isolated_data = json.dumps(isolated, sort_keys=True, separators=(',',':')) isolated_hash = isolateserver_mock.hash_content(isolated_data) requests = [ (isolated['files']['archive1']['h'], archive), (isolated['files']['c']['h'], files['c']), ] requests.append((isolated_hash, isolated_data)) requests = [ ( server + '/api/isolateservice/v1/retrieve', { 'data': { 'digest': h.encode('utf-8'), 'namespace': { 'namespace': 'default-gzip', 'digest_hash': 'sha-1', 'compression': 'flate', }, 'offset': 0, }, 'read_timeout': 60, }, {'content': base64.b64encode(zlib.compress(v))}, ) for h, v in requests ] cmd = [ 'download', '--isolate-server', server, '--target', self.tempdir, '--isolated', isolated_hash, ] self.expected_requests(requests) self.assertEqual(0, isolateserver.main(cmd)) expected = dict( (os.path.join(self.tempdir, k), v) for k, v in files.iteritems()) self.assertEqual(expected, actual) expected_stdout = ( 'To run this test please run from the directory %s:\n Absurb command\n' % os.path.join(self.tempdir, 'a')) self.checkOutput(expected_stdout, '')
def test_download_isolated_tar_archive(self): # Test downloading an isolated tree. server_ref = isolate_storage.ServerRef('http://example.com', 'default-gzip') files = { os.path.join('a', 'foo'): ('Content', 0o500), 'b': ('More content', 0o400), 'c': ('Even more content!', 0o500), } # Generate a tar archive tf = io.BytesIO() with tarfile.TarFile(mode='w', fileobj=tf) as tar: f1 = tarfile.TarInfo() f1.type = tarfile.REGTYPE f1.name = 'a/foo' f1.size = 7 f1.mode = 0o570 tar.addfile(f1, io.BytesIO('Content')) f2 = tarfile.TarInfo() f2.type = tarfile.REGTYPE f2.name = 'b' f2.size = 12 f2.mode = 0o666 tar.addfile(f2, io.BytesIO('More content')) archive = tf.getvalue() isolated = { 'command': ['Absurb', 'command'], 'relative_cwd': 'a', 'files': { 'archive1': { 'h': isolateserver_fake.hash_content(archive), 's': len(archive), 't': 'tar', }, 'c': { 'h': isolateserver_fake.hash_content(files['c'][0]), 's': len(files['c'][0]), }, }, 'read_only': 1, 'version': isolated_format.ISOLATED_FILE_VERSION, } isolated_data = json.dumps(isolated, sort_keys=True, separators=(',', ':')) isolated_hash = isolateserver_fake.hash_content(isolated_data) requests = [ (isolated['files']['archive1']['h'], archive), (isolated['files']['c']['h'], files['c'][0]), ] requests.append((isolated_hash, isolated_data)) requests = [ ( '%s/_ah/api/isolateservice/v1/retrieve' % server_ref.url, { 'data': { 'digest': h.encode('utf-8'), 'namespace': { 'namespace': 'default-gzip', 'digest_hash': 'sha-1', 'compression': 'flate', }, 'offset': 0, }, 'read_timeout': 60, }, {'content': base64.b64encode(zlib.compress(v))}, ) for h, v in requests ] cmd = [ 'download', '--isolate-server', server_ref.url, '--namespace', server_ref.namespace, '--target', os.path.join(self.tempdir, 'target'), '--isolated', isolated_hash, '--cache', os.path.join(self.tempdir, 'cache'), ] self.expected_requests(requests) self.assertEqual(0, isolateserver.main(cmd)) expected = { os.path.join(self.tempdir, 'target', k): v for k, v in files.items() } actual = self._get_actual() self.assertEqual(expected, actual) expected_stdout = ( 'To run this test please run from the directory %s:\n Absurb command\n' % os.path.join(self.tempdir, 'target', 'a')) self.checkOutput(expected_stdout, '')