def _archive_given_files(self, files):
"""Given a list of files, call isolateserver_archive.py with them. Then
verify they are all on the server."""
args = [
sys.executable,
os.path.join(ROOT_DIR, 'isolateserver_archive.py'), '--remote',
ISOLATE_SERVER, '--namespace', self.namespace
]
if '-v' in sys.argv:
args.append('--verbose')
args.extend(
os.path.join(TEST_DATA_DIR, filename) for filename in files)
self.assertEqual(0, subprocess.call(args))
# Try to download the files from the server.
file_hashes = [
isolateserver_archive.sha1_file(os.path.join(TEST_DATA_DIR, f))
for f in files
]
for i in range(len(files)):
download_url = '%scontent/retrieve/%s/%s' % (
ISOLATE_SERVER, self.namespace, file_hashes[i])
downloaded_file = isolateserver_archive.url_open(download_url,
retry_404=True),
self.assertTrue(downloaded_file is not None,
'File %s was missing from the server' % files[i])
# Ensure the files are listed as present on the server.
contains_hash_url = '%scontent/contains/%s?token=%s&from_smoke_test=1' % (
ISOLATE_SERVER, self.namespace, self.token)
body = ''.join(binascii.unhexlify(h) for h in file_hashes)
expected = chr(1) * len(files)
MAX_ATTEMPTS = 10
for i in xrange(MAX_ATTEMPTS):
# AppEngine's database is eventually consistent and isolateserver do not
# use transaction for performance reasons, so even if one request was able
# to retrieve the file, an subsequent may not see it! So retry a few time
# until the database becomes consistent with regard to these entities.
response = isolateserver_archive.url_open(
contains_hash_url,
data=body,
content_type='application/octet-stream').read()
if response == expected:
break
# GAE is exposing its internal data inconsistency.
if i != (MAX_ATTEMPTS - 1):
print('Visible datastore inconsistency, retrying.')
time.sleep(0.1)
self.assertEqual(expected, response)
def _archive_given_files(self, files):
"""Given a list of files, call isolateserver_archive.py with them. Then
verify they are all on the server."""
args = [
sys.executable,
os.path.join(ROOT_DIR, 'isolateserver_archive.py'),
'--remote', ISOLATE_SERVER,
'--namespace', self.namespace
]
if '-v' in sys.argv:
args.append('--verbose')
args.extend(os.path.join(TEST_DATA_DIR, filename) for filename in files)
self.assertEqual(0, subprocess.call(args))
# Try to download the files from the server.
file_hashes = [
isolateserver_archive.sha1_file(os.path.join(TEST_DATA_DIR, f))
for f in files
]
for i in range(len(files)):
download_url = '%scontent/retrieve/%s/%s' % (
ISOLATE_SERVER, self.namespace, file_hashes[i])
downloaded_file = isolateserver_archive.url_read(download_url,
retry_404=True),
self.assertTrue(downloaded_file is not None,
'File %s was missing from the server' % files[i])
# Ensure the files are listed as present on the server.
contains_hash_url = '%scontent/contains/%s?token=%s&from_smoke_test=1' % (
ISOLATE_SERVER, self.namespace, self.token)
body = ''.join(binascii.unhexlify(h) for h in file_hashes)
expected = chr(1) * len(files)
MAX_ATTEMPTS = 10
for i in xrange(MAX_ATTEMPTS):
# AppEngine's database is eventually consistent and isolateserver do not
# use transaction for performance reasons, so even if one request was able
# to retrieve the file, an subsequent may not see it! So retry a few time
# until the database becomes consistent with regard to these entities.
response = isolateserver_archive.url_read(
contains_hash_url,
data=body,
content_type='application/octet-stream')
if response == expected:
break
# GAE is exposing its internal data inconsistency.
if i != (MAX_ATTEMPTS - 1):
print('Visible datastore inconsistency, retrying.')
time.sleep(0.1)
self.assertEqual(expected, response)
def _archive_given_files(self, files):
"""Given a list of files, call isolateserver_archive.py with them. Then
verify they are all on the server."""
args = [
sys.executable,
os.path.join(ROOT_DIR, "isolateserver_archive.py"),
"--remote",
ISOLATE_SERVER,
"--namespace",
self.namespace,
]
if "-v" in sys.argv:
args.append("--verbose")
args.extend(os.path.join(TEST_DATA_DIR, filename) for filename in files)
self.assertEqual(0, subprocess.call(args))
# Try to download the files from the server.
file_hashes = [isolateserver_archive.sha1_file(os.path.join(TEST_DATA_DIR, f)) for f in files]
for i in range(len(files)):
download_url = "%scontent/retrieve/%s/%s" % (ISOLATE_SERVER, self.namespace, file_hashes[i])
downloaded_file = (isolateserver_archive.url_open(download_url, retry_404=True),)
self.assertTrue(downloaded_file is not None, "File %s was missing from the server" % files[i])
# Ensure the files are listed as present on the server.
contains_hash_url = "%scontent/contains/%s?token=%s&from_smoke_test=1" % (
ISOLATE_SERVER,
self.namespace,
self.token,
)
body = "".join(binascii.unhexlify(h) for h in file_hashes)
expected = chr(1) * len(files)
for _ in xrange(10):
# AppEngine's database is eventually consistent and isolateserver do not
# use transaction for performance reasons, so even if one request was able
# to retrieve the file, an subsequent may not see it! So retry a few time
# until the database becomes consistent with regard to these entities.
response = isolateserver_archive.url_open(
contains_hash_url, data=body, content_type="application/octet-stream"
).read()
if response == expected:
break
self.assertEquals(expected, response)
def test_present(self):
files = [
os.path.join(BASE_PATH, 'isolateserver_archive', f)
for f in ('small_file.txt', 'empty_file.txt')
]
sha1encoded = ''.join(
binascii.unhexlify(isolateserver_archive.sha1_file(f)) for f in files)
path = 'http://random/'
self._requests = [
(path + 'content/get_token', {}, StringIO.StringIO('foo bar')),
(
path + 'content/contains/default-gzip?token=foo%20bar',
{'data': sha1encoded, 'content_type': 'application/octet-stream'},
StringIO.StringIO('\1\1'),
),
]
result = isolateserver_archive.main(['--remote', path] + files)
self.assertEqual(0, result)