def _archive_given_files(self, files):
        """Given a list of files, call isolateserver_archive.py with them. Then
    verify they are all on the server."""
        args = [
            sys.executable,
            os.path.join(ROOT_DIR, 'isolateserver_archive.py'), '--remote',
            ISOLATE_SERVER, '--namespace', self.namespace
        ]
        if '-v' in sys.argv:
            args.append('--verbose')
        args.extend(
            os.path.join(TEST_DATA_DIR, filename) for filename in files)

        self.assertEqual(0, subprocess.call(args))

        # Try to download the files from the server.
        file_hashes = [
            isolateserver_archive.sha1_file(os.path.join(TEST_DATA_DIR, f))
            for f in files
        ]
        for i in range(len(files)):
            download_url = '%scontent/retrieve/%s/%s' % (
                ISOLATE_SERVER, self.namespace, file_hashes[i])

            downloaded_file = isolateserver_archive.url_open(download_url,
                                                             retry_404=True),
            self.assertTrue(downloaded_file is not None,
                            'File %s was missing from the server' % files[i])

        # Ensure the files are listed as present on the server.
        contains_hash_url = '%scontent/contains/%s?token=%s&from_smoke_test=1' % (
            ISOLATE_SERVER, self.namespace, self.token)

        body = ''.join(binascii.unhexlify(h) for h in file_hashes)
        expected = chr(1) * len(files)
        MAX_ATTEMPTS = 10
        for i in xrange(MAX_ATTEMPTS):
            # AppEngine's database is eventually consistent and isolateserver do not
            # use transaction for performance reasons, so even if one request was able
            # to retrieve the file, an subsequent may not see it! So retry a few time
            # until the database becomes consistent with regard to these entities.
            response = isolateserver_archive.url_open(
                contains_hash_url,
                data=body,
                content_type='application/octet-stream').read()
            if response == expected:
                break
            # GAE is exposing its internal data inconsistency.
            if i != (MAX_ATTEMPTS - 1):
                print('Visible datastore inconsistency, retrying.')
                time.sleep(0.1)
        self.assertEqual(expected, response)
  def _archive_given_files(self, files):
    """Given a list of files, call isolateserver_archive.py with them. Then
    verify they are all on the server."""
    args = [
        sys.executable,
        os.path.join(ROOT_DIR, 'isolateserver_archive.py'),
        '--remote', ISOLATE_SERVER,
        '--namespace', self.namespace
    ]
    if '-v' in sys.argv:
      args.append('--verbose')
    args.extend(os.path.join(TEST_DATA_DIR, filename) for filename in files)

    self.assertEqual(0, subprocess.call(args))

    # Try to download the files from the server.
    file_hashes = [
        isolateserver_archive.sha1_file(os.path.join(TEST_DATA_DIR, f))
        for f in files
    ]
    for i in range(len(files)):
      download_url = '%scontent/retrieve/%s/%s' % (
          ISOLATE_SERVER, self.namespace, file_hashes[i])

      downloaded_file = isolateserver_archive.url_read(download_url,
                                                       retry_404=True),
      self.assertTrue(downloaded_file is not None,
                      'File %s was missing from the server' % files[i])

    # Ensure the files are listed as present on the server.
    contains_hash_url = '%scontent/contains/%s?token=%s&from_smoke_test=1' % (
        ISOLATE_SERVER, self.namespace, self.token)

    body = ''.join(binascii.unhexlify(h) for h in file_hashes)
    expected = chr(1) * len(files)
    MAX_ATTEMPTS = 10
    for i in xrange(MAX_ATTEMPTS):
      # AppEngine's database is eventually consistent and isolateserver do not
      # use transaction for performance reasons, so even if one request was able
      # to retrieve the file, an subsequent may not see it! So retry a few time
      # until the database becomes consistent with regard to these entities.
      response = isolateserver_archive.url_read(
          contains_hash_url,
          data=body,
          content_type='application/octet-stream')
      if response == expected:
        break
      # GAE is exposing its internal data inconsistency.
      if i != (MAX_ATTEMPTS - 1):
        print('Visible datastore inconsistency, retrying.')
        time.sleep(0.1)
    self.assertEqual(expected, response)
Beispiel #3
0
    def _archive_given_files(self, files):
        """Given a list of files, call isolateserver_archive.py with them. Then
    verify they are all on the server."""
        args = [
            sys.executable,
            os.path.join(ROOT_DIR, "isolateserver_archive.py"),
            "--remote",
            ISOLATE_SERVER,
            "--namespace",
            self.namespace,
        ]
        if "-v" in sys.argv:
            args.append("--verbose")
        args.extend(os.path.join(TEST_DATA_DIR, filename) for filename in files)

        self.assertEqual(0, subprocess.call(args))

        # Try to download the files from the server.
        file_hashes = [isolateserver_archive.sha1_file(os.path.join(TEST_DATA_DIR, f)) for f in files]
        for i in range(len(files)):
            download_url = "%scontent/retrieve/%s/%s" % (ISOLATE_SERVER, self.namespace, file_hashes[i])

            downloaded_file = (isolateserver_archive.url_open(download_url, retry_404=True),)
            self.assertTrue(downloaded_file is not None, "File %s was missing from the server" % files[i])

        # Ensure the files are listed as present on the server.
        contains_hash_url = "%scontent/contains/%s?token=%s&from_smoke_test=1" % (
            ISOLATE_SERVER,
            self.namespace,
            self.token,
        )

        body = "".join(binascii.unhexlify(h) for h in file_hashes)
        expected = chr(1) * len(files)
        for _ in xrange(10):
            # AppEngine's database is eventually consistent and isolateserver do not
            # use transaction for performance reasons, so even if one request was able
            # to retrieve the file, an subsequent may not see it! So retry a few time
            # until the database becomes consistent with regard to these entities.
            response = isolateserver_archive.url_open(
                contains_hash_url, data=body, content_type="application/octet-stream"
            ).read()
            if response == expected:
                break
        self.assertEquals(expected, response)
Beispiel #4
0
 def test_present(self):
   files = [
     os.path.join(BASE_PATH, 'isolateserver_archive', f)
     for f in ('small_file.txt', 'empty_file.txt')
   ]
   sha1encoded = ''.join(
       binascii.unhexlify(isolateserver_archive.sha1_file(f)) for f in files)
   path = 'http://random/'
   self._requests = [
     (path + 'content/get_token', {}, StringIO.StringIO('foo bar')),
     (
       path + 'content/contains/default-gzip?token=foo%20bar',
       {'data': sha1encoded, 'content_type': 'application/octet-stream'},
       StringIO.StringIO('\1\1'),
     ),
   ]
   result = isolateserver_archive.main(['--remote', path] + files)
   self.assertEqual(0, result)