Example #1
0
    def test_save_named(self):
        cache = self.get_cache(_get_policies())
        self.assertEqual([], sorted(fs.listdir(cache.cache_dir)))

        self._add_one_item(cache, 2)
        with fs.open(os.path.join(cache.cache_dir, cache.STATE_FILE)) as f:
            old_content = json.load(f)
        # It's immediately saved.
        items = lru.LRUDict.load(
            os.path.join(cache.cache_dir, cache.STATE_FILE))
        self.assertEqual(1, len(items))
        _key, (v, _timestamp) = items.get_oldest()
        # This depends on the inner format as generated by NamedCache.
        entry_dir_name = v[0]
        self.assertEqual(
            sorted([entry_dir_name, cache.NAMED_DIR, cache.STATE_FILE]),
            sorted(fs.listdir(cache.cache_dir)))

        cache.save()
        self.assertEqual(
            sorted([entry_dir_name, cache.NAMED_DIR, cache.STATE_FILE]),
            sorted(fs.listdir(cache.cache_dir)))
        with fs.open(os.path.join(cache.cache_dir, cache.STATE_FILE)) as f:
            new_content = json.load(f)
        # That's because uninstall() called from self._add_one_item()
        # causes an implicit save(). See uninstall() comments for more details.
        self.assertEqual(new_content, old_content)
Example #2
0
  def test_putfile(self):
    tmpoutdir = None
    tmpindir = None

    try:
      tmpindir = tempfile.mkdtemp(prefix='isolateserver_test')
      infile = os.path.join(tmpindir, u'in')
      with fs.open(infile, 'wb') as f:
        f.write('data')

      tmpoutdir = tempfile.mkdtemp(prefix='isolateserver_test')

      # Copy as fileobj
      fo = os.path.join(tmpoutdir, u'fo')
      isolateserver.putfile(io.BytesIO('data'), fo)
      self.assertEqual(True, fs.exists(fo))
      self.assertEqual(False, fs.islink(fo))
      self.assertFile(fo, 'data')

      # Copy with partial fileobj
      pfo = os.path.join(tmpoutdir, u'pfo')
      fobj = io.BytesIO('adatab')
      fobj.read(1)  # Read the 'a'
      isolateserver.putfile(fobj, pfo, size=4)
      self.assertEqual(True, fs.exists(pfo))
      self.assertEqual(False, fs.islink(pfo))
      self.assertEqual('b', fobj.read())
      self.assertFile(pfo, 'data')

      # Copy as not readonly
      cp = os.path.join(tmpoutdir, u'cp')
      with fs.open(infile, 'rb') as f:
        isolateserver.putfile(f, cp, file_mode=0o755)
      self.assertEqual(True, fs.exists(cp))
      self.assertEqual(False, fs.islink(cp))
      self.assertFile(cp, 'data')

      # Use hardlink
      hl = os.path.join(tmpoutdir, u'hl')
      with fs.open(infile, 'rb') as f:
        isolateserver.putfile(f, hl, use_symlink=False)
      self.assertEqual(True, fs.exists(hl))
      self.assertEqual(False, fs.islink(hl))
      self.assertFile(hl, 'data')

      # Use symlink
      sl = os.path.join(tmpoutdir, u'sl')
      with fs.open(infile, 'rb') as f:
        isolateserver.putfile(f, sl, use_symlink=True)
      self.assertEqual(True, fs.exists(sl))
      self.assertEqual(True, fs.islink(sl))
      self.assertEqual('data', fs.open(sl, 'rb').read())
      self.assertFile(sl, 'data')

    finally:
      if tmpindir:
        file_path.rmtree(tmpindir)
      if tmpoutdir:
        file_path.rmtree(tmpoutdir)
Example #3
0
 def put_to_named_cache(self, manager, cache_name, file_name, contents):
     """Puts files into named cache."""
     cache_dir = os.path.join(self.tempdir, 'put_to_named_cache')
     manager.install(cache_dir, cache_name)
     with fs.open(os.path.join(cache_dir, file_name), 'wb') as f:
         f.write(contents)
     manager.uninstall(cache_dir, cache_name)
Example #4
0
def prepare_for_archival(options, cwd):
  """Loads the isolated file and create 'infiles' for archival."""
  complete_state = load_complete_state(
      options, cwd, options.subdir, False)
  # Make sure that complete_state isn't modified until save_files() is
  # called, because any changes made to it here will propagate to the files
  # created (which is probably not intended).
  complete_state.save_files()

  infiles = complete_state.saved_state.files
  # Add all the .isolated files.
  isolated_hash = []
  isolated_files = [
    options.isolated,
  ] + complete_state.saved_state.child_isolated_files
  for item in isolated_files:
    item_path = os.path.join(
        os.path.dirname(complete_state.isolated_filepath), item)
    # Do not use isolated_format.hash_file() here because the file is
    # likely smallish (under 500kb) and its file size is needed.
    with fs.open(item_path, 'rb') as f:
      content = f.read()
    isolated_hash.append(
        complete_state.saved_state.algo(content).hexdigest())
    isolated_metadata = {
      'h': isolated_hash[-1],
      's': len(content),
      'priority': '0'
    }
    infiles[item_path] = isolated_metadata
  return complete_state, infiles, isolated_hash
Example #5
0
def prepare_for_archival(options, cwd):
    """Loads the isolated file and create 'infiles' for archival."""
    complete_state = load_complete_state(options, cwd, options.subdir, False)
    # Make sure that complete_state isn't modified until save_files() is
    # called, because any changes made to it here will propagate to the files
    # created (which is probably not intended).
    complete_state.save_files()

    infiles = complete_state.saved_state.files
    # Add all the .isolated files.
    isolated_hash = []
    isolated_files = [
        options.isolated,
    ] + complete_state.saved_state.child_isolated_files
    for item in isolated_files:
        item_path = os.path.join(
            os.path.dirname(complete_state.isolated_filepath), item)
        # Do not use isolated_format.hash_file() here because the file is
        # likely smallish (under 500kb) and its file size is needed.
        with fs.open(item_path, 'rb') as f:
            content = f.read()
        isolated_hash.append(
            complete_state.saved_state.algo(content).hexdigest())
        isolated_metadata = {
            'h': isolated_hash[-1],
            's': len(content),
            'priority': '0'
        }
        infiles[item_path] = isolated_metadata
    return complete_state, infiles, isolated_hash
Example #6
0
 def getfileobj(self, digest):
     try:
         f = fs.open(self._path(digest), 'rb')
         with self._lock:
             self._used.append(self._lru[digest])
         return f
     except IOError:
         raise CacheMiss(digest)
Example #7
0
 def test_rmtree_unicode(self):
     subdir = os.path.join(self.tempdir, "hi")
     fs.mkdir(subdir)
     filepath = os.path.join(subdir, u"\u0627\u0644\u0635\u064A\u0646\u064A\u0629")
     with fs.open(filepath, "wb") as f:
         f.write("hi")
     # In particular, it fails when the input argument is a str.
     file_path.rmtree(str(subdir))
Example #8
0
 def test_cleanup_unexpected(self):
     # cleanup() delete unexpected file in the cache directory.
     fs.mkdir(self.cache_dir)
     with fs.open(os.path.join(self.cache_dir, u'junk'), 'w') as f:
         f.write('random')
     cache = self.get_cache(_get_policies())
     self.assertEqual(['junk'], fs.listdir(cache.cache_dir))
     self.assertEqual(True, cache.cleanup())
     self.assertEqual([cache.STATE_FILE], fs.listdir(cache.cache_dir))
Example #9
0
 def test_rmtree_unicode(self):
   subdir = os.path.join(self.tempdir, 'hi')
   fs.mkdir(subdir)
   filepath = os.path.join(
       subdir, u'\u0627\u0644\u0635\u064A\u0646\u064A\u0629')
   with fs.open(filepath, 'wb') as f:
     f.write('hi')
   # In particular, it fails when the input argument is a str.
   file_path.rmtree(str(subdir))
Example #10
0
        def test_expand_symlinks_path_case(self):
            # Ensures that the resulting path case is fixed on case insensitive file
            # system.
            fs.symlink('dest', os.path.join(self.cwd, u'link'))
            fs.mkdir(os.path.join(self.cwd, u'Dest'))
            fs.open(os.path.join(self.cwd, u'Dest', u'file.txt'), 'w').close()

            relfile, symlinks = isolated_format._expand_symlinks(
                self.cwd, u'.')
            self.assertEqual((u'.', []), (relfile, symlinks))

            relfile, symlinks = isolated_format._expand_symlinks(
                self.cwd, u'link')
            self.assertEqual((u'Dest', [u'link']), (relfile, symlinks))

            relfile, symlinks = isolated_format._expand_symlinks(
                self.cwd, u'link/File.txt')
            self.assertEqual((u'Dest/file.txt', [u'link']),
                             (relfile, symlinks))
Example #11
0
def CMDput_bot_config(parser, args):
  """Uploads a new version of bot_config.py."""
  options, args = parser.parse_args(args)
  if len(args) != 1:
    parser.error('Must specify file to upload')
  url = options.swarming + '/_ah/api/swarming/v1/server/put_bot_config'
  path = unicode(os.path.abspath(args[0]))
  with fs.open(path, 'rb') as f:
    content = f.read().decode('utf-8')
  data = net.url_read_json(url, data={'content': content})
  print data
  return 0
def load_included_isolate(isolate_dir, isolate_path):
  if os.path.isabs(isolate_path):
    raise IsolateError(
        'Failed to load configuration; absolute include path \'%s\'' %
        isolate_path)
  included_isolate = os.path.normpath(os.path.join(isolate_dir, isolate_path))
  if sys.platform == 'win32':
    if included_isolate[0].lower() != isolate_dir[0].lower():
      raise IsolateError('Can\'t reference a .isolate file from another drive')
  with fs.open(included_isolate, 'r') as f:
    return load_isolate_as_config(
        os.path.dirname(included_isolate), eval_content(f.read()), None)
Example #13
0
 def test_symlink_input_absolute_path(self):
     # A symlink is outside of the checkout, it should be treated as a normal
     # directory.
     # .../src
     # .../src/out -> .../tmp/foo
     # .../tmp
     # .../tmp/foo
     src = os.path.join(self.cwd, u'src')
     src_out = os.path.join(src, u'out')
     tmp = os.path.join(self.cwd, u'tmp')
     tmp_foo = os.path.join(tmp, u'foo')
     fs.mkdir(src)
     fs.mkdir(tmp)
     fs.mkdir(tmp_foo)
     # The problem was that it's an absolute path, so it must be considered a
     # normal directory.
     fs.symlink(tmp, src_out)
     fs.open(os.path.join(tmp_foo, u'bar.txt'), 'w').close()
     relfile, symlinks = isolated_format._expand_symlinks(
         src, u'out/foo/bar.txt')
     self.assertEqual((u'out/foo/bar.txt', []), (relfile, symlinks))
Example #14
0
def hash_file(filepath, algo):
  """Calculates the hash of a file without reading it all in memory at once.

  |algo| should be one of hashlib hashing algorithm.
  """
  digest = algo()
  with fs.open(filepath, 'rb') as f:
    while True:
      chunk = f.read(DISK_FILE_CHUNK)
      if not chunk:
        break
      digest.update(chunk)
  return digest.hexdigest()
Example #15
0
def hash_file(filepath, algo):
  """Calculates the hash of a file without reading it all in memory at once.

  |algo| should be one of hashlib hashing algorithm.
  """
  digest = algo()
  with fs.open(filepath, 'rb') as f:
    while True:
      chunk = f.read(DISK_FILE_CHUNK)
      if not chunk:
        break
      digest.update(chunk)
  return digest.hexdigest()
Example #16
0
def CMDcollect(parser, args):
  """Retrieves results of one or multiple Swarming task by its ID.

  The result can be in multiple part if the execution was sharded. It can
  potentially have retries.
  """
  add_collect_options(parser)
  parser.add_option(
      '-j', '--json',
      help='Load the task ids from .json as saved by trigger --dump-json')
  options, args = parser.parse_args(args)
  if not args and not options.json:
    parser.error('Must specify at least one task id or --json.')
  if args and options.json:
    parser.error('Only use one of task id or --json.')

  if options.json:
    options.json = unicode(os.path.abspath(options.json))
    try:
      with fs.open(options.json, 'rb') as f:
        data = json.load(f)
    except (IOError, ValueError):
      parser.error('Failed to open %s' % options.json)
    try:
      tasks = sorted(
          data['tasks'].itervalues(), key=lambda x: x['shard_index'])
      args = [t['task_id'] for t in tasks]
    except (KeyError, TypeError):
      parser.error('Failed to process %s' % options.json)
    if options.timeout is None:
      options.timeout = (
          data['request']['properties']['execution_timeout_secs'] +
          data['request']['expiration_secs'] + 10.)
  else:
    valid = frozenset('0123456789abcdef')
    if any(not valid.issuperset(task_id) for task_id in args):
      parser.error('Task ids are 0-9a-f.')

  try:
    return collect(
        options.swarming,
        args,
        options.timeout,
        options.decorate,
        options.print_status_updates,
        options.task_summary_json,
        options.task_output_dir,
        options.perf)
  except Failure:
    on_error.report(None)
    return 1
Example #17
0
        def test_file_to_metadata_path_case_collapse(self):
            # Ensure setting the collapse_symlink option doesn't include the symlinks
            basedir = os.path.join(self.cwd, u'basedir')
            fs.mkdir(basedir)
            subdir = os.path.join(basedir, u'subdir')
            fs.mkdir(subdir)
            linkdir = os.path.join(basedir, u'linkdir')
            fs.mkdir(linkdir)

            foo_file = os.path.join(subdir, u'Foo.txt')
            fs.open(foo_file, 'w').close()
            sym_file = os.path.join(basedir, u'linkdir', u'Sym.txt')
            fs.symlink('../subdir/Foo.txt', sym_file)

            actual = isolated_format.file_to_metadata(sym_file, True, True)
            actual['h'] = isolated_format.hash_file(sym_file, ALGO)
            expected = {
                # SHA-1 of empty string
                'h': 'da39a3ee5e6b4b0d3255bfef95601890afd80709',
                'm': 256,
                's': 0,
            }
            self.assertEqual(expected, actual)
 def getfileobj(self, digest):
     try:
         f = fs.open(self._path(digest), 'rb')
     except IOError:
         raise CacheMiss(digest)
     with self._lock:
         try:
             self._used.append(self._lru[digest])
         except KeyError:
             # If the digest is not actually in _lru, assume it is a cache miss.
             # Existing file will be overwritten by whoever uses the cache and added
             # to _lru.
             f.close()
             raise CacheMiss(digest)
     return f
def load_included_isolate(isolate_dir, isolate_path):
  if os.path.isabs(isolate_path):
    raise IsolateError(
        'Failed to load configuration; absolute include path \'%s\'' %
        isolate_path)
  included_isolate = os.path.normpath(os.path.join(isolate_dir, isolate_path))
  if sys.platform == 'win32':
    if included_isolate[0].lower() != isolate_dir[0].lower():
      raise IsolateError(
          'Can\'t reference a .isolate file from another drive')
  with fs.open(included_isolate, 'r') as f:
    return load_isolate_as_config(
        os.path.dirname(included_isolate),
        eval_content(f.read()),
        None)
Example #20
0
 def test_upgrade(self):
     # Make sure upgrading works. This is temporary as eventually all bots will
     # be updated.
     now = time.time()
     fs.mkdir(self.cache_dir)
     fs.mkdir(os.path.join(self.cache_dir, 'f1'))
     with fs.open(os.path.join(self.cache_dir, 'f1', 'hello'), 'wb') as f:
         f.write('world')
     # v1
     old = {
         'version': 2,
         'items': [
             ['cache1', ['f1', now]],
         ],
     }
     c = local_caching.NamedCache
     with fs.open(os.path.join(self.cache_dir, c.STATE_FILE), 'w') as f:
         json.dump(old, f)
     # It automatically upgrades to v2.
     cache = self.get_cache(_get_policies())
     expected = {u'cache1': ((u'f1', len('world')), now)}
     self.assertEqual(expected, dict(cache._lru._items.iteritems()))
     self.assertEqual([u'f1', cache.STATE_FILE],
                      sorted(fs.listdir(cache.cache_dir)))
Example #21
0
def file_write(path, content_generator):
    """Writes file content as generated by content_generator.

  Creates the intermediary directory as needed.

  Returns the number of bytes written.

  Meant to be mocked out in unit tests.
  """
    file_path.ensure_tree(os.path.dirname(path))
    total = 0
    with fs.open(path, 'wb') as f:
        for d in content_generator:
            total += len(d)
            f.write(d)
    return total
    def test_symlink_absolute(self):
        # A symlink to an absolute path is valid.
        # /dir
        # /dir/file
        # /ld -> /dir
        # /lf -> /ld/file
        dirpath = os.path.join(self.tempdir, 'dir')
        filepath = os.path.join(dirpath, 'file')
        fs.mkdir(dirpath)
        write_content(filepath, b'hello')

        linkfile = os.path.join(self.tempdir, 'lf')
        linkdir = os.path.join(self.tempdir, 'ld')
        dstfile = os.path.join(linkdir, 'file')
        fs.symlink(dstfile, linkfile)
        fs.symlink(dirpath, linkdir)

        self.assertEqual(True, fs.islink(linkfile))
        self.assertEqual(True, fs.islink(linkdir))
        self.assertEqual(dstfile, fs.readlink(linkfile))
        self.assertEqual(dirpath, fs.readlink(linkdir))
        self.assertEqual(['file'], fs.listdir(linkdir))
        # /lf resolves to /dir/file.
        with fs.open(linkfile) as f:
            self.assertEqual('hello', f.read())

        # Ensures that followlinks is respected in walk().
        expected = [
            (self.tempdir, ['dir', 'ld'], ['lf']),
            (dirpath, [], ['file']),
        ]
        actual = [
            (r, sorted(d), sorted(f))
            for r, d, f in sorted(fs.walk(self.tempdir, followlinks=False))
        ]
        self.assertEqual(expected, actual)
        expected = [
            (self.tempdir, ['dir', 'ld'], ['lf']),
            (dirpath, [], ['file']),
            (linkdir, [], ['file']),
        ]
        actual = [
            (r, sorted(d), sorted(f))
            for r, d, f in sorted(fs.walk(self.tempdir, followlinks=True))
        ]
        self.assertEqual(expected, actual)
Example #23
0
 def test_cycle_twice(self):
     # Ensure that named symlink works.
     cache = self.get_cache(_get_policies())
     dest_dir = os.path.join(self.tempdir, 'dest')
     self.assertEqual(0, cache.install(dest_dir, u'1'))
     with fs.open(os.path.join(dest_dir, u'hi'), 'wb') as f:
         f.write('hello')
     self.assertEqual(5, cache.uninstall(dest_dir, u'1'))
     self.assertEqual([u'1'],
                      fs.listdir(
                          os.path.join(cache.cache_dir, cache.NAMED_DIR)))
     self.assertEqual(True, cache.cleanup())
     self.assertEqual(5, cache.install(dest_dir, u'1'))
     self.assertEqual(5, cache.uninstall(dest_dir, u'1'))
     self.assertEqual([u'1'],
                      fs.listdir(
                          os.path.join(cache.cache_dir, cache.NAMED_DIR)))
     self.assertEqual([u'hi'],
                      fs.listdir(
                          os.path.join(cache.cache_dir, cache.NAMED_DIR,
                                       u'1')))
Example #24
0
  def test_fileobj_path(self):
    # No path on in-memory objects
    self.assertIs(None, isolateserver.fileobj_path(io.BytesIO('hello')))

    # Path on opened files
    thisfile = os.path.join(test_env.TESTS_DIR, 'isolateserver_test.py')
    f = fs.open(thisfile)
    result = isolateserver.fileobj_path(f)
    self.assertIsInstance(result, six.text_type)
    self.assertSequenceEqual(result, thisfile)

    # Path on temporary files
    tf = tempfile.NamedTemporaryFile()
    result = isolateserver.fileobj_path(tf)
    self.assertIsInstance(result, six.text_type)
    self.assertSequenceEqual(result, tf.name)

    # No path on files which are no longer on the file system
    tf = tempfile.NamedTemporaryFile(delete=False)
    fs.unlink(tf.name.decode(sys.getfilesystemencoding()))
    self.assertIs(None, isolateserver.fileobj_path(tf))
Example #25
0
    def test_cleanup_unexpected_named(self):
        # cleanup() deletes unexpected symlink and directory in named/.
        fs.mkdir(self.cache_dir)
        c = local_caching.NamedCache
        fs.mkdir(os.path.join(self.cache_dir, c.NAMED_DIR))
        p = os.path.join(self.cache_dir, c.NAMED_DIR, u'junk_file')
        with fs.open(p, 'w') as f:
            f.write('random')
        fs.mkdir(os.path.join(self.cache_dir, c.NAMED_DIR, u'junk_dir'))
        fs.symlink('invalid_dest',
                   os.path.join(self.cache_dir, c.NAMED_DIR, u'junk_link'))

        cache = self.get_cache(_get_policies())
        self.assertEqual([cache.NAMED_DIR], fs.listdir(cache.cache_dir))
        self.assertEqual(
            ['junk_dir', 'junk_file', 'junk_link'],
            sorted(fs.listdir(os.path.join(cache.cache_dir, cache.NAMED_DIR))))
        self.assertEqual(True, cache.cleanup())
        self.assertEqual([cache.NAMED_DIR, cache.STATE_FILE],
                         sorted(fs.listdir(cache.cache_dir)))
        self.assertEqual([],
                         fs.listdir(
                             os.path.join(cache.cache_dir, cache.NAMED_DIR)))
Example #26
0
    def test_load_corrupted_state(self):
        # cleanup() handles a broken state file.
        fs.mkdir(self.cache_dir)
        c = local_caching.NamedCache
        with fs.open(os.path.join(self.cache_dir, c.STATE_FILE), 'w') as f:
            f.write('}}}}')
        fs.makedirs(os.path.join(self.cache_dir, '1'), 0777)

        cache = self.get_cache(_get_policies())
        self._add_one_item(cache, 1)
        self.assertTrue(
            fs.exists(os.path.join(cache.cache_dir, cache.NAMED_DIR, '1')))
        self.assertTrue(
            fs.islink(os.path.join(cache.cache_dir, cache.NAMED_DIR, '1')))
        self.assertEqual([], cache.trim())
        self.assertTrue(
            fs.exists(os.path.join(cache.cache_dir, cache.NAMED_DIR, '1')))
        self.assertTrue(
            fs.islink(os.path.join(cache.cache_dir, cache.NAMED_DIR, '1')))
        self.assertEqual(True, cache.cleanup())
        self.assertEqual(
            sorted([cache.NAMED_DIR, cache.STATE_FILE, cache._lru[u'1'][0]]),
            sorted(fs.listdir(cache.cache_dir)))
Example #27
0
 def _add_one_item(self, cache, size):
     """Adds one item of |size| bytes in the cache and returns the created name.
 """
     # Don't allow 0 byte items here. This doesn't work for named cache.
     self.assertTrue(size)
     data = _gen_data(size)
     if isinstance(cache, local_caching.ContentAddressedCache):
         # This covers both MemoryContentAddressedCache and
         # DiskContentAddressedCache.
         return cache.write(self._algo(data).hexdigest(), [data])
     elif isinstance(cache, local_caching.NamedCache):
         # In this case, map a named cache, add a file, unmap it.
         dest_dir = os.path.join(self.tempdir, 'dest')
         self.assertFalse(fs.exists(dest_dir))
         name = unicode(size)
         cache.install(dest_dir, name)
         # Put a file in there named 'hello', otherwise it'll stay empty.
         with fs.open(os.path.join(dest_dir, 'hello'), 'wb') as f:
             f.write(data)
         cache.uninstall(dest_dir, name)
         self.assertFalse(fs.exists(dest_dir))
         return name
     self.fail('Unexpected cache type %r' % cache)
     return None
Example #28
0
def CMDquery_list(parser, args):
  """Returns list of all the Swarming APIs that can be used with command
  'query'.
  """
  parser.add_option(
      '--json', help='Path to JSON output file (otherwise prints to stdout)')
  options, args = parser.parse_args(args)
  if args:
    parser.error('No argument allowed.')

  try:
    apis = endpoints_api_discovery_apis(options.swarming)
  except APIError as e:
    parser.error(str(e))
  if options.json:
    options.json = unicode(os.path.abspath(options.json))
    with fs.open(options.json, 'wb') as f:
      json.dump(apis, f)
  else:
    help_url = (
      'https://apis-explorer.appspot.com/apis-explorer/?base=%s/_ah/api#p/' %
      options.swarming)
    for api_id, api in sorted(apis.iteritems()):
      print api_id
      print '  ' + api['description']
      for resource_name, resource in sorted(api['resources'].iteritems()):
        print ''
        for method_name, method in sorted(resource['methods'].iteritems()):
          # Only list the GET ones.
          if method['httpMethod'] != 'GET':
            continue
          print '- %s.%s: %s' % (
              resource_name, method_name, method['path'])
          print '  ' + method['description']
          print '  %s%s%s' % (help_url, api['servicePath'], method['id'])
  return 0
Example #29
0
def write_content(filepath, content):
  with fs.open(filepath, 'wb') as f:
    f.write(content)
Example #30
0
  def load_isolate(
      self, cwd, isolate_file, path_variables, config_variables,
      extra_variables, blacklist, ignore_broken_items):
    """Updates self.isolated and self.saved_state with information loaded from a
    .isolate file.

    Processes the loaded data, deduce root_dir, relative_cwd.
    """
    # Make sure to not depend on os.getcwd().
    assert os.path.isabs(isolate_file), isolate_file
    isolate_file = file_path.get_native_path_case(isolate_file)
    logging.info(
        'CompleteState.load_isolate(%s, %s, %s, %s, %s, %s)',
        cwd, isolate_file, path_variables, config_variables, extra_variables,
        ignore_broken_items)

    # Config variables are not affected by the paths and must be used to
    # retrieve the paths, so update them first.
    self.saved_state.update_config(config_variables)

    with fs.open(isolate_file, 'r') as f:
      # At that point, variables are not replaced yet in command and infiles.
      # infiles may contain directory entries and is in posix style.
      command, infiles, read_only, isolate_cmd_dir = (
          isolate_format.load_isolate_for_config(
              os.path.dirname(isolate_file), f.read(),
              self.saved_state.config_variables))

    # Processes the variables with the new found relative root. Note that 'cwd'
    # is used when path variables are used.
    path_variables = normalize_path_variables(
        cwd, path_variables, isolate_cmd_dir)
    # Update the rest of the saved state.
    self.saved_state.update(isolate_file, path_variables, extra_variables)

    total_variables = self.saved_state.path_variables.copy()
    total_variables.update(self.saved_state.config_variables)
    total_variables.update(self.saved_state.extra_variables)
    command = [
        isolate_format.eval_variables(i, total_variables) for i in command
    ]

    total_variables = self.saved_state.path_variables.copy()
    total_variables.update(self.saved_state.extra_variables)
    infiles = [
        isolate_format.eval_variables(f, total_variables) for f in infiles
    ]
    # root_dir is automatically determined by the deepest root accessed with the
    # form '../../foo/bar'. Note that path variables must be taken in account
    # too, add them as if they were input files.
    self.saved_state.root_dir = isolate_format.determine_root_dir(
        isolate_cmd_dir, infiles + self.saved_state.path_variables.values())
    # The relative directory is automatically determined by the relative path
    # between root_dir and the directory containing the .isolate file,
    # isolate_base_dir.
    relative_cwd = os.path.relpath(isolate_cmd_dir, self.saved_state.root_dir)
    # Now that we know where the root is, check that the path_variables point
    # inside it.
    for k, v in self.saved_state.path_variables.iteritems():
      dest = os.path.join(isolate_cmd_dir, relative_cwd, v)
      if not file_path.path_starts_with(self.saved_state.root_dir, dest):
        raise isolated_format.MappingError(
            'Path variable %s=%r points outside the inferred root directory '
            '%s; %s'
            % (k, v, self.saved_state.root_dir, dest))
    # Normalize the files based to self.saved_state.root_dir. It is important to
    # keep the trailing os.path.sep at that step.
    infiles = [
      file_path.relpath(
          file_path.normpath(os.path.join(isolate_cmd_dir, f)),
          self.saved_state.root_dir)
      for f in infiles
    ]
    follow_symlinks = sys.platform != 'win32'
    # Expand the directories by listing each file inside. Up to now, trailing
    # os.path.sep must be kept.
    infiles = isolated_format.expand_directories_and_symlinks(
        self.saved_state.root_dir,
        infiles,
        tools.gen_blacklist(blacklist),
        follow_symlinks,
        ignore_broken_items)

    # Finally, update the new data to be able to generate the foo.isolated file,
    # the file that is used by run_isolated.py.
    self.saved_state.update_isolated(command, infiles, read_only, relative_cwd)
    logging.debug(self)
Example #31
0
 def assertFile(self, path, contents):
   self.assertTrue(fs.exists(path), 'File %s doesn\'t exist!' % path)
   self.assertMultiLineEqual(contents, fs.open(path, 'rb').read())
Example #32
0
def write_content(filepath, content):
  with fs.open(filepath, 'wb') as f:
    f.write(content)
Example #33
0
def read_file(path):
    with fs.open(path, 'rb') as f:
        return f.read()
Example #34
0
def write_file(path, contents):
    with fs.open(path, 'wb') as f:
        f.write(contents)
Example #35
0
    def load_isolate(self, cwd, isolate_file, path_variables, config_variables,
                     extra_variables, blacklist, ignore_broken_items,
                     collapse_symlinks):
        """Updates self.isolated and self.saved_state with information loaded from a
    .isolate file.

    Processes the loaded data, deduce root_dir, relative_cwd.
    """
        # Make sure to not depend on os.getcwd().
        assert os.path.isabs(isolate_file), isolate_file
        isolate_file = file_path.get_native_path_case(isolate_file)
        logging.info('CompleteState.load_isolate(%s, %s, %s, %s, %s, %s, %s)',
                     cwd, isolate_file, path_variables, config_variables,
                     extra_variables, ignore_broken_items, collapse_symlinks)

        # Config variables are not affected by the paths and must be used to
        # retrieve the paths, so update them first.
        self.saved_state.update_config(config_variables)

        with fs.open(isolate_file, 'r') as f:
            # At that point, variables are not replaced yet in command and infiles.
            # infiles may contain directory entries and is in posix style.
            command, infiles, read_only, isolate_cmd_dir = (
                isolate_format.load_isolate_for_config(
                    os.path.dirname(isolate_file), f.read(),
                    self.saved_state.config_variables))

        # Processes the variables with the new found relative root. Note that 'cwd'
        # is used when path variables are used.
        path_variables = normalize_path_variables(cwd, path_variables,
                                                  isolate_cmd_dir)
        # Update the rest of the saved state.
        self.saved_state.update(isolate_file, path_variables, extra_variables)

        total_variables = self.saved_state.path_variables.copy()
        total_variables.update(self.saved_state.config_variables)
        total_variables.update(self.saved_state.extra_variables)
        command = [
            isolate_format.eval_variables(i, total_variables) for i in command
        ]

        total_variables = self.saved_state.path_variables.copy()
        total_variables.update(self.saved_state.extra_variables)
        infiles = [
            isolate_format.eval_variables(f, total_variables) for f in infiles
        ]
        # root_dir is automatically determined by the deepest root accessed with the
        # form '../../foo/bar'. Note that path variables must be taken in account
        # too, add them as if they were input files.
        self.saved_state.root_dir = isolate_format.determine_root_dir(
            isolate_cmd_dir,
            infiles + self.saved_state.path_variables.values())
        # The relative directory is automatically determined by the relative path
        # between root_dir and the directory containing the .isolate file,
        # isolate_base_dir.
        relative_cwd = os.path.relpath(isolate_cmd_dir,
                                       self.saved_state.root_dir)
        # Now that we know where the root is, check that the path_variables point
        # inside it.
        for k, v in self.saved_state.path_variables.iteritems():
            dest = os.path.join(isolate_cmd_dir, relative_cwd, v)
            if not file_path.path_starts_with(self.saved_state.root_dir, dest):
                raise isolated_format.MappingError(
                    'Path variable %s=%r points outside the inferred root directory '
                    '%s; %s' % (k, v, self.saved_state.root_dir, dest))
        # Normalize the files based to self.saved_state.root_dir. It is important to
        # keep the trailing os.path.sep at that step.
        infiles = [
            file_path.relpath(
                file_path.normpath(os.path.join(isolate_cmd_dir, f)),
                self.saved_state.root_dir) for f in infiles
        ]
        follow_symlinks = False
        if not collapse_symlinks:
            follow_symlinks = sys.platform != 'win32'
        # Expand the directories by listing each file inside. Up to now, trailing
        # os.path.sep must be kept.
        infiles = isolated_format.expand_directories_and_symlinks(
            self.saved_state.root_dir, infiles, tools.gen_blacklist(blacklist),
            follow_symlinks, ignore_broken_items)

        # Finally, update the new data to be able to generate the foo.isolated file,
        # the file that is used by run_isolated.py.
        self.saved_state.update_isolated(command, infiles, read_only,
                                         relative_cwd)
        logging.debug(self)