Exemple #1
0
def recreate_tree(outdir, indir, infiles, action):
  """Creates a new tree with only the input files in it.

  Arguments:
    outdir:    Output directory to create the files in.
    indir:     Root directory the infiles are based in.
    infiles:   List of files to map from |indir| to |outdir|.
    action:    See assert below.
  """
  logging.debug(
      'recreate_tree(%s, %s, %s, %s)' % (outdir, indir, infiles, action))
  logging.info('Mapping from %s to %s' % (indir, outdir))

  assert action in (
      run_test_from_archive.HARDLINK,
      run_test_from_archive.SYMLINK,
      run_test_from_archive.COPY)
  outdir = os.path.normpath(outdir)
  if not os.path.isdir(outdir):
    logging.info ('Creating %s' % outdir)
    os.makedirs(outdir)
  # Do not call abspath until the directory exists.
  outdir = os.path.abspath(outdir)

  for relfile in infiles:
    infile = os.path.join(indir, relfile)
    outfile = os.path.join(outdir, relfile)
    outsubdir = os.path.dirname(outfile)
    if not os.path.isdir(outsubdir):
      os.makedirs(outsubdir)
    run_test_from_archive.link_file(outfile, infile, action)
Exemple #2
0
def recreate_tree(outdir, indir, infiles, action):
    """Creates a new tree with only the input files in it.

  Arguments:
    outdir:    Output directory to create the files in.
    indir:     Root directory the infiles are based in.
    infiles:   List of files to map from |indir| to |outdir|.
    action:    See assert below.
  """
    logging.debug('recreate_tree(%s, %s, %s, %s)' %
                  (outdir, indir, infiles, action))
    logging.info('Mapping from %s to %s' % (indir, outdir))

    assert action in (run_test_from_archive.HARDLINK,
                      run_test_from_archive.SYMLINK,
                      run_test_from_archive.COPY)
    outdir = os.path.normpath(outdir)
    if not os.path.isdir(outdir):
        logging.info('Creating %s' % outdir)
        os.makedirs(outdir)
    # Do not call abspath until the directory exists.
    outdir = os.path.abspath(outdir)

    for relfile in infiles:
        infile = os.path.join(indir, relfile)
        outfile = os.path.join(outdir, relfile)
        outsubdir = os.path.dirname(outfile)
        if not os.path.isdir(outsubdir):
            os.makedirs(outsubdir)
        run_test_from_archive.link_file(outfile, infile, action)
Exemple #3
0
def recreate_tree(outdir, indir, infiles, action, as_sha1):
  """Creates a new tree with only the input files in it.

  Arguments:
    outdir:    Output directory to create the files in.
    indir:     Root directory the infiles are based in.
    infiles:   dict of files to map from |indir| to |outdir|.
    action:    See assert below.
    as_sha1:   Output filename is the sha1 instead of relfile.
  """
  logging.info(
      'recreate_tree(outdir=%s, indir=%s, files=%d, action=%s, as_sha1=%s)' %
      (outdir, indir, len(infiles), action, as_sha1))

  assert action in (
      run_test_from_archive.HARDLINK,
      run_test_from_archive.SYMLINK,
      run_test_from_archive.COPY)
  outdir = os.path.normpath(outdir)
  if not os.path.isdir(outdir):
    logging.info ('Creating %s' % outdir)
    os.makedirs(outdir)
  # Do not call abspath until the directory exists.
  outdir = os.path.abspath(outdir)

  for relfile, metadata in infiles.iteritems():
    infile = os.path.join(indir, relfile)
    if as_sha1:
      # Do the hashtable specific checks.
      if 'link' in metadata:
        # Skip links when storing a hashtable.
        continue
      outfile = os.path.join(outdir, metadata['sha-1'])
      if os.path.isfile(outfile):
        # Just do a quick check that the file size matches. No need to stat()
        # again the input file, grab the value from the dict.
        if metadata['size'] == os.stat(outfile).st_size:
          continue
        else:
          logging.warn('Overwritting %s' % metadata['sha-1'])
          os.remove(outfile)
    else:
      outfile = os.path.join(outdir, relfile)
      outsubdir = os.path.dirname(outfile)
      if not os.path.isdir(outsubdir):
        os.makedirs(outsubdir)

    if metadata.get('touched_only') == True:
      open(outfile, 'ab').close()
    elif 'link' in metadata:
      pointed = metadata['link']
      logging.debug('Symlink: %s -> %s' % (outfile, pointed))
      os.symlink(pointed, outfile)
    else:
      run_test_from_archive.link_file(outfile, infile, action)
Exemple #4
0
def CMDhashtable(args):
  """Creates a hash table content addressed object store.

  All the files listed in the .result file are put in the output directory with
  the file name being the sha-1 of the file's content.
  """
  parser = OptionParserIsolate(command='hashtable')
  options, _ = parser.parse_args(args)
  complete_state = load_complete_state(options, WITH_HASH)

  options.outdir = (
      options.outdir or os.path.join(complete_state.resultdir, 'hashtable'))
  if not os.path.isdir(options.outdir):
    os.makedirs(options.outdir)

  # Map each of the file.
  for relfile, properties in complete_state.result.files.iteritems():
    infile = os.path.join(complete_state.root_dir, relfile)
    if not 'sha-1' in properties:
      # It's a symlink.
      continue
    outfile = os.path.join(options.outdir, properties['sha-1'])
    if os.path.isfile(outfile):
      # Just do a quick check that the file size matches. No need to stat()
      # again the input file, grab the value from the dict.
      out_size = os.stat(outfile).st_size
      in_size = complete_state.result.files[relfile]['size']
      if in_size == out_size:
        continue
      # Otherwise, an exception will be raised.
    logging.info('%s -> %s' % (relfile, properties['sha-1']))
    run_test_from_archive.link_file(
        outfile, infile, run_test_from_archive.HARDLINK)

  complete_state.save_files()

  # Also archive the .result file.
  with open(complete_state.result_file, 'rb') as f:
    result_hash = hashlib.sha1(f.read()).hexdigest()
  logging.info(
      '%s -> %s' % (os.path.basename(complete_state.result_file), result_hash))
  outfile = os.path.join(options.outdir, result_hash)
  if os.path.isfile(outfile):
    # Just do a quick check that the file size matches. If they do, skip the
    # archive. This mean the build result didn't change at all.
    out_size = os.stat(outfile).st_size
    in_size = os.stat(complete_state.result_file).st_size
    if in_size == out_size:
      return 0

  run_test_from_archive.link_file(
      outfile, complete_state.result_file, run_test_from_archive.HARDLINK)
  return 0
Exemple #5
0
def CMDhashtable(args):
  """Creates a hash table content addressed object store.

  All the files listed in the .result file are put in the output directory with
  the file name being the sha-1 of the file's content.
  """
  parser = OptionParserIsolate(command='hashtable')
  options, _ = parser.parse_args(args)

  success = False
  try:
    complete_state = load_complete_state(options, WITH_HASH)
    options.outdir = (
        options.outdir or os.path.join(complete_state.resultdir, 'hashtable'))
    recreate_tree(
        outdir=options.outdir,
        indir=complete_state.root_dir,
        infiles=complete_state.result.files,
        action=run_test_from_archive.HARDLINK,
        as_sha1=True)

    complete_state.save_files()

    # Also archive the .result file.
    with open(complete_state.result_file, 'rb') as f:
      result_hash = hashlib.sha1(f.read()).hexdigest()
    logging.info(
        '%s -> %s' %
        (os.path.basename(complete_state.result_file), result_hash))
    outfile = os.path.join(options.outdir, result_hash)
    if os.path.isfile(outfile):
      # Just do a quick check that the file size matches. If they do, skip the
      # archive. This mean the build result didn't change at all.
      out_size = os.stat(outfile).st_size
      in_size = os.stat(complete_state.result_file).st_size
      if in_size == out_size:
        success = True
        return 0

    run_test_from_archive.link_file(
        outfile, complete_state.result_file, run_test_from_archive.HARDLINK)
    success = True
    return 0
  finally:
    # If the command failed, delete the .results file if it exists. This is
    # important so no stale swarm job is executed.
    if not success and os.path.isfile(options.result):
      os.remove(options.result)
Exemple #6
0
def MODEhashtable(outdir, state):
    outdir = outdir or os.path.join(state.resultdir, "hashtable")
    if not os.path.isdir(outdir):
        os.makedirs(outdir)
    for relfile, properties in state.result.files.iteritems():
        infile = os.path.join(state.root_dir, relfile)
        outfile = os.path.join(outdir, properties["sha-1"])
        if os.path.isfile(outfile):
            # Just do a quick check that the file size matches. No need to stat()
            # again the input file, grab the value from the dict.
            out_size = os.stat(outfile).st_size
            in_size = state.result.files[relfile].get("size") or os.stat(infile).st_size
            if in_size == out_size:
                continue
            # Otherwise, an exception will be raised.
        print "Mapping %s -> %s" % (outfile, infile)
        run_test_from_archive.link_file(outfile, infile, run_test_from_archive.HARDLINK)
    return 0
Exemple #7
0
def MODEhashtable(outdir, state):
    outdir = (outdir or os.path.join(state.resultdir, 'hashtable'))
    if not os.path.isdir(outdir):
        os.makedirs(outdir)
    for relfile, properties in state.result.files.iteritems():
        infile = os.path.join(state.root_dir, relfile)
        outfile = os.path.join(outdir, properties['sha-1'])
        if os.path.isfile(outfile):
            # Just do a quick check that the file size matches. No need to stat()
            # again the input file, grab the value from the dict.
            out_size = os.stat(outfile).st_size
            in_size = (state.result.files[relfile].get('size')
                       or os.stat(infile).st_size)
            if in_size == out_size:
                continue
            # Otherwise, an exception will be raised.
        print '%s -> %s' % (relfile, properties['sha-1'])
        run_test_from_archive.link_file(outfile, infile,
                                        run_test_from_archive.HARDLINK)
    return 0
Exemple #8
0
def MODEhashtable(outdir, indir, dictfiles, _read_only, _cmd, _relative_cwd,
                  resultfile):
    outdir = outdir or os.path.join(os.path.dirname(resultfile), 'hashtable')
    if not os.path.isdir(outdir):
        os.makedirs(outdir)
    for relfile, properties in dictfiles.iteritems():
        infile = os.path.join(indir, relfile)
        outfile = os.path.join(outdir, properties['sha-1'])
        if os.path.isfile(outfile):
            # Just do a quick check that the file size matches. No need to stat()
            # again the input file, grab the value from the dict.
            out_size = os.stat(outfile).st_size
            in_size = dictfiles.get(infile,
                                    {}).get('size') or os.stat(infile).st_size
            if in_size == out_size:
                continue
            # Otherwise, an exception will be raised.
        run_test_from_archive.link_file(outfile, infile,
                                        run_test_from_archive.HARDLINK)
    return 0
Exemple #9
0
def MODEhashtable(outdir, indir, data):
  outdir = (
      outdir or os.path.join(os.path.dirname(data['resultdir']), 'hashtable'))
  if not os.path.isdir(outdir):
    os.makedirs(outdir)
  for relfile, properties in data['files'].iteritems():
    infile = os.path.join(indir, relfile)
    outfile = os.path.join(outdir, properties['sha-1'])
    if os.path.isfile(outfile):
      # Just do a quick check that the file size matches. No need to stat()
      # again the input file, grab the value from the dict.
      out_size = os.stat(outfile).st_size
      in_size = (
          data.get('files', {}).get(infile, {}).get('size') or
          os.stat(infile).st_size)
      if in_size == out_size:
        continue
      # Otherwise, an exception will be raised.
    run_test_from_archive.link_file(
        outfile, infile, run_test_from_archive.HARDLINK)
  return 0
Exemple #10
0
def recreate_tree(outdir, indir, infiles, action):
  """Creates a new tree with only the input files in it.

  Arguments:
    outdir:    Output directory to create the files in.
    indir:     Root directory the infiles are based in.
    infiles:   dict of files to map from |indir| to |outdir|.
    action:    See assert below.
  """
  logging.debug(
      'recreate_tree(%s, %s, %s, %s)' % (outdir, indir, len(infiles), action))
  logging.info('Mapping from %s to %s' % (indir, outdir))

  assert action in (
      run_test_from_archive.HARDLINK,
      run_test_from_archive.SYMLINK,
      run_test_from_archive.COPY)
  outdir = os.path.normpath(outdir)
  if not os.path.isdir(outdir):
    logging.info ('Creating %s' % outdir)
    os.makedirs(outdir)
  # Do not call abspath until the directory exists.
  outdir = os.path.abspath(outdir)

  for relfile, metadata in infiles.iteritems():
    infile = os.path.join(indir, relfile)
    outfile = os.path.join(outdir, relfile)
    outsubdir = os.path.dirname(outfile)
    if not os.path.isdir(outsubdir):
      os.makedirs(outsubdir)
    if metadata.get('touched_only') == True:
      open(outfile, 'ab').close()
    elif 'link' in metadata:
      pointed = metadata['link']
      logging.debug('Symlink: %s -> %s' % (outfile, pointed))
      os.symlink(pointed, outfile)
    else:
      run_test_from_archive.link_file(outfile, infile, action)