def recreate_tree(outdir, indir, infiles, action):
"""Creates a new tree with only the input files in it.
Arguments:
outdir: Output directory to create the files in.
indir: Root directory the infiles are based in.
infiles: List of files to map from |indir| to |outdir|.
action: See assert below.
"""
logging.debug(
'recreate_tree(%s, %s, %s, %s)' % (outdir, indir, infiles, action))
logging.info('Mapping from %s to %s' % (indir, outdir))
assert action in (
run_test_from_archive.HARDLINK,
run_test_from_archive.SYMLINK,
run_test_from_archive.COPY)
outdir = os.path.normpath(outdir)
if not os.path.isdir(outdir):
logging.info ('Creating %s' % outdir)
os.makedirs(outdir)
# Do not call abspath until the directory exists.
outdir = os.path.abspath(outdir)
for relfile in infiles:
infile = os.path.join(indir, relfile)
outfile = os.path.join(outdir, relfile)
outsubdir = os.path.dirname(outfile)
if not os.path.isdir(outsubdir):
os.makedirs(outsubdir)
run_test_from_archive.link_file(outfile, infile, action)
def recreate_tree(outdir, indir, infiles, action):
"""Creates a new tree with only the input files in it.
Arguments:
outdir: Output directory to create the files in.
indir: Root directory the infiles are based in.
infiles: List of files to map from |indir| to |outdir|.
action: See assert below.
"""
logging.debug('recreate_tree(%s, %s, %s, %s)' %
(outdir, indir, infiles, action))
logging.info('Mapping from %s to %s' % (indir, outdir))
assert action in (run_test_from_archive.HARDLINK,
run_test_from_archive.SYMLINK,
run_test_from_archive.COPY)
outdir = os.path.normpath(outdir)
if not os.path.isdir(outdir):
logging.info('Creating %s' % outdir)
os.makedirs(outdir)
# Do not call abspath until the directory exists.
outdir = os.path.abspath(outdir)
for relfile in infiles:
infile = os.path.join(indir, relfile)
outfile = os.path.join(outdir, relfile)
outsubdir = os.path.dirname(outfile)
if not os.path.isdir(outsubdir):
os.makedirs(outsubdir)
run_test_from_archive.link_file(outfile, infile, action)
def recreate_tree(outdir, indir, infiles, action, as_sha1):
"""Creates a new tree with only the input files in it.
Arguments:
outdir: Output directory to create the files in.
indir: Root directory the infiles are based in.
infiles: dict of files to map from |indir| to |outdir|.
action: See assert below.
as_sha1: Output filename is the sha1 instead of relfile.
"""
logging.info(
'recreate_tree(outdir=%s, indir=%s, files=%d, action=%s, as_sha1=%s)' %
(outdir, indir, len(infiles), action, as_sha1))
assert action in (
run_test_from_archive.HARDLINK,
run_test_from_archive.SYMLINK,
run_test_from_archive.COPY)
outdir = os.path.normpath(outdir)
if not os.path.isdir(outdir):
logging.info ('Creating %s' % outdir)
os.makedirs(outdir)
# Do not call abspath until the directory exists.
outdir = os.path.abspath(outdir)
for relfile, metadata in infiles.iteritems():
infile = os.path.join(indir, relfile)
if as_sha1:
# Do the hashtable specific checks.
if 'link' in metadata:
# Skip links when storing a hashtable.
continue
outfile = os.path.join(outdir, metadata['sha-1'])
if os.path.isfile(outfile):
# Just do a quick check that the file size matches. No need to stat()
# again the input file, grab the value from the dict.
if metadata['size'] == os.stat(outfile).st_size:
continue
else:
logging.warn('Overwritting %s' % metadata['sha-1'])
os.remove(outfile)
else:
outfile = os.path.join(outdir, relfile)
outsubdir = os.path.dirname(outfile)
if not os.path.isdir(outsubdir):
os.makedirs(outsubdir)
if metadata.get('touched_only') == True:
open(outfile, 'ab').close()
elif 'link' in metadata:
pointed = metadata['link']
logging.debug('Symlink: %s -> %s' % (outfile, pointed))
os.symlink(pointed, outfile)
else:
run_test_from_archive.link_file(outfile, infile, action)
def CMDhashtable(args):
"""Creates a hash table content addressed object store.
All the files listed in the .result file are put in the output directory with
the file name being the sha-1 of the file's content.
"""
parser = OptionParserIsolate(command='hashtable')
options, _ = parser.parse_args(args)
complete_state = load_complete_state(options, WITH_HASH)
options.outdir = (
options.outdir or os.path.join(complete_state.resultdir, 'hashtable'))
if not os.path.isdir(options.outdir):
os.makedirs(options.outdir)
# Map each of the file.
for relfile, properties in complete_state.result.files.iteritems():
infile = os.path.join(complete_state.root_dir, relfile)
if not 'sha-1' in properties:
# It's a symlink.
continue
outfile = os.path.join(options.outdir, properties['sha-1'])
if os.path.isfile(outfile):
# Just do a quick check that the file size matches. No need to stat()
# again the input file, grab the value from the dict.
out_size = os.stat(outfile).st_size
in_size = complete_state.result.files[relfile]['size']
if in_size == out_size:
continue
# Otherwise, an exception will be raised.
logging.info('%s -> %s' % (relfile, properties['sha-1']))
run_test_from_archive.link_file(
outfile, infile, run_test_from_archive.HARDLINK)
complete_state.save_files()
# Also archive the .result file.
with open(complete_state.result_file, 'rb') as f:
result_hash = hashlib.sha1(f.read()).hexdigest()
logging.info(
'%s -> %s' % (os.path.basename(complete_state.result_file), result_hash))
outfile = os.path.join(options.outdir, result_hash)
if os.path.isfile(outfile):
# Just do a quick check that the file size matches. If they do, skip the
# archive. This mean the build result didn't change at all.
out_size = os.stat(outfile).st_size
in_size = os.stat(complete_state.result_file).st_size
if in_size == out_size:
return 0
run_test_from_archive.link_file(
outfile, complete_state.result_file, run_test_from_archive.HARDLINK)
return 0
def CMDhashtable(args):
"""Creates a hash table content addressed object store.
All the files listed in the .result file are put in the output directory with
the file name being the sha-1 of the file's content.
"""
parser = OptionParserIsolate(command='hashtable')
options, _ = parser.parse_args(args)
success = False
try:
complete_state = load_complete_state(options, WITH_HASH)
options.outdir = (
options.outdir or os.path.join(complete_state.resultdir, 'hashtable'))
recreate_tree(
outdir=options.outdir,
indir=complete_state.root_dir,
infiles=complete_state.result.files,
action=run_test_from_archive.HARDLINK,
as_sha1=True)
complete_state.save_files()
# Also archive the .result file.
with open(complete_state.result_file, 'rb') as f:
result_hash = hashlib.sha1(f.read()).hexdigest()
logging.info(
'%s -> %s' %
(os.path.basename(complete_state.result_file), result_hash))
outfile = os.path.join(options.outdir, result_hash)
if os.path.isfile(outfile):
# Just do a quick check that the file size matches. If they do, skip the
# archive. This mean the build result didn't change at all.
out_size = os.stat(outfile).st_size
in_size = os.stat(complete_state.result_file).st_size
if in_size == out_size:
success = True
return 0
run_test_from_archive.link_file(
outfile, complete_state.result_file, run_test_from_archive.HARDLINK)
success = True
return 0
finally:
# If the command failed, delete the .results file if it exists. This is
# important so no stale swarm job is executed.
if not success and os.path.isfile(options.result):
os.remove(options.result)
def MODEhashtable(outdir, state):
outdir = outdir or os.path.join(state.resultdir, "hashtable")
if not os.path.isdir(outdir):
os.makedirs(outdir)
for relfile, properties in state.result.files.iteritems():
infile = os.path.join(state.root_dir, relfile)
outfile = os.path.join(outdir, properties["sha-1"])
if os.path.isfile(outfile):
# Just do a quick check that the file size matches. No need to stat()
# again the input file, grab the value from the dict.
out_size = os.stat(outfile).st_size
in_size = state.result.files[relfile].get("size") or os.stat(infile).st_size
if in_size == out_size:
continue
# Otherwise, an exception will be raised.
print "Mapping %s -> %s" % (outfile, infile)
run_test_from_archive.link_file(outfile, infile, run_test_from_archive.HARDLINK)
return 0
def MODEhashtable(outdir, state):
outdir = (outdir or os.path.join(state.resultdir, 'hashtable'))
if not os.path.isdir(outdir):
os.makedirs(outdir)
for relfile, properties in state.result.files.iteritems():
infile = os.path.join(state.root_dir, relfile)
outfile = os.path.join(outdir, properties['sha-1'])
if os.path.isfile(outfile):
# Just do a quick check that the file size matches. No need to stat()
# again the input file, grab the value from the dict.
out_size = os.stat(outfile).st_size
in_size = (state.result.files[relfile].get('size')
or os.stat(infile).st_size)
if in_size == out_size:
continue
# Otherwise, an exception will be raised.
print '%s -> %s' % (relfile, properties['sha-1'])
run_test_from_archive.link_file(outfile, infile,
run_test_from_archive.HARDLINK)
return 0
def MODEhashtable(outdir, indir, dictfiles, _read_only, _cmd, _relative_cwd,
resultfile):
outdir = outdir or os.path.join(os.path.dirname(resultfile), 'hashtable')
if not os.path.isdir(outdir):
os.makedirs(outdir)
for relfile, properties in dictfiles.iteritems():
infile = os.path.join(indir, relfile)
outfile = os.path.join(outdir, properties['sha-1'])
if os.path.isfile(outfile):
# Just do a quick check that the file size matches. No need to stat()
# again the input file, grab the value from the dict.
out_size = os.stat(outfile).st_size
in_size = dictfiles.get(infile,
{}).get('size') or os.stat(infile).st_size
if in_size == out_size:
continue
# Otherwise, an exception will be raised.
run_test_from_archive.link_file(outfile, infile,
run_test_from_archive.HARDLINK)
return 0
def MODEhashtable(outdir, indir, data):
outdir = (
outdir or os.path.join(os.path.dirname(data['resultdir']), 'hashtable'))
if not os.path.isdir(outdir):
os.makedirs(outdir)
for relfile, properties in data['files'].iteritems():
infile = os.path.join(indir, relfile)
outfile = os.path.join(outdir, properties['sha-1'])
if os.path.isfile(outfile):
# Just do a quick check that the file size matches. No need to stat()
# again the input file, grab the value from the dict.
out_size = os.stat(outfile).st_size
in_size = (
data.get('files', {}).get(infile, {}).get('size') or
os.stat(infile).st_size)
if in_size == out_size:
continue
# Otherwise, an exception will be raised.
run_test_from_archive.link_file(
outfile, infile, run_test_from_archive.HARDLINK)
return 0
def recreate_tree(outdir, indir, infiles, action):
"""Creates a new tree with only the input files in it.
Arguments:
outdir: Output directory to create the files in.
indir: Root directory the infiles are based in.
infiles: dict of files to map from |indir| to |outdir|.
action: See assert below.
"""
logging.debug(
'recreate_tree(%s, %s, %s, %s)' % (outdir, indir, len(infiles), action))
logging.info('Mapping from %s to %s' % (indir, outdir))
assert action in (
run_test_from_archive.HARDLINK,
run_test_from_archive.SYMLINK,
run_test_from_archive.COPY)
outdir = os.path.normpath(outdir)
if not os.path.isdir(outdir):
logging.info ('Creating %s' % outdir)
os.makedirs(outdir)
# Do not call abspath until the directory exists.
outdir = os.path.abspath(outdir)
for relfile, metadata in infiles.iteritems():
infile = os.path.join(indir, relfile)
outfile = os.path.join(outdir, relfile)
outsubdir = os.path.dirname(outfile)
if not os.path.isdir(outsubdir):
os.makedirs(outsubdir)
if metadata.get('touched_only') == True:
open(outfile, 'ab').close()
elif 'link' in metadata:
pointed = metadata['link']
logging.debug('Symlink: %s -> %s' % (outfile, pointed))
os.symlink(pointed, outfile)
else:
run_test_from_archive.link_file(outfile, infile, action)