def recreate_tree(outdir, indir, infiles, action, as_hash):
"""Creates a new tree with only the input files in it.
Arguments:
outdir: Output directory to create the files in.
indir: Root directory the infiles are based in.
infiles: dict of files to map from |indir| to |outdir|.
action: One of accepted action of file_path.link_file().
as_hash: Output filename is the hash instead of relfile.
"""
logging.info(
"recreate_tree(outdir=%s, indir=%s, files=%d, action=%s, as_hash=%s)"
% (outdir, indir, len(infiles), action, as_hash)
)
assert os.path.isabs(outdir) and outdir == os.path.normpath(outdir), outdir
if not os.path.isdir(outdir):
logging.info("Creating %s" % outdir)
os.makedirs(outdir)
for relfile, metadata in infiles.iteritems():
infile = os.path.join(indir, relfile)
if as_hash:
# Do the hashtable specific checks.
if "l" in metadata:
# Skip links when storing a hashtable.
continue
outfile = os.path.join(outdir, metadata["h"])
if os.path.isfile(outfile):
# Just do a quick check that the file size matches. No need to stat()
# again the input file, grab the value from the dict.
if not "s" in metadata:
raise isolated_format.MappingError("Misconfigured item %s: %s" % (relfile, metadata))
if metadata["s"] == os.stat(outfile).st_size:
continue
else:
logging.warn("Overwritting %s" % metadata["h"])
os.remove(outfile)
else:
outfile = os.path.join(outdir, relfile)
outsubdir = os.path.dirname(outfile)
if not os.path.isdir(outsubdir):
os.makedirs(outsubdir)
if "l" in metadata:
pointed = metadata["l"]
logging.debug("Symlink: %s -> %s" % (outfile, pointed))
# symlink doesn't exist on Windows.
os.symlink(pointed, outfile) # pylint: disable=E1101
else:
file_path.link_file(outfile, infile, action)
def recreate_tree(outdir, indir, infiles, action, as_hash):
"""Creates a new tree with only the input files in it.
Arguments:
outdir: Output directory to create the files in.
indir: Root directory the infiles are based in.
infiles: dict of files to map from |indir| to |outdir|.
action: One of accepted action of file_path.link_file().
as_hash: Output filename is the hash instead of relfile.
"""
logging.info(
'recreate_tree(outdir=%s, indir=%s, files=%d, action=%s, as_hash=%s)' %
(outdir, indir, len(infiles), action, as_hash))
assert os.path.isabs(outdir) and outdir == os.path.normpath(outdir), outdir
if not os.path.isdir(outdir):
logging.info('Creating %s' % outdir)
os.makedirs(outdir)
for relfile, metadata in infiles.iteritems():
infile = os.path.join(indir, relfile)
if as_hash:
# Do the hashtable specific checks.
if 'l' in metadata:
# Skip links when storing a hashtable.
continue
outfile = os.path.join(outdir, metadata['h'])
if os.path.isfile(outfile):
# Just do a quick check that the file size matches. No need to stat()
# again the input file, grab the value from the dict.
if not 's' in metadata:
raise isolated_format.MappingError(
'Misconfigured item %s: %s' % (relfile, metadata))
if metadata['s'] == os.stat(outfile).st_size:
continue
else:
logging.warn('Overwritting %s' % metadata['h'])
os.remove(outfile)
else:
outfile = os.path.join(outdir, relfile)
outsubdir = os.path.dirname(outfile)
if not os.path.isdir(outsubdir):
os.makedirs(outsubdir)
if 'l' in metadata:
pointed = metadata['l']
logging.debug('Symlink: %s -> %s' % (outfile, pointed))
# symlink doesn't exist on Windows.
os.symlink(pointed, outfile) # pylint: disable=E1101
else:
file_path.link_file(outfile, infile, action)
def link_outputs_to_outdir(run_dir, out_dir, outputs):
"""Links any named outputs to out_dir so they can be uploaded.
Raises an error if the file already exists in that directory.
"""
if not outputs:
return
isolateserver.create_directories(out_dir, outputs)
for o in outputs:
try:
file_path.link_file(os.path.join(out_dir, o),
os.path.join(run_dir, o),
file_path.HARDLINK_WITH_FALLBACK)
except OSError as e:
logging.info("Couldn't collect output file %s: %s", o, e)
def link_outputs_to_outdir(run_dir, out_dir, outputs):
"""Links any named outputs to out_dir so they can be uploaded.
Raises an error if the file already exists in that directory.
"""
if not outputs:
return
isolateserver.create_directories(out_dir, outputs)
for o in outputs:
try:
file_path.link_file(os.path.join(out_dir, o),
os.path.join(run_dir, o),
file_path.HARDLINK_WITH_FALLBACK)
except OSError as e:
# TODO(aludwin): surface this error
sys.stderr.write('<Could not return file %s: %s>' % (o, e))
def link_outputs_to_outdir(run_dir, out_dir, outputs):
"""Links any named outputs to out_dir so they can be uploaded.
Raises an error if the file already exists in that directory.
"""
if not outputs:
return
isolateserver.create_directories(out_dir, outputs)
for o in outputs:
try:
infile = os.path.join(run_dir, o)
outfile = os.path.join(out_dir, o)
if fs.islink(infile):
# TODO(aludwin): handle directories
fs.copy2(infile, outfile)
else:
file_path.link_file(outfile, infile, file_path.HARDLINK_WITH_FALLBACK)
except OSError as e:
logging.info("Couldn't collect output file %s: %s", o, e)
def copy_recursively(src, dst):
"""Efficiently copies a file or directory from src_dir to dst_dir.
`item` may be a file, directory, or a symlink to a file or directory.
All symlinks are replaced with their targets, so the resulting
directory structure in dst_dir will never have any symlinks.
To increase speed, copy_recursively hardlinks individual files into the
(newly created) directory structure if possible, unlike Python's
shutil.copytree().
"""
orig_src = src
try:
# Replace symlinks with their final target.
while fs.islink(src):
res = fs.readlink(src)
src = os.path.join(os.path.dirname(src), res)
# TODO(sadafm): Explicitly handle cyclic symlinks.
# Note that fs.isfile (which is a wrapper around os.path.isfile) throws
# an exception if src does not exist. A warning will be logged in that case.
if fs.isfile(src):
file_path.link_file(dst, src, file_path.HARDLINK_WITH_FALLBACK)
return
if not fs.exists(dst):
os.makedirs(dst)
for child in fs.listdir(src):
copy_recursively(os.path.join(src, child), os.path.join(dst, child))
except OSError as e:
if e.errno == errno.ENOENT:
logging.warning('Path %s does not exist or %s is a broken symlink',
src, orig_src)
else:
logging.info("Couldn't collect output file %s: %s", src, e)
