def recreate_tree(outdir, indir, infiles, action, as_hash): """Creates a new tree with only the input files in it. Arguments: outdir: Output directory to create the files in. indir: Root directory the infiles are based in. infiles: dict of files to map from |indir| to |outdir|. action: One of accepted action of file_path.link_file(). as_hash: Output filename is the hash instead of relfile. """ logging.info( "recreate_tree(outdir=%s, indir=%s, files=%d, action=%s, as_hash=%s)" % (outdir, indir, len(infiles), action, as_hash) ) assert os.path.isabs(outdir) and outdir == os.path.normpath(outdir), outdir if not os.path.isdir(outdir): logging.info("Creating %s" % outdir) os.makedirs(outdir) for relfile, metadata in infiles.iteritems(): infile = os.path.join(indir, relfile) if as_hash: # Do the hashtable specific checks. if "l" in metadata: # Skip links when storing a hashtable. continue outfile = os.path.join(outdir, metadata["h"]) if os.path.isfile(outfile): # Just do a quick check that the file size matches. No need to stat() # again the input file, grab the value from the dict. if not "s" in metadata: raise isolated_format.MappingError("Misconfigured item %s: %s" % (relfile, metadata)) if metadata["s"] == os.stat(outfile).st_size: continue else: logging.warn("Overwritting %s" % metadata["h"]) os.remove(outfile) else: outfile = os.path.join(outdir, relfile) outsubdir = os.path.dirname(outfile) if not os.path.isdir(outsubdir): os.makedirs(outsubdir) if "l" in metadata: pointed = metadata["l"] logging.debug("Symlink: %s -> %s" % (outfile, pointed)) # symlink doesn't exist on Windows. os.symlink(pointed, outfile) # pylint: disable=E1101 else: file_path.link_file(outfile, infile, action)
def recreate_tree(outdir, indir, infiles, action, as_hash): """Creates a new tree with only the input files in it. Arguments: outdir: Output directory to create the files in. indir: Root directory the infiles are based in. infiles: dict of files to map from |indir| to |outdir|. action: One of accepted action of file_path.link_file(). as_hash: Output filename is the hash instead of relfile. """ logging.info( 'recreate_tree(outdir=%s, indir=%s, files=%d, action=%s, as_hash=%s)' % (outdir, indir, len(infiles), action, as_hash)) assert os.path.isabs(outdir) and outdir == os.path.normpath(outdir), outdir if not os.path.isdir(outdir): logging.info('Creating %s' % outdir) os.makedirs(outdir) for relfile, metadata in infiles.iteritems(): infile = os.path.join(indir, relfile) if as_hash: # Do the hashtable specific checks. if 'l' in metadata: # Skip links when storing a hashtable. continue outfile = os.path.join(outdir, metadata['h']) if os.path.isfile(outfile): # Just do a quick check that the file size matches. No need to stat() # again the input file, grab the value from the dict. if not 's' in metadata: raise isolated_format.MappingError( 'Misconfigured item %s: %s' % (relfile, metadata)) if metadata['s'] == os.stat(outfile).st_size: continue else: logging.warn('Overwritting %s' % metadata['h']) os.remove(outfile) else: outfile = os.path.join(outdir, relfile) outsubdir = os.path.dirname(outfile) if not os.path.isdir(outsubdir): os.makedirs(outsubdir) if 'l' in metadata: pointed = metadata['l'] logging.debug('Symlink: %s -> %s' % (outfile, pointed)) # symlink doesn't exist on Windows. os.symlink(pointed, outfile) # pylint: disable=E1101 else: file_path.link_file(outfile, infile, action)
def link_outputs_to_outdir(run_dir, out_dir, outputs): """Links any named outputs to out_dir so they can be uploaded. Raises an error if the file already exists in that directory. """ if not outputs: return isolateserver.create_directories(out_dir, outputs) for o in outputs: try: file_path.link_file(os.path.join(out_dir, o), os.path.join(run_dir, o), file_path.HARDLINK_WITH_FALLBACK) except OSError as e: logging.info("Couldn't collect output file %s: %s", o, e)
def link_outputs_to_outdir(run_dir, out_dir, outputs): """Links any named outputs to out_dir so they can be uploaded. Raises an error if the file already exists in that directory. """ if not outputs: return isolateserver.create_directories(out_dir, outputs) for o in outputs: try: file_path.link_file(os.path.join(out_dir, o), os.path.join(run_dir, o), file_path.HARDLINK_WITH_FALLBACK) except OSError as e: # TODO(aludwin): surface this error sys.stderr.write('<Could not return file %s: %s>' % (o, e))
def link_outputs_to_outdir(run_dir, out_dir, outputs): """Links any named outputs to out_dir so they can be uploaded. Raises an error if the file already exists in that directory. """ if not outputs: return isolateserver.create_directories(out_dir, outputs) for o in outputs: try: infile = os.path.join(run_dir, o) outfile = os.path.join(out_dir, o) if fs.islink(infile): # TODO(aludwin): handle directories fs.copy2(infile, outfile) else: file_path.link_file(outfile, infile, file_path.HARDLINK_WITH_FALLBACK) except OSError as e: logging.info("Couldn't collect output file %s: %s", o, e)
def copy_recursively(src, dst): """Efficiently copies a file or directory from src_dir to dst_dir. `item` may be a file, directory, or a symlink to a file or directory. All symlinks are replaced with their targets, so the resulting directory structure in dst_dir will never have any symlinks. To increase speed, copy_recursively hardlinks individual files into the (newly created) directory structure if possible, unlike Python's shutil.copytree(). """ orig_src = src try: # Replace symlinks with their final target. while fs.islink(src): res = fs.readlink(src) src = os.path.join(os.path.dirname(src), res) # TODO(sadafm): Explicitly handle cyclic symlinks. # Note that fs.isfile (which is a wrapper around os.path.isfile) throws # an exception if src does not exist. A warning will be logged in that case. if fs.isfile(src): file_path.link_file(dst, src, file_path.HARDLINK_WITH_FALLBACK) return if not fs.exists(dst): os.makedirs(dst) for child in fs.listdir(src): copy_recursively(os.path.join(src, child), os.path.join(dst, child)) except OSError as e: if e.errno == errno.ENOENT: logging.warning('Path %s does not exist or %s is a broken symlink', src, orig_src) else: logging.info("Couldn't collect output file %s: %s", src, e)