def _fec_decode(ns): logging.debug('UNFEC pass started') tmpd = tempfile.mkdtemp(dir=os.getcwd()) logging.debug('created tempdir at %s' % tmpd) # walk first input dir, decode as we go along for root, dirs, files in os.walk(ns.inputs[0]): unrooted = os.path.relpath(root, ns.inputs[0]) logging.debug('unrooted path: %s' % unrooted) for dname in dirs: osubdir = os.path.join(tmpd, dname) os.mkdir(osubdir) logging.debug('created: %s' % osubdir) for f in files: # get real name rname = re.split('\.[0-9]*_[0-9]*\.fec$', f, re.IGNORECASE)[0] logging.debug('processing chunks for file: %s' % rname) # get all the file chunks into a list fecs = [] for indir in ns.inputs: gpath = common.fec_glob(os.path.join(indir, unrooted, rname)) fecs.extend(glob.glob(gpath)) logging.debug('FEC chunks found for %s: %s' % (rname, fecs)) fec_fds = [open(fec, 'rb') for fec in fecs] try: outpath = os.path.join(tmpd, unrooted, rname) outfd = open(outpath, 'wb') filefec.decode_from_files(outfd, fec_fds, False) logging.debug('decoded successfully to %s' % outpath) except filefec.InsufficientShareFilesError as e: logging.debug('failed to write %s' % outpath) sys.stderr.write(repr(e)) common.cleanup(tmpd) sys.exit(ERR['INSUF_SHARES']) # all done, rename to output dir if os.path.exists(ns.output) and ns.force: shutil.rmtree(ns.output) logging.debug('removed existing output dir at %s' % ns.output) shutil.move(tmpd, ns.output) logging.debug('renamed temp dir %s to output dir %s' % (tmpd, ns.output)) logging.info('UNFEC pass completed')
def _fec_encode(ns): logging.info('FEC pass started') tmpd = tempfile.mkdtemp(dir=os.getcwd()) logging.debug('created temp dir at %s' % tmpd) # total shares tshares = len(ns.outputs) for root, dirs, files in os.walk(ns.input): # output dir, name mapping od = root.replace(ns.input, os.path.basename(tmpd)) # recreate tree structure in temp dir for dname in dirs: osubdir = os.path.join(tmpd, dname) os.mkdir(osubdir) logging.debug('created %s' % osubdir) for f in files: fpath = os.path.join(root, f) logging.debug('processing file: %s' % fpath) with open(os.path.join(root, f)) as fd: fsize = os.path.getsize(fpath) logging.debug('FEC %s (%d bytes)' % (fpath, fsize)) filefec.encode_to_files(fd, fsize, od, f, ns.shares, tshares, '.fec', ns.force, False) logging.info('FEC pass completed') logging.info('Distribution pass started') for root, dirs, files in os.walk(ns.input): unrooted = os.path.relpath(root, ns.input) logging.debug('unrooted path: %s' % unrooted) # map dir tree structure unto output directories for outdir in ns.outputs: for dname in dirs: try: osubdir = os.path.join(outdir, dname) os.mkdir(osubdir) logging.debug('created %s' % osubdir) except OSError: logging.debug('exists: %s' % osubdir) for f in files: # glob on FEC output files to build list of things to distribute gexpr = common.fec_glob(f) gpath = os.path.join(tmpd, unrooted, gexpr) logging.debug('glob path for %s: %s' % (f, gpath)) fecs = [os.path.basename(fec) for fec in glob.glob(gpath)] logging.debug('FEC chunks for %s: %s' % (f, fecs)) if len(fecs) != tshares: logging.debug('len(fecs)=%d;shares=%d' % (len(fecs), tshares)) sys.stdout.write('Chunks and output dir counts mismatch\n') common.cleanup(tmpd) sys.exit(ERR['CHUNK_COUNT_MISMATCH']) # spread chunks over output dirs for idx, fec in enumerate(fecs): ofec = os.path.join(ns.outputs[idx], unrooted, fec) if not ns.force and os.path.exists(ofec): logging.debug('chunk collision: %s' % ofec) sys.stderr.write('Some chunks with the same name exist\n') common.cleanup(tmpd) sys.exit(ERR['NO_OVERWRITE']) ifec = os.path.join(tmpd, unrooted, fec) logging.debug('input FEC for %s: %s' % (f, ifec)) shutil.copyfile(ifec, ofec) logging.debug('wrote %s' % ofec) logging.info('Distribution pass completed') common.cleanup(tmpd)