def _fec_decode(ns):
logging.debug('UNFEC pass started')
tmpd = tempfile.mkdtemp(dir=os.getcwd())
logging.debug('created tempdir at %s' % tmpd)
# walk first input dir, decode as we go along
for root, dirs, files in os.walk(ns.inputs[0]):
unrooted = os.path.relpath(root, ns.inputs[0])
logging.debug('unrooted path: %s' % unrooted)
for dname in dirs:
osubdir = os.path.join(tmpd, dname)
os.mkdir(osubdir)
logging.debug('created: %s' % osubdir)
for f in files:
# get real name
rname = re.split('\.[0-9]*_[0-9]*\.fec$', f, re.IGNORECASE)[0]
logging.debug('processing chunks for file: %s' % rname)
# get all the file chunks into a list
fecs = []
for indir in ns.inputs:
gpath = common.fec_glob(os.path.join(indir, unrooted, rname))
fecs.extend(glob.glob(gpath))
logging.debug('FEC chunks found for %s: %s' % (rname, fecs))
fec_fds = [open(fec, 'rb') for fec in fecs]
try:
outpath = os.path.join(tmpd, unrooted, rname)
outfd = open(outpath, 'wb')
filefec.decode_from_files(outfd, fec_fds, False)
logging.debug('decoded successfully to %s' % outpath)
except filefec.InsufficientShareFilesError as e:
logging.debug('failed to write %s' % outpath)
sys.stderr.write(repr(e))
common.cleanup(tmpd)
sys.exit(ERR['INSUF_SHARES'])
# all done, rename to output dir
if os.path.exists(ns.output) and ns.force:
shutil.rmtree(ns.output)
logging.debug('removed existing output dir at %s' % ns.output)
shutil.move(tmpd, ns.output)
logging.debug('renamed temp dir %s to output dir %s' % (tmpd, ns.output))
logging.info('UNFEC pass completed')
def _fec_encode(ns):
logging.info('FEC pass started')
tmpd = tempfile.mkdtemp(dir=os.getcwd())
logging.debug('created temp dir at %s' % tmpd)
# total shares
tshares = len(ns.outputs)
for root, dirs, files in os.walk(ns.input):
# output dir, name mapping
od = root.replace(ns.input, os.path.basename(tmpd))
# recreate tree structure in temp dir
for dname in dirs:
osubdir = os.path.join(tmpd, dname)
os.mkdir(osubdir)
logging.debug('created %s' % osubdir)
for f in files:
fpath = os.path.join(root, f)
logging.debug('processing file: %s' % fpath)
with open(os.path.join(root, f)) as fd:
fsize = os.path.getsize(fpath)
logging.debug('FEC %s (%d bytes)' % (fpath, fsize))
filefec.encode_to_files(fd, fsize, od, f, ns.shares, tshares,
'.fec', ns.force, False)
logging.info('FEC pass completed')
logging.info('Distribution pass started')
for root, dirs, files in os.walk(ns.input):
unrooted = os.path.relpath(root, ns.input)
logging.debug('unrooted path: %s' % unrooted)
# map dir tree structure unto output directories
for outdir in ns.outputs:
for dname in dirs:
try:
osubdir = os.path.join(outdir, dname)
os.mkdir(osubdir)
logging.debug('created %s' % osubdir)
except OSError:
logging.debug('exists: %s' % osubdir)
for f in files:
# glob on FEC output files to build list of things to distribute
gexpr = common.fec_glob(f)
gpath = os.path.join(tmpd, unrooted, gexpr)
logging.debug('glob path for %s: %s' % (f, gpath))
fecs = [os.path.basename(fec) for fec in glob.glob(gpath)]
logging.debug('FEC chunks for %s: %s' % (f, fecs))
if len(fecs) != tshares:
logging.debug('len(fecs)=%d;shares=%d' % (len(fecs), tshares))
sys.stdout.write('Chunks and output dir counts mismatch\n')
common.cleanup(tmpd)
sys.exit(ERR['CHUNK_COUNT_MISMATCH'])
# spread chunks over output dirs
for idx, fec in enumerate(fecs):
ofec = os.path.join(ns.outputs[idx], unrooted, fec)
if not ns.force and os.path.exists(ofec):
logging.debug('chunk collision: %s' % ofec)
sys.stderr.write('Some chunks with the same name exist\n')
common.cleanup(tmpd)
sys.exit(ERR['NO_OVERWRITE'])
ifec = os.path.join(tmpd, unrooted, fec)
logging.debug('input FEC for %s: %s' % (f, ifec))
shutil.copyfile(ifec, ofec)
logging.debug('wrote %s' % ofec)
logging.info('Distribution pass completed')
common.cleanup(tmpd)
