def process_dir(delete_source,
dir_src,
bn_dir_src,
dir_dst,
dir_tmp):
logging.debug("Processing dir BEGIN: %s" % dir_src)
zfile_path_tmp=tempfile.mktemp(dir=dir_tmp)
logging.debug("Creating archive '%s'" % zfile_path_tmp)
try:
zfile=zipfile.ZipFile(zfile_path_tmp, "w")
os.chdir(dir_src)
code, files=get_root_files(dir_src, strip_dirname=True)
if code!="ok":
raise Exception("Can't get files from dir: %s" % dir_src)
for fichier in files:
zfile.write(fichier)
zfile.close()
except Exception, e:
rm(zfile_path_tmp)
raise Exception("Can't generate zip archive: %s (%s)" % (zfile_path_tmp, e))
def process_dir(ext_done, mode_dir, del_dir_dst, dir_src, bn_dir_src, dir_dst, dir_tmp):
logging.debug("Processing dir: %s" % dir_src)
if not mode_dir:
code, maybe_files=get_root_files(dir_src, strip_dirname=True)
if not code.startswith("ok"):
raise Exception("Can't get the filenames of dir: %s" % dir_src)
for f in maybe_files:
if not f.endswith(ext_done):
df="%s.%s" % (f, ext_done)
if not df in maybe_files:
logging.debug("Missing 'done' file for: %s" % f)
return
do_move(mode_dir, ext_done, del_dir_dst, dir_src, bn_dir_src, dir_dst, dir_tmp)
def do_move(mode_dir, ext_done, del_dir_dst, dir_src, bn_dir_src, dir_dst, dir_tmp):
"""
- Move path directory to a temporary dir in 'dir_tmp'
- Delete 'file.ext_done' files
- Move path directory to 'dir_dst'
"""
dpath, tdir, tpath=do_common(mode_dir, bn_dir_src, del_dir_dst, dir_src, dir_dst, dir_tmp)
if not mode_dir:
## next, delete all files with extension 'ext_done'
code, maybe_files=get_root_files(tpath)
if not code.startswith("ok"):
rmdir(tdir)
raise Exception("Can't get the filenames of temp dir: %s" % tpath)
def _cmp(path):
return path.endswith(ext_done)
liste=filter(_cmp, maybe_files)
logging.debug("Deleting '%s' files with extension '%s'" % (len(liste), ext_done))
for f in liste:
code, _=rm(f)
if not code.startswith("ok"):
logging.warning("Can't delete file '%s'... aborting" % f)
rmdir(tdir)
return
## last, move to final destination
logging.debug("Moving '%s' to final dir '%s'" % (tpath, dpath))
code, _=move(tpath, dpath)
rmdir(tdir)
logging.debug("Removed temp dir: %s" % tdir)
if not code.startswith("ok"):
raise Exception("Can't move '%s' to directory" % dpath)
logging.info("Processed directory: %s" % dir_src)
def run(path_source=None
,polling_interval=None
,ext_include=None
,ext_exclude=None
,batch_size=None
,**_
):
if ext_include is not None and ext_exclude is not None:
raise Exception("'ee' and 'ei' options are mutually exclusive")
criteria="include" if ext_include else "exclude"
elist=ext_include or ext_exclude
code, path_source=resolve_path(path_source)
if not code.startswith("ok"):
raise Exception("can't resolve path '%s'" % path_source)
ppid=os.getppid()
logging.info("Process pid: %s" % os.getpid())
logging.info("Parent pid : %s" % ppid)
logging.info("Starting loop...")
while True:
if os.getppid()!=ppid:
logging.warning("Parent terminated... exiting")
break
code, files=get_root_files(path_source)
l=filter_files_by_ext(criteria, elist, (code, files))
if l:
for bunch in batch(l, batch_size):
output(path_source, bunch)
logging.debug("...sleeping for %s seconds" % polling_interval)
sleep(polling_interval)
def run(source_path=None, move_path=None, check_path=None,
batch_size=5,
polling_interval=None, enable_delete=False
,**_):
if check_path is not None:
ct=check_transition()
if enable_delete and move_path is not None:
raise Exception("Options '-mp' and '-d' are mutually exclusive")
code, rp=resolve_path(source_path)
if not code.startswith("ok"):
raise Exception("can't resolve source path '%s'" % source_path)
source_path=rp
if move_path is not None:
code, rp=resolve_path(move_path)
if not code.startswith("ok"):
raise Exception("can't resolve 'move_path' '%s'" % move_path)
move_path=rp
logging.info("Creating (if necessary) 'move' path: %s" % move_path)
code, msg=mkdir_p(move_path)
if not code.startswith("ok"):
raise Exception("Can't create move path '%s': %s" % (move_path, str(msg)))
logging.info("Checking if 'move' directory is writable")
code, msg=can_write(move_path)
if not code.startswith("ok"):
raise Exception("Can't write to 'move' directory")
to_skip=[]
logging.info("Process pid: %s" % os.getpid())
ppid=os.getppid()
logging.info("Parent pid : %s" % ppid)
logging.info("Starting loop...")
while True:
if os.getppid()!=ppid:
logging.warning("Parent terminated... exiting")
break
if check_path is not None:
try: exists=os.path.exists(check_path)
except: exists=False
maybe_tr, _=ct.send(exists)
if maybe_tr=="tr" and exists:
logging.info("Check path: passed")
if maybe_tr=="tr" and not exists:
logging.info("Check path: failed - skipping")
else:
## fake 'exists'
exists=True
if exists:
code, files=get_root_files(source_path)
if not code.startswith("ok"):
logging.error("Can't get root files from %s" % source_path)
else:
###############################################################
files=files[:batch_size]
try:
for src_file in files:
if src_file in to_skip:
continue
code, _=can_write(src_file)
if not code.startswith("ok"):
to_skip.append(src_file)
logging.error("Would not be able to move/delete source file '%s'... skipping streaming" % src_file)
continue
dst_file=None
if move_path is not None:
bn=os.path.basename(src_file)
dst_file=os.path.join(move_path, bn)
code, maybe_error=process(src_file, dst_file, enable_delete)
if not code.startswith("ok"):
to_skip.append(src_file)
logging.warning("Problem processing file '%s': %s" % (src_file, maybe_error))
except BrokenPipe:
raise
except KeyboardInterrupt:
raise
except Exception, e:
logging.error("processing file '%s': %s" % (src_file, str(e)))
###############################################################
logging.debug("...sleeping for %s seconds" % polling_interval)
sleep(polling_interval)
def maybe_process_ok(ctx, _ok, _, primary_path, compare_path, just_basename, dest_path):
if dest_path:
logging.debug("Emptying dest path: %s" % dest_path)
rmdir(dest_path)
code, _msg=mkdir_p(dest_path)
if code!="ok":
raise Exception("Can't create path: %s" % dest_path)
### log rate limiter helper -- need to update status in context 'ctx'
doOnTransition(ctx, "status_file.contents", "down", True, None)
codep, primary_files=get_root_files(primary_path, strip_dirname=True)
codec, compare_files=get_root_files(compare_path, strip_dirname=True)
### output some log info on transitions
tm=ctx["tm"]
tm.send(("pp_log", codep=="ok"))
tm.send(("zp_log", codec=="ok"))
### not much to do if either path isn't accessible...
if not codep.startswith("ok") or not codec.startswith("ok"):
return
exts=ctx["exts"]
if exts is not None:
primary_files=filter(filtre(exts), primary_files)
compare_files=filter(filtre(exts), compare_files)
def _mapper(path):
bn=os.path.basename(path)
return os.path.splitext(bn)[0]
if just_basename:
pfiles=map(_mapper, primary_files)
cfiles=map(_mapper, compare_files)
else:
pfiles=primary_files
cfiles=compare_files
try:
setpf=set(pfiles)
setcf=set(cfiles)
common=setpf.intersection(setcf)
diff={
"pp": primary_path
,"zp": compare_path
,"pp-zp": list(setpf-setcf)
,"zp-pp": list(setcf-setpf)
,"common": list(common)
}
topic_name=ctx["topic_name"]
if topic_name is not None:
diff["topic"]=topic_name
if ctx["just_list"]:
doout(ctx, diff, dest_path)
else:
stdoutj(diff)
stdoutf()
except Exception, e:
logging.error("Can't compute diff between paths: %s" % str(e))
def run(source_path=None, dest_path=None, check_path=None,
batch_size=5,
polling_interval=None, delete_fetch_error=False
,**_):
if check_path is not None:
ct=check_transition()
logging.info("Creating (if necessary) destination path: %s" % dest_path)
code, msg=mkdir_p(dest_path)
if not code.startswith("ok"):
raise Exception("Can't create destination path '%s': %s" % (dest_path, str(msg)))
to_skip=[]
ppid=os.getppid()
logging.info("Process pid: %s" % os.getpid())
logging.info("Parent pid : %s" % ppid)
logging.info("Starting loop...")
while True:
if os.getppid()!=ppid:
logging.warning("Parent terminated... exiting")
break
if check_path is not None:
try: exists=os.path.exists(check_path)
except: exists=False
maybe_tr, _=ct.send(exists)
if maybe_tr=="tr" and exists:
logging.info("Check path: passed")
if maybe_tr=="tr" and not exists:
logging.info("Check path: failed - skipping")
else:
## fake 'exists'
exists=True
if exists:
code, files=get_root_files(source_path)
if not code.startswith("ok"):
logging.error("Can't get root files from %s" % source_path)
continue
###############################################################
files=files[:batch_size]
try:
for src_file in files:
if src_file in to_skip:
continue
code, _=can_write(src_file)
if not code.startswith("ok"):
to_skip.append(src_file)
logging.error("Would not be able to delete source file '%s'... skipping download" % src_file)
continue
process(src_file, dest_path, delete_fetch_error)
except BrokenPipe:
raise
except Exception, e:
logging.error("processing file '%s': %s" % (src_file, str(e)))
###############################################################
logging.debug("...waiting for %s seconds (max)" % polling_interval)
### Implement a "pass-through" for stdin --> stdout
### whilst also handling a maximum time-out
start_time=time.time()
while True:
ir, _w, _e=select.select([sys.stdin], [], [], polling_interval)
if len(ir):
iline=sys.stdin.readline()
sys.stdout.write(iline)
elapsed_time = time.time() - start_time
if elapsed_time > polling_interval:
break