def process_dir(bucket, _dir, delete_source, extf, path_move):
logging.progress("Processing: %s" % _dir)
parts=os.path.splitext(_dir)
bn=os.path.basename(parts[0])
filename="%s.%s" % (bn, extf)
logging.progress("Generating: file '%s' in bucket '%s'" % (filename, bucket.name))
k=S3Key(bucket)
k.key=filename
try:
k.set_contents_from_string("")
except:
logging.warning("Can't generate file '%s' in bucket '%s'" % (filename, bucket.name))
return
if delete_source:
logging.progress("Deleting: %s" % _dir)
rmdir(_dir)
return
if path_move is not None:
bn_src_dir=os.path.basename(_dir)
ddir=os.path.join(path_move, bn_src_dir)
rmdir(ddir)
logging.progress("Moving: %s => %s" % (_dir, ddir))
code, msg=move(_dir, ddir)
if not code.startswith("ok"):
logging.warning("Can't move '%s' to '%s': %s" % (_dir, ddir, msg))
def maybe_del_dir(mode_dir, del_dir_dst, bn_dir_src, dir_dst):
dpath=os.path.join(dir_dst, bn_dir_src)
if mode_dir:
dpath=os.path.splitext(dpath)[0]
if del_dir_dst:
logging.debug("Deleting destination directory: %s" % dpath)
rmdir(dpath)
return dpath
def do_move(mode_dir, ext_done, del_dir_dst, dir_src, bn_dir_src, dir_dst, dir_tmp):
"""
- Move path directory to a temporary dir in 'dir_tmp'
- Delete 'file.ext_done' files
- Move path directory to 'dir_dst'
"""
dpath, tdir, tpath=do_common(mode_dir, bn_dir_src, del_dir_dst, dir_src, dir_dst, dir_tmp)
if not mode_dir:
## next, delete all files with extension 'ext_done'
code, maybe_files=get_root_files(tpath)
if not code.startswith("ok"):
rmdir(tdir)
raise Exception("Can't get the filenames of temp dir: %s" % tpath)
def _cmp(path):
return path.endswith(ext_done)
liste=filter(_cmp, maybe_files)
logging.debug("Deleting '%s' files with extension '%s'" % (len(liste), ext_done))
for f in liste:
code, _=rm(f)
if not code.startswith("ok"):
logging.warning("Can't delete file '%s'... aborting" % f)
rmdir(tdir)
return
## last, move to final destination
logging.debug("Moving '%s' to final dir '%s'" % (tpath, dpath))
code, _=move(tpath, dpath)
rmdir(tdir)
logging.debug("Removed temp dir: %s" % tdir)
if not code.startswith("ok"):
raise Exception("Can't move '%s' to directory" % dpath)
logging.info("Processed directory: %s" % dir_src)
def maybe_process_ok(ctx, _ok, _, primary_path, compare_path, just_basename, dest_path):
if dest_path:
logging.debug("Emptying dest path: %s" % dest_path)
rmdir(dest_path)
code, _msg=mkdir_p(dest_path)
if code!="ok":
raise Exception("Can't create path: %s" % dest_path)
### log rate limiter helper -- need to update status in context 'ctx'
doOnTransition(ctx, "status_file.contents", "down", True, None)
codep, primary_files=get_root_files(primary_path, strip_dirname=True)
codec, compare_files=get_root_files(compare_path, strip_dirname=True)
### output some log info on transitions
tm=ctx["tm"]
tm.send(("pp_log", codep=="ok"))
tm.send(("zp_log", codec=="ok"))
### not much to do if either path isn't accessible...
if not codep.startswith("ok") or not codec.startswith("ok"):
return
exts=ctx["exts"]
if exts is not None:
primary_files=filter(filtre(exts), primary_files)
compare_files=filter(filtre(exts), compare_files)
def _mapper(path):
bn=os.path.basename(path)
return os.path.splitext(bn)[0]
if just_basename:
pfiles=map(_mapper, primary_files)
cfiles=map(_mapper, compare_files)
else:
pfiles=primary_files
cfiles=compare_files
try:
setpf=set(pfiles)
setcf=set(cfiles)
common=setpf.intersection(setcf)
diff={
"pp": primary_path
,"zp": compare_path
,"pp-zp": list(setpf-setcf)
,"zp-pp": list(setcf-setpf)
,"common": list(common)
}
topic_name=ctx["topic_name"]
if topic_name is not None:
diff["topic"]=topic_name
if ctx["just_list"]:
doout(ctx, diff, dest_path)
else:
stdoutj(diff)
stdoutf()
except Exception, e:
logging.error("Can't compute diff between paths: %s" % str(e))
zfile=zipfile.ZipFile(zfile_path_tmp, "w")
os.chdir(dir_src)
code, files=get_root_files(dir_src, strip_dirname=True)
if code!="ok":
raise Exception("Can't get files from dir: %s" % dir_src)
for fichier in files:
zfile.write(fichier)
zfile.close()
except Exception, e:
rm(zfile_path_tmp)
raise Exception("Can't generate zip archive: %s (%s)" % (zfile_path_tmp, e))
dfile_path=os.path.join(dir_dst, bn_dir_src+".zip")
logging.debug("Moving to %s" % dfile_path)
os.rename(zfile_path_tmp, dfile_path)
if delete_source:
logging.debug("Deleting source subdir: %s" % dir_src)
rmdir(dir_src)
logging.debug("Processing dir ENDS: %s" % dir_src)
return dfile_path