def simulate(fil, s3key_name, enable_delete, p_dst):
pprint_kv("File to be uploaded", fil)
pprint_kv(" Filename used on s3", s3key_name)
if enable_delete:
_c, f_is_writable = can_write(fil)
pprint_kv(" File would be deleted", fil)
if not f_is_writable:
pprint_kv(" ! File can't be deleted", fil)
else:
if p_dst is not None:
bname = os.path.basename(fil)
fdst = os.path.join(p_dst, bname)
_c, d_is_writable = can_write(p_dst)
pprint_kv(" File would be moved to", fdst)
if not d_is_writable:
pprint_kv(" ! File can't be moved to", p_dst)
def run(
dst_path=None,
topic_name=None,
queue_name=None,
mod_sub=None,
mod_pub=None,
polling_interval=None,
force=False,
node_id=None,
force_delete=False,
delete_queue=None,
proto_n=None,
proto_m=None,
**_
):
signal.signal(signal.SIGTERM, handlerSigTerm)
### STARTUP CHECKS
##################
if proto_n > proto_m:
raise Exception("Parameter 'n' must be smaller than 'm'")
if os.path.isdir(dst_path):
raise Exception("'dst_path' must be a filename, not a directory")
if force_delete:
logging.info("Attempting to delete '%s'" % dst_path)
rm(dst_path)
if os.path.isfile(dst_path):
raise Exception("'dst_path' must not exists at startup: use -fd to delete")
dir_path = os.path.dirname(dst_path)
code, _msg = can_write(dir_path)
if not code.startswith("ok"):
raise Exception("directory '%s' is not writable" % dir_path)
### SETUP
###########################
if node_id is None:
node_id = str(uuid.uuid1())
logging.info("Node id: %s" % node_id)
proc = protocol_processor(node_id, proto_n, proto_m)
### START MAIN LOOP
######################################
run_aws(node_id, proc, polling_interval, queue_name, topic_name, dst_path, delete_queue)
"""
def run(source_path=None, move_path=None, check_path=None,
batch_size=5,
polling_interval=None, enable_delete=False
,**_):
if check_path is not None:
ct=check_transition()
if enable_delete and move_path is not None:
raise Exception("Options '-mp' and '-d' are mutually exclusive")
code, rp=resolve_path(source_path)
if not code.startswith("ok"):
raise Exception("can't resolve source path '%s'" % source_path)
source_path=rp
if move_path is not None:
code, rp=resolve_path(move_path)
if not code.startswith("ok"):
raise Exception("can't resolve 'move_path' '%s'" % move_path)
move_path=rp
logging.info("Creating (if necessary) 'move' path: %s" % move_path)
code, msg=mkdir_p(move_path)
if not code.startswith("ok"):
raise Exception("Can't create move path '%s': %s" % (move_path, str(msg)))
logging.info("Checking if 'move' directory is writable")
code, msg=can_write(move_path)
if not code.startswith("ok"):
raise Exception("Can't write to 'move' directory")
to_skip=[]
logging.info("Process pid: %s" % os.getpid())
ppid=os.getppid()
logging.info("Parent pid : %s" % ppid)
logging.info("Starting loop...")
while True:
if os.getppid()!=ppid:
logging.warning("Parent terminated... exiting")
break
if check_path is not None:
try: exists=os.path.exists(check_path)
except: exists=False
maybe_tr, _=ct.send(exists)
if maybe_tr=="tr" and exists:
logging.info("Check path: passed")
if maybe_tr=="tr" and not exists:
logging.info("Check path: failed - skipping")
else:
## fake 'exists'
exists=True
if exists:
code, files=get_root_files(source_path)
if not code.startswith("ok"):
logging.error("Can't get root files from %s" % source_path)
else:
###############################################################
files=files[:batch_size]
try:
for src_file in files:
if src_file in to_skip:
continue
code, _=can_write(src_file)
if not code.startswith("ok"):
to_skip.append(src_file)
logging.error("Would not be able to move/delete source file '%s'... skipping streaming" % src_file)
continue
dst_file=None
if move_path is not None:
bn=os.path.basename(src_file)
dst_file=os.path.join(move_path, bn)
code, maybe_error=process(src_file, dst_file, enable_delete)
if not code.startswith("ok"):
to_skip.append(src_file)
logging.warning("Problem processing file '%s': %s" % (src_file, maybe_error))
except BrokenPipe:
raise
except KeyboardInterrupt:
raise
except Exception, e:
logging.error("processing file '%s': %s" % (src_file, str(e)))
###############################################################
logging.debug("...sleeping for %s seconds" % polling_interval)
sleep(polling_interval)
def run(source_path=None, dest_path=None, check_path=None,
batch_size=5,
polling_interval=None, delete_fetch_error=False
,**_):
if check_path is not None:
ct=check_transition()
logging.info("Creating (if necessary) destination path: %s" % dest_path)
code, msg=mkdir_p(dest_path)
if not code.startswith("ok"):
raise Exception("Can't create destination path '%s': %s" % (dest_path, str(msg)))
to_skip=[]
ppid=os.getppid()
logging.info("Process pid: %s" % os.getpid())
logging.info("Parent pid : %s" % ppid)
logging.info("Starting loop...")
while True:
if os.getppid()!=ppid:
logging.warning("Parent terminated... exiting")
break
if check_path is not None:
try: exists=os.path.exists(check_path)
except: exists=False
maybe_tr, _=ct.send(exists)
if maybe_tr=="tr" and exists:
logging.info("Check path: passed")
if maybe_tr=="tr" and not exists:
logging.info("Check path: failed - skipping")
else:
## fake 'exists'
exists=True
if exists:
code, files=get_root_files(source_path)
if not code.startswith("ok"):
logging.error("Can't get root files from %s" % source_path)
continue
###############################################################
files=files[:batch_size]
try:
for src_file in files:
if src_file in to_skip:
continue
code, _=can_write(src_file)
if not code.startswith("ok"):
to_skip.append(src_file)
logging.error("Would not be able to delete source file '%s'... skipping download" % src_file)
continue
process(src_file, dest_path, delete_fetch_error)
except BrokenPipe:
raise
except Exception, e:
logging.error("processing file '%s': %s" % (src_file, str(e)))
###############################################################
logging.debug("...waiting for %s seconds (max)" % polling_interval)
### Implement a "pass-through" for stdin --> stdout
### whilst also handling a maximum time-out
start_time=time.time()
while True:
ir, _w, _e=select.select([sys.stdin], [], [], polling_interval)
if len(ir):
iline=sys.stdin.readline()
sys.stdout.write(iline)
elapsed_time = time.time() - start_time
if elapsed_time > polling_interval:
break
def run(dest_path=None
,**_):
if dest_path is not None:
if not os.path.isdir(dest_path):
raise Exception("Expecting a valid destination path '%s'" % dest_path)
ppid=os.getppid()
logging.info("Process pid: %s" % os.getpid())
logging.info("Parent pid : %s" % ppid)
logging.info("Starting loop...")
while True:
if os.getppid()!=ppid:
logging.warning("Parent terminated... exiting")
break
iline=sys.stdin.readline().strip()
#################### VALIDATE
## if we received two strings on the same line: url dst_path
bits=iline.split(" ")
l=len(bits)
if l > 2 or l==0:
logging.error("Invalid input line: %s" % iline)
continue
url=bits[0]
bn=os.path.basename(url)
if len(bits)==2:
path=bits[1]
else:
if dest_path is not None:
path=os.path.join(dest_path, bn)
else:
logging.warning("Didn't receive 'dest_path' from stdin and none specified on command line...")
continue
####### WRITE CAPABILITY VERIFICATION
code, result=can_write(path)
if not code.startswith("ok") or not result:
logging.warning("Won't be able to write to path '%s'... skipping download" % path)
continue
####### DOWNLOAD
code, (http_code, headers, data)=fetch(url)
if not code.startswith("ok"):
logging.warning("Error attempting to download: %s" % url)
continue
try: http_code=int(http_code)
except: pass
if http_code!=200:
logging.warning("Can't fetch url '%s', http response code: %s" % (url, http_code))
continue
code, msg=atomic_write(path, data)
if not code.startswith("ok"):
raise Exception("Can't write to file '%s': %s" % (path, msg))
ctx={
"dest_filename": path
,"url": url
,"http_code": http_code
,"headers": headers
}
try: sys.stdout.write(json.dumps(ctx)+"\n")
except:
raise Exception("Exiting... probably broken pipe")