def cleanup():
inform("~~~~~~~~EXITING - Cleaning up~~~~~~~~~~~")
try:
if monitor is not None:
monitor.stop()
except Exception,e:
inform("Exception during stopping monitor %s " % e)
def start_working(file): signal.signal(signal.SIGINT, signal.SIG_IGN) inform( "Should be working on %s" % file) ftw = FileTaskWorker(DIR,file,flush_frequency) hbeat = FileTaskWorkerHBeat(ftw) result = False try: hbeat.start() result = ftw.run() hbeat.stop() except KeyboardInterrupt: inform( "Handling keyboard interrupt" ) ftw.stop() if ftw is not None else None result = False except Exception,e: print e result = False
def parse_arguments(help=False):
inform("Cpu count is %d" % cpu_count)
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description="""
Unzip and Map-Reduce Log Files
===================
The most important files are "zip_task_manager.py", "file_task_monitor" -> "file_task_worker"
- Basic Usage
`./run.sh <directory>`
script to execute the script
=============================
- Parts
Use `src/start.py --input-dir=<directory>`
to convert Unzip files into log files
Use `src/file_task_monitor.py --input-dir=<directory>`
to map reduce log files
Use `args.py` or `./run.sh -h`
to print help
""")
parser.add_argument("--input-dir",help="Where to read log files",required=True)
parser.add_argument("--max-proc",help="maximum processor count. Runs as a separate program.",default=cpu_count,type=int)
parser.add_argument("--flush",help="setup how frequent the read write op will be",type=int,default=10000)
if help:
parser.print_help()
sys.exit(0)
else:
args = parser.parse_args()
for attr, value in args.__dict__.iteritems():
inform( "Params: %s = (%s) " % (attr, value) )
if args.max_proc <= 0:
logging.warn("=====Setting Max_Processor=%d. MaxProc(%d) was invalid" % (cpu_count,args.max_proc))
args.max_proc = cpu_count
elif args.max_proc <= 1 and cpu_count > 2:
logging.warn("=====%d CPU exists. You selected only %d Process to run" % (cpu_count,args.max_proc))
elif args.max_proc > cpu_count:
logging.warn("=====%d CPU exist. You selected %d processes to run. Not suggested!!")
if args.input_dir is None or args.input_dir == "":
inform("Directory is not SET! Pass Directory as a parameter")
sys.exit(0)
if args.input_dir[-1] != "/":
args.input_dir += "/"
return args
def signal_handler(signal, frame): inform( "Handling signal %s %s"%(signal,frame )) # monitor.stop() sys.exit(0)
"$set" : { "status" : "quit" }
}, multi=True,upsert=False,safe=True
)
def cleanup():
inform("~~~~~~~~EXITING - Cleaning up~~~~~~~~~~~")
try:
if monitor is not None:
monitor.stop()
except Exception,e:
inform("Exception during stopping monitor %s " % e)
finally:
inform("BYEEE!")
atexit.register(cleanup)
if __name__ == "__main__":
global monitor
global DIR
global flush_frequency
monitor = None
signals_to_handle = [signal.SIGINT,signal.SIGHUP,signal.SIGQUIT,signal.SIGABRT,signal.SIGALRM]
args = parse_arguments()
DIR = ""
flush_frequency = args.flush