def cleanup():
	inform("~~~~~~~~EXITING - Cleaning up~~~~~~~~~~~")
	try:
		if monitor is not None:
			monitor.stop()
	except Exception,e:
		inform("Exception during stopping monitor %s " % e)
def start_working(file):
	signal.signal(signal.SIGINT, signal.SIG_IGN)
	inform( "Should be working on %s" % file)
	ftw = FileTaskWorker(DIR,file,flush_frequency)
	hbeat = FileTaskWorkerHBeat(ftw)
	result = False

	try:
		hbeat.start()
		result = ftw.run()

		hbeat.stop()

	except KeyboardInterrupt:
		inform( "Handling keyboard interrupt" )
		ftw.stop() if ftw is not None else None
		result = False
	except Exception,e:
		print e
		result = False
Beispiel #3
0
def parse_arguments(help=False):

	inform("Cpu count is %d" % cpu_count)

	parser = argparse.ArgumentParser(
									 formatter_class=argparse.RawDescriptionHelpFormatter,
									 description="""
Unzip and Map-Reduce Log Files
===================
The most important files are "zip_task_manager.py", "file_task_monitor" -> "file_task_worker"
- Basic Usage
	`./run.sh <directory>`
script to execute the script
=============================
- Parts
Use `src/start.py --input-dir=<directory>`
to convert Unzip files into log files

Use `src/file_task_monitor.py --input-dir=<directory>`
to map reduce log files

Use `args.py` or `./run.sh -h`
to print help
""")

	parser.add_argument("--input-dir",help="Where to read log files",required=True)
	parser.add_argument("--max-proc",help="maximum processor count. Runs as a separate program.",default=cpu_count,type=int)
	parser.add_argument("--flush",help="setup how frequent the read write op will be",type=int,default=10000)

	if help:
		parser.print_help()
		sys.exit(0)
	else:
		args = parser.parse_args()

	for attr, value in args.__dict__.iteritems():
		inform( "Params: %s = (%s) " % (attr, value) )

	if args.max_proc <= 0:
		logging.warn("=====Setting Max_Processor=%d. MaxProc(%d) was invalid" % (cpu_count,args.max_proc))
		args.max_proc = cpu_count
	elif args.max_proc <= 1 and cpu_count > 2:
		logging.warn("=====%d CPU exists. You selected only %d Process to run" % (cpu_count,args.max_proc))
	elif args.max_proc > cpu_count:
		logging.warn("=====%d CPU exist. You selected %d processes to run. Not suggested!!")

	if args.input_dir is None or args.input_dir == "":
		inform("Directory is not SET! Pass Directory as a parameter")
		sys.exit(0)

	if args.input_dir[-1] != "/":
		args.input_dir += "/"


	return args
	def signal_handler(signal, frame):
		inform( "Handling signal %s %s"%(signal,frame ))
#		monitor.stop()
		sys.exit(0)
					"$set" : { "status" : "quit" }
				}, multi=True,upsert=False,safe=True

		)



def cleanup():
	inform("~~~~~~~~EXITING - Cleaning up~~~~~~~~~~~")
	try:
		if monitor is not None:
			monitor.stop()
	except Exception,e:
		inform("Exception during stopping monitor %s " % e)
	finally:
		inform("BYEEE!")

atexit.register(cleanup)



if __name__ == "__main__":
	global monitor
	global DIR
	global flush_frequency

	monitor = None
	signals_to_handle = [signal.SIGINT,signal.SIGHUP,signal.SIGQUIT,signal.SIGABRT,signal.SIGALRM]
	args = parse_arguments()
	DIR = ""
	flush_frequency = args.flush