def main(): try: ## Rad the CFG file for the process and set params. cfg_file = os.path.join("/home/hadoop/", sys.argv[1] + ".ini") except IndexError: ## Catch Exception and print on screen. print "Pass valid cfg file parameter" sys.exit(1) ## Intialize the Logger object. log = Logger(logging.INFO, cfg_file) log.addFileHandler(logging.DEBUG) log.addMailHandler(log.m_config["cluster_aws_mailprfx"] +" ERROR: AWS process failed for the day",logging.ERROR) try: ## Set env variables for the process. conf = configuration(cfg_file,True) log.info(os.environ["job_prefix"] + " process started for the day..") ## Create Hive class object and create queries for execution objhive = Hive(log, os.environ['hive_path']) hive_options = '-f' loop_count = 0 ## Get the list of SQL and SQL files for execution log.info("Getting Hive Sql list...\n") sql_name=os.environ["sql_names"].split(",") sql_filename=os.environ["sql_files"].split(",") ## Loop through all the sql for the process. while loop_count < len(sql_filename): log.info('Hive '+ sql_name[loop_count]+' SQL Execution started.') objhive.buildHql(hive_options, os.environ["sql_dir"]+ '/' + sql_filename[loop_count]) log.info("SQL File:- " + objhive.hive_sql) objhive.execute() log.info('SQL Execution completed successfully...\n') loop_count=loop_count+1 log.info(os.environ["job_prefix"] + " process completed successfully for the day..") ## Cleanup and send the success mail for the process. log.cleanup() log.addMailHandler(log.m_config["cluster_aws_mailprfx"] +" SUCCESS: AWS process completed successfully for the day",logging.ERROR) log.sendlog() except Exception, e: print e log.info(os.environ["job_prefix"] + " process thrown exception "+ str(e) +"\n") log.info("##======= Please Find Proces Temp Log Below =======##\n") log.cleanup() log.addMailHandler(log.m_config["cluster_aws_mailprfx"]+" EXCEPTION: AWS process on cluster thrown exception "+ str(e) +"..",logging.ERROR) log.sendlog("error",50) sys.exit(1)
def main(): try: opts, args = getopt.getopt(sys.argv[1:], 'c:') total_param = len(sys.argv) if total_param != 3: print ("Usage: %s -c config_file" % sys.argv[0]) sys.exit(1) except getopt.GetoptError as e: print (str(e)) print ("Usage: %s -c config_file" % sys.argv[0]) sys.exit(1) for opt, args in opts: if opt == '-c': cfg_file = args print "Config file is " + cfg_file #initiate logger log = Logger(logging.INFO, cfg_file) prcstime = getTime('YYMMDD') +'_'+getTime('HHMMSS') ## Add file and Mal handlers to the logger module. log.addFileHandler(logging.INFO) log.addMailHandler(log.m_config['cluster_launch_mailprfx'] +" Error: While Launching AWS cluster for the day.." + prcstime,logging.ERROR) try: ##Generate Object of EMR class and pass the logger class object. emr = EMR(cfg_file, log) log.info("AWS EMR "+ emr.m_configfile["job_prefix"].upper() + " process started for the day") ## Check if Dependency Check is required for the cluster launch. if emr.m_configfile["dep_check"] == 'TRUE': dep_status = emr.checkDependency(emr.m_configfile["dep_process"]) ## Check Dependency status if dep_status == 0: log.info("Dependency process for "+ emr.m_configfile["job_prefix"] + " process completed successfully for the day.") else: log.info("Dependency Process has not completed for the day Exiting... after Wait") log.cleanup() log.addMailHandler(log.m_config['cluster_launch_mailprfx']+" Error:Dependency Check Failed while Launching cluster for the day."+prcstime,logging.ERROR) log.sendlog() sys.exit(1) else: log.info("Dependency Check is not required for the Process..") #build cluster launch command emr.buildEMRCommand() log.info("Generated the Cluster launch script for the process " + emr.m_configfile["job_prefix"]) emr.launchCluster() log.info("Launched the emr cluster successfully..") log.info("======================================") log.info("Cluster Id is ====>> " + emr.cluster_id[0] + '') log.info("======================================") #Print cluster launch command. log.cleanup() log.addMailHandler(log.m_config['cluster_launch_mailprfx']+" SUCCESS: EMR cluster launched sucessfully for the day.." + prcstime ,logging.ERROR) log.sendlog() except Exception as e: print (str(e)) log.info("Usage: %s -c config_file" % sys.argv[0]) log.cleanup() log.addMailHandler(log.m_config['cluster_launch_mailprfx']+" EXCEPTION: "+ str(e) +" While Launching AWS Cluster." + prcstime ,logging.ERROR) log.sendlog() sys.exit(1)
def main(): try: opts, args = getopt.getopt(sys.argv[1:], 'c:') total_param = len(sys.argv) if total_param != 3: print("Usage: %s -c config_file" % sys.argv[0]) sys.exit(1) except getopt.GetoptError as e: print(str(e)) print("Usage: %s -c config_file" % sys.argv[0]) sys.exit(1) for opt, args in opts: if opt == '-c': cfg_file = args print "Config file is " + cfg_file #initiate logger log = Logger(logging.INFO, cfg_file) prcstime = getTime('YYMMDD') + '_' + getTime('HHMMSS') ## Add file and Mal handlers to the logger module. log.addFileHandler(logging.INFO) log.addMailHandler( log.m_config['cluster_launch_mailprfx'] + " Error: While Launching AWS cluster for the day.." + prcstime, logging.ERROR) try: ##Generate Object of EMR class and pass the logger class object. emr = EMR(cfg_file, log) log.info("AWS EMR " + emr.m_configfile["job_prefix"].upper() + " process started for the day") ## Check if Dependency Check is required for the cluster launch. if emr.m_configfile["dep_check"] == 'TRUE': dep_status = emr.checkDependency(emr.m_configfile["dep_process"]) ## Check Dependency status if dep_status == 0: log.info("Dependency process for " + emr.m_configfile["job_prefix"] + " process completed successfully for the day.") else: log.info( "Dependency Process has not completed for the day Exiting... after Wait" ) log.cleanup() log.addMailHandler( log.m_config['cluster_launch_mailprfx'] + " Error:Dependency Check Failed while Launching cluster for the day." + prcstime, logging.ERROR) log.sendlog() sys.exit(1) else: log.info("Dependency Check is not required for the Process..") #build cluster launch command emr.buildEMRCommand() log.info("Generated the Cluster launch script for the process " + emr.m_configfile["job_prefix"]) emr.launchCluster() log.info("Launched the emr cluster successfully..") log.info("======================================") log.info("Cluster Id is ====>> " + emr.cluster_id[0] + '') log.info("======================================") #Print cluster launch command. log.cleanup() log.addMailHandler( log.m_config['cluster_launch_mailprfx'] + " SUCCESS: EMR cluster launched sucessfully for the day.." + prcstime, logging.ERROR) log.sendlog() except Exception as e: print(str(e)) log.info("Usage: %s -c config_file" % sys.argv[0]) log.cleanup() log.addMailHandler( log.m_config['cluster_launch_mailprfx'] + " EXCEPTION: " + str(e) + " While Launching AWS Cluster." + prcstime, logging.ERROR) log.sendlog() sys.exit(1)