def main():
try:
## Rad the CFG file for the process and set params.
cfg_file = os.path.join("/home/hadoop/", sys.argv[1] + ".ini")
except IndexError:
## Catch Exception and print on screen.
print "Pass valid cfg file parameter"
sys.exit(1)
## Intialize the Logger object.
log = Logger(logging.INFO, cfg_file)
log.addFileHandler(logging.DEBUG)
log.addMailHandler(log.m_config["cluster_aws_mailprfx"] +" ERROR: AWS process failed for the day",logging.ERROR)
try:
## Set env variables for the process.
conf = configuration(cfg_file,True)
log.info(os.environ["job_prefix"] + " process started for the day..")
## Create Hive class object and create queries for execution
objhive = Hive(log, os.environ['hive_path'])
hive_options = '-f'
loop_count = 0
## Get the list of SQL and SQL files for execution
log.info("Getting Hive Sql list...\n")
sql_name=os.environ["sql_names"].split(",")
sql_filename=os.environ["sql_files"].split(",")
## Loop through all the sql for the process.
while loop_count < len(sql_filename):
log.info('Hive '+ sql_name[loop_count]+' SQL Execution started.')
objhive.buildHql(hive_options, os.environ["sql_dir"]+ '/' + sql_filename[loop_count])
log.info("SQL File:- " + objhive.hive_sql)
objhive.execute()
log.info('SQL Execution completed successfully...\n')
loop_count=loop_count+1
log.info(os.environ["job_prefix"] + " process completed successfully for the day..")
## Cleanup and send the success mail for the process.
log.cleanup()
log.addMailHandler(log.m_config["cluster_aws_mailprfx"] +" SUCCESS: AWS process completed successfully for the day",logging.ERROR)
log.sendlog()
except Exception, e:
print e
log.info(os.environ["job_prefix"] + " process thrown exception "+ str(e) +"\n")
log.info("##======= Please Find Proces Temp Log Below =======##\n")
log.cleanup()
log.addMailHandler(log.m_config["cluster_aws_mailprfx"]+" EXCEPTION: AWS process on cluster thrown exception "+ str(e) +"..",logging.ERROR)
log.sendlog("error",50)
sys.exit(1)
def main():
try:
opts, args = getopt.getopt(sys.argv[1:], 'c:')
total_param = len(sys.argv)
if total_param != 3:
print ("Usage: %s -c config_file" % sys.argv[0])
sys.exit(1)
except getopt.GetoptError as e:
print (str(e))
print ("Usage: %s -c config_file" % sys.argv[0])
sys.exit(1)
for opt, args in opts:
if opt == '-c':
cfg_file = args
print "Config file is " + cfg_file
#initiate logger
log = Logger(logging.INFO, cfg_file)
prcstime = getTime('YYMMDD') +'_'+getTime('HHMMSS')
## Add file and Mal handlers to the logger module.
log.addFileHandler(logging.INFO)
log.addMailHandler(log.m_config['cluster_launch_mailprfx'] +" Error: While Launching AWS cluster for the day.." + prcstime,logging.ERROR)
try:
##Generate Object of EMR class and pass the logger class object.
emr = EMR(cfg_file, log)
log.info("AWS EMR "+ emr.m_configfile["job_prefix"].upper() + " process started for the day")
## Check if Dependency Check is required for the cluster launch.
if emr.m_configfile["dep_check"] == 'TRUE':
dep_status = emr.checkDependency(emr.m_configfile["dep_process"])
## Check Dependency status
if dep_status == 0:
log.info("Dependency process for "+ emr.m_configfile["job_prefix"] + " process completed successfully for the day.")
else:
log.info("Dependency Process has not completed for the day Exiting... after Wait")
log.cleanup()
log.addMailHandler(log.m_config['cluster_launch_mailprfx']+" Error:Dependency Check Failed while Launching cluster for the day."+prcstime,logging.ERROR)
log.sendlog()
sys.exit(1)
else:
log.info("Dependency Check is not required for the Process..")
#build cluster launch command
emr.buildEMRCommand()
log.info("Generated the Cluster launch script for the process " + emr.m_configfile["job_prefix"])
emr.launchCluster()
log.info("Launched the emr cluster successfully..")
log.info("======================================")
log.info("Cluster Id is ====>> " + emr.cluster_id[0] + '')
log.info("======================================")
#Print cluster launch command.
log.cleanup()
log.addMailHandler(log.m_config['cluster_launch_mailprfx']+" SUCCESS: EMR cluster launched sucessfully for the day.." + prcstime ,logging.ERROR)
log.sendlog()
except Exception as e:
print (str(e))
log.info("Usage: %s -c config_file" % sys.argv[0])
log.cleanup()
log.addMailHandler(log.m_config['cluster_launch_mailprfx']+" EXCEPTION: "+ str(e) +" While Launching AWS Cluster." + prcstime ,logging.ERROR)
log.sendlog()
sys.exit(1)
def main():
try:
opts, args = getopt.getopt(sys.argv[1:], 'c:')
total_param = len(sys.argv)
if total_param != 3:
print("Usage: %s -c config_file" % sys.argv[0])
sys.exit(1)
except getopt.GetoptError as e:
print(str(e))
print("Usage: %s -c config_file" % sys.argv[0])
sys.exit(1)
for opt, args in opts:
if opt == '-c':
cfg_file = args
print "Config file is " + cfg_file
#initiate logger
log = Logger(logging.INFO, cfg_file)
prcstime = getTime('YYMMDD') + '_' + getTime('HHMMSS')
## Add file and Mal handlers to the logger module.
log.addFileHandler(logging.INFO)
log.addMailHandler(
log.m_config['cluster_launch_mailprfx'] +
" Error: While Launching AWS cluster for the day.." + prcstime,
logging.ERROR)
try:
##Generate Object of EMR class and pass the logger class object.
emr = EMR(cfg_file, log)
log.info("AWS EMR " + emr.m_configfile["job_prefix"].upper() +
" process started for the day")
## Check if Dependency Check is required for the cluster launch.
if emr.m_configfile["dep_check"] == 'TRUE':
dep_status = emr.checkDependency(emr.m_configfile["dep_process"])
## Check Dependency status
if dep_status == 0:
log.info("Dependency process for " +
emr.m_configfile["job_prefix"] +
" process completed successfully for the day.")
else:
log.info(
"Dependency Process has not completed for the day Exiting... after Wait"
)
log.cleanup()
log.addMailHandler(
log.m_config['cluster_launch_mailprfx'] +
" Error:Dependency Check Failed while Launching cluster for the day."
+ prcstime, logging.ERROR)
log.sendlog()
sys.exit(1)
else:
log.info("Dependency Check is not required for the Process..")
#build cluster launch command
emr.buildEMRCommand()
log.info("Generated the Cluster launch script for the process " +
emr.m_configfile["job_prefix"])
emr.launchCluster()
log.info("Launched the emr cluster successfully..")
log.info("======================================")
log.info("Cluster Id is ====>> " + emr.cluster_id[0] + '')
log.info("======================================")
#Print cluster launch command.
log.cleanup()
log.addMailHandler(
log.m_config['cluster_launch_mailprfx'] +
" SUCCESS: EMR cluster launched sucessfully for the day.." +
prcstime, logging.ERROR)
log.sendlog()
except Exception as e:
print(str(e))
log.info("Usage: %s -c config_file" % sys.argv[0])
log.cleanup()
log.addMailHandler(
log.m_config['cluster_launch_mailprfx'] + " EXCEPTION: " + str(e) +
" While Launching AWS Cluster." + prcstime, logging.ERROR)
log.sendlog()
sys.exit(1)
