def s3cmdGET(s3cfg_path, s3_path, local_path, shellOutput=True): if s3cmdLS(s3cfg_path, s3_path, shellOutput): retries = 3 sleep = 5 while retries > 0: retries -= 1 cmd = "s3cmd -c %s get --force %s %s" %(s3cfg_path, s3_path, local_path) output = auxiliary.execute(cmd, shellOutput=shellOutput) if output.strip()[-4:] == 'done': return True time.sleep(sleep) return False
def s3cmdLS(s3cfg_path, s3_path, shellOutput=True): cmd = "s3cmd -c %s ls %s" %(s3cfg_path, s3_path) if auxiliary.execute(cmd, shellOutput=shellOutput): return True else: return False
def s3cmdDEL(s3cfg_path, s3_path, shellOutput=True): cmd = "s3cmd -c %s del %s" %(s3cfg_path, s3_path) auxiliary.execute(cmd, shellOutput=shellOutput)
def s3cmdPUT(s3cfg_path, local_path, s3_path, shellOutput=True): cmd = "s3cmd -c %s put %s %s" %(s3cfg_path, local_path, s3_path) auxiliary.execute(cmd, shellOutput=shellOutput)
def main(config): ts_start = datetime.now() logger = auxiliary.getLogger() s3cfg = auxiliary.read_option(config, 'general', 's3cfg') app_input_s3_path_full = auxiliary.read_option(config, 'general', 'app_input_file') app_output_s3_path_full = auxiliary.read_option(config, 'general', 'app_output_file') licence = auxiliary.read_option(config, 'general', 'licence') grok = auxiliary.read_option(config, 'general', 'grok') hgs = auxiliary.read_option(config, 'general', 'hgs') current_dir = os.getcwd() app_input_name = os.path.basename(app_input_s3_path_full) app_input_path = os.path.join(current_dir, app_input_name) app_output_name = os.path.basename(app_output_s3_path_full) app_output_path = os.path.join(current_dir, app_output_name) logger.info('Starting %s script...\n' % (__name__)) logger.info('Configuration options:') logger.info('Current directory: %s'% (current_dir)) logger.info('Input archive location: %s'% (app_input_s3_path_full)) logger.info('Output archive location: %s'% (app_output_s3_path_full)) logger.info('GROK location: %s'% (grok)) logger.info('HGS location: %s\n'% (hgs)) logger.info('Downloading input archive') s3tools.s3cmdGET(s3cfg, app_input_s3_path_full, app_input_path, shellOutput=False) hgs_file = auxiliary.seekZipForElement(app_input_path, AppTypes.HGS)[0] working_dir = os.path.join(current_dir, os.path.split(hgs_file)[0]) root_dir = os.path.join(current_dir, auxiliary.getRootDir(hgs_file)) logger.info('Working directory: %s'% (working_dir)) logger.info('Extracting input archive...') auxiliary.extractZip(app_input_path, current_dir) os.chdir(working_dir) cmd = '%s 1>&1 | tee -a %s' %(grok, 'grok.log') logger.info('Executing GROK: %s' % (cmd)) auxiliary.execute(cmd) cmd = '%s 1>&1 | tee -a %s' %(hgs, 'hgs.log') logger.info('Executing HGS: %s' % (cmd)) auxiliary.execute(cmd) os.chdir(current_dir) logger.info('Creating output archive...') auxiliary.createZip(app_output_path[:-4], root_dir) logger.info('Uploading output archive') #while not s3tools.s3cmdLS(s3cfg, app_output_s3_path_full, shellOutput=False): s3tools.s3cmdPUT(s3cfg, app_output_path, app_output_s3_path_full, shellOutput=False) logger.info('Done.') ts_end = datetime.now() ts_dif = ts_end - ts_start print 'GROK & HGS execution + claud I/O operations took: %s seconds' % ts_dif.total_seconds() print 'Executed on machine: %s' % messaging.getIPAdreess()