示例#1
0
 def _handle_local_arch(self, arch_path):
     ready_archs = []
     
     (arch_root_path, arch_name) = os.path.split(arch_path)
     
     hgs_files = auxiliary.seekZipForElement(arch_path, AppTypes.HGS)
     if len(hgs_files) > 0:
         extract_dir = os.path.join(arch_root_path, 'hgs_extract')
         auxiliary.extractZip(arch_path, extract_dir)
     
         if len(hgs_files) > 0:
             app_type = AppTypes.HGS
             
             for app_path in hgs_files:
                 app_root_dir = auxiliary.getRootDir(app_path)
                 
                 if app_root_dir != '':
                     hgs_root_dir = os.path.join(extract_dir, app_root_dir)
                     name = app_root_dir
                 else:
                     hgs_root_dir = extract_dir
                     name = arch_name
                 
                 hgs_arch_name = os.path.join(arch_root_path, str(uuid.uuid4()) + '.zip')
                 auxiliary.createZip(hgs_arch_name[:-4], hgs_root_dir)
                 
                 ready_archs.append({'arch_path': hgs_arch_name, 'app_type': app_type, 'app_name': name})
                 
     return ready_archs
示例#2
0
def main(config):
    ts_start = datetime.now()
    logger = auxiliary.getLogger()
        
    s3cfg = auxiliary.read_option(config, 'general', 's3cfg')
    
    app_input_s3_path_full = auxiliary.read_option(config, 'general', 'app_input_file')
    app_output_s3_path_full = auxiliary.read_option(config, 'general', 'app_output_file')
    licence = auxiliary.read_option(config, 'general', 'licence')
    grok = auxiliary.read_option(config, 'general', 'grok')
    hgs = auxiliary.read_option(config, 'general', 'hgs')
    
    current_dir = os.getcwd()
    app_input_name = os.path.basename(app_input_s3_path_full)
    app_input_path = os.path.join(current_dir, app_input_name)
    app_output_name = os.path.basename(app_output_s3_path_full)
    app_output_path = os.path.join(current_dir, app_output_name)
    
    logger.info('Starting %s script...\n' % (__name__))
    logger.info('Configuration options:')
    logger.info('Current directory: %s'% (current_dir))
    logger.info('Input archive location: %s'% (app_input_s3_path_full))
    logger.info('Output archive location: %s'% (app_output_s3_path_full))
    logger.info('GROK location: %s'% (grok))
    logger.info('HGS location: %s\n'% (hgs))
    
    logger.info('Downloading input archive')
    s3tools.s3cmdGET(s3cfg, app_input_s3_path_full, app_input_path, shellOutput=False)
    
    hgs_file = auxiliary.seekZipForElement(app_input_path, AppTypes.HGS)[0]
    working_dir = os.path.join(current_dir, os.path.split(hgs_file)[0])
    root_dir = os.path.join(current_dir, auxiliary.getRootDir(hgs_file))
    logger.info('Working directory: %s'% (working_dir))
    
    logger.info('Extracting input archive...')
    auxiliary.extractZip(app_input_path, current_dir)
    os.chdir(working_dir)

    cmd = '%s 1>&1 | tee -a %s' %(grok, 'grok.log')
    logger.info('Executing GROK: %s' % (cmd))
    auxiliary.execute(cmd)
    
    cmd = '%s 1>&1 | tee -a %s' %(hgs, 'hgs.log')
    logger.info('Executing HGS: %s' % (cmd))
    auxiliary.execute(cmd)
    
    os.chdir(current_dir)
    logger.info('Creating output archive...')
    auxiliary.createZip(app_output_path[:-4], root_dir)
    
    logger.info('Uploading output archive')
    
    #while not s3tools.s3cmdLS(s3cfg, app_output_s3_path_full, shellOutput=False):
    s3tools.s3cmdPUT(s3cfg, app_output_path, app_output_s3_path_full, shellOutput=False)
    
    logger.info('Done.')
    
    ts_end = datetime.now()
    ts_dif = ts_end - ts_start
    print 'GROK & HGS execution + claud I/O operations took: %s seconds' % ts_dif.total_seconds()
    print 'Executed on machine: %s' % messaging.getIPAdreess()