def compression_rec_file(logger, ts, target_info_dict): """ Compression record file :param logger: Logger :param ts: System time :param target_info_dict: Target information dictionary """ global COMPRESSION_CNT # Compression record file logger.info("1. Compression record file") target_dir_path = target_info_dict.get('directory_path') if target_dir_path[-1] == "/": target_dir_path = target_dir_path[:-1] compression_file_date = int(target_info_dict.get('compression_file_date')) tmp_date = (datetime.fromtimestamp(ts) - timedelta(days=compression_file_date)).strftime('%Y%m%d') target_dir_path += "/{0}".format(tmp_date) logger.info("Target directory path : {0}".format(target_dir_path)) if not os.path.exists(target_dir_path): logger.info("Target directory not existed") return enc = target_info_dict.get('enc') w_ob = os.walk(target_dir_path) for dir_path, sub_dirs, files in w_ob: for file_name in files: try: file_path = os.path.join(dir_path, file_name) if not os.path.exists(file_path): logger.debug("Not existed -> {0}".format(file_path)) continue ts = time.time() dt = datetime.fromtimestamp(ts).strftime('%Y%m%d%H%M%S') if not file_name.startswith('comp_'): logger.info('Compression file [{0}]'.format(file_path)) if file_name.endswith('.enc'): decrypt_file([file_path]) tmp_file_name = file_name[:-4].replace('.', '_') else: tmp_file_name = file_name.replace('.', '_') tmp_file_path = file_path if not file_path.endswith('enc') else file_path[:-4] s16_file_path = "{0}.s16".format(os.path.join(dir_path, tmp_file_name)) wav_file_path = "{0}/comp_{1}.wav".format(dir_path, tmp_file_name) os.rename(tmp_file_path, s16_file_path) sox_cmd = 'sox -r 8000 -c 1 {0} -r 8000 -c 1 -e gsm {1}'.format(s16_file_path, wav_file_path) sub_process(logger, sox_cmd) COMPRESSION_CNT += 1 del_garbage(logger, s16_file_path) if enc: rename_file_path = "{0}/encrypting_comp_{1}.wav".format(dir_path, tmp_file_name) logger.debug("Rename [ {0} -> {1} ]".format(wav_file_path, rename_file_path)) os.rename(wav_file_path, rename_file_path) logger.debug("Encrypt [ {0} ]".format(rename_file_path)) encrypt_file([rename_file_path]) logger.info('Compression file [{0}], The time required = {1}'.format( file_path, elapsed_time(dt))) except Exception: exc_info = traceback.format_exc() logger.error(exc_info) continue
def upload_data_to_db(logger, oracle, checked_dat_data_dict): """ Upload data to database :param logger: Logger :param oracle: Oracle :param checked_dat_data_dict: Checked dat data dictionary """ global UPLOAD_CNT global ERROR_CNT logger.debug("dat data upload to DB") for file_path, dat_data_dict_list in checked_dat_data_dict.items(): task_date = os.path.basename(file_path)[:8] for dat_data_dict in dat_data_dict_list: try: logger.debug("Target dat data -> TASK_DATE : {0}, FILE_NAME : {1}".format( task_date, dat_data_dict['FILE_NAME'])) if oracle.check_marketing_rec_meta_tb(task_date, dat_data_dict['FILE_NAME']): oracle.update_data_to_rec_meta( task_date=task_date, file_name=dat_data_dict['FILE_NAME'], cu_id=dat_data_dict['CU_ID'], cu_name=dat_data_dict['CU_NAME'], associator_cd=dat_data_dict['ASSOCIATOR_CD'], associator_name=dat_data_dict['ASSOCIATOR_NAME'], creator_id='MA_REC' ) else: oracle.insert_data_to_rec_meta( task_date=task_date, file_name=dat_data_dict['FILE_NAME'], cu_id=dat_data_dict['CU_ID'], cu_name=dat_data_dict['CU_NAME'], associator_cd=dat_data_dict['ASSOCIATOR_CD'], associator_name=dat_data_dict['ASSOCIATOR_NAME'], creator_id='MA_REC' ) oracle.conn.commit() UPLOAD_CNT += 1 except Exception: exc_info = traceback.format_exc() logger.error('upsert error -> {0}'.format('|'.join(dat_data_dict.values()))) logger.error(exc_info) oracle.conn.rollback() ERROR_CNT += 1 continue call_date = os.path.basename(file_path)[:8] dat_output_dir_path = '{0}/{1}/{2}/{3}'.format( UPLOAD_CONFIG['dat_output_path'], call_date[:4], call_date[4:6], call_date[6:8]) dat_output_file_path = os.path.join(dat_output_dir_path, os.path.basename(file_path)) if not os.path.exists(dat_output_dir_path): os.makedirs(dat_output_dir_path) if os.path.exists(dat_output_file_path): del_garbage(logger, dat_output_file_path) logger.debug("Move dat file to output directory.") shutil.move(file_path, dat_output_dir_path) encrypt_file([dat_output_file_path])
def rollback_rec_file(logger, input_dir_path): """ Rollback compression record file :param logger: Logger :param input_dir_path: Input directory path """ global ERR_CNT global RECORD_CNT # Rollback compression record file logger.info("Rollback compression record file") if input_dir_path[-1] == "/": input_dir_path = input_dir_path[:-1] logger.info("Target directory path : {0}".format(input_dir_path)) temp_input_dir_path = "{0}/temp_{1}".format(CONFIG['temp_rollback_dir_path'], os.path.basename(input_dir_path)) logger.info("Copy record file directory -> {0}".format(temp_input_dir_path)) if os.path.exists(temp_input_dir_path): del_garbage(logger, temp_input_dir_path) copy_tree(input_dir_path, temp_input_dir_path) w_ob = os.walk(temp_input_dir_path) for dir_path, sub_dirs, files in w_ob: for file_name in files: target_file = os.path.join(dir_path, file_name) logger.info("Target file -> {0}".format(target_file)) try: # Decrypt file if target_file.endswith('.enc'): logger.debug("Decrypt {0}".format(target_file)) decrypt_file([target_file]) decrypted_rec_file = target_file[:-4] if target_file.endswith('.enc') else target_file temp_rec_file_name = os.path.basename(decrypted_rec_file) temp_rec_file_name = temp_rec_file_name[5:] if temp_rec_file_name.startswith('comp_') else temp_rec_file_name temp_rec_file_name = temp_rec_file_name[:-4] if temp_rec_file_name.endswith('.wav') else temp_rec_file_name modified_rec_file_name = temp_rec_file_name.replace("_", ".").replace("__", ".") output_rec_file = "{0}/{1}".format(os.path.dirname(target_file), modified_rec_file_name) sox_cmd = "sox -t wav {0} -r 8000 -b 16 -t raw {1}".format(decrypted_rec_file, output_rec_file) sub_process(logger, sox_cmd) logger.debug("Encrypt {0}".format(output_rec_file)) encrypt_file([output_rec_file]) RECORD_CNT += 1 del_garbage(logger, decrypted_rec_file) if not os.path.exists(CONFIG['decompression_dir_path']): os.makedirs(CONFIG['decompression_dir_path']) if os.path.exists("{0}/{1}.enc".format(CONFIG['decompression_dir_path'], modified_rec_file_name)): del_garbage(logger, "{0}/{1}.enc".format(CONFIG['decompression_dir_path'], modified_rec_file_name)) if os.path.exists("{0}.enc".format(output_rec_file)): shutil.move("{0}.enc".format(output_rec_file), CONFIG['decompression_dir_path']) except Exception: ERR_CNT += 1 exc_info = traceback.format_exc() logger.error("Can't rollback record file -> {0}".format(target_file)) logger.error(exc_info) continue del_garbage(logger, temp_input_dir_path)
def error_process(logger, file_path): """ Error file process :param logger: Logger :param file_path: Json file path """ global ERROR_CNT ERROR_CNT += 1 try: now_dt = datetime.now() logger.error("Error process. [Json = {0}]".format(file_path)) error_dir_path = '{0}/error_data/{1}/{2}/{3}'.format( os.path.dirname(file_path), now_dt.strftime("%Y"), now_dt.strftime("%m"), now_dt.strftime("%d")) if not os.path.exists(error_dir_path): os.makedirs(error_dir_path) error_json_file_path = '{0}/{1}'.format(error_dir_path, os.path.basename(file_path)) enc_error_json_file_path = '{0}.enc'.format(error_json_file_path) if os.path.exists(enc_error_json_file_path): del_garbage(logger, enc_error_json_file_path) logger.error('Error json file move {0} -> {1}'.format( file_path, error_dir_path)) shutil.move(file_path, error_dir_path) logger.error('Error json file encrypt {0} -> {1}'.format( error_json_file_path, enc_error_json_file_path)) encrypt_file([error_json_file_path]) file_name = os.path.splitext(os.path.basename(file_path))[0] if file_name: record_file_list = glob.glob('{0}/{1}.*'.format( os.path.dirname(file_path), file_name)) record_file_list += glob.glob('{0}/{1}_*'.format( os.path.dirname(file_path), file_name)) if len(record_file_list) > 0: for record_file_path in record_file_list: error_file_path = "{0}/{1}".format( error_dir_path, os.path.basename(record_file_path)) enc_error_file_path = '{0}.enc'.format(error_file_path) if os.path.exists(enc_error_file_path): del_garbage(logger, enc_error_file_path) shutil.move(record_file_path, error_dir_path) logger.error('Error record file move {0} -> {1}'.format( record_file_path, error_dir_path)) encrypt_file([error_file_path]) logger.error('Error record file encrypt {0} -> {1}'.format( error_file_path, enc_error_file_path)) except Exception: exc_info = traceback.format_exc() logger.critical("Critical error {0}".format(exc_info))
def processing(dir_list): """ Processing :param dir_list: List of directory """ # Add logging logger_args = { 'base_path': CONFIG['log_dir_path'], 'log_file_name': CONFIG['encrypt_log_file_name'], 'log_level': CONFIG['log_level'] } logger = set_logger_period_of_time(logger_args) ts = time.time() st = datetime.fromtimestamp(ts).strftime('%Y-%m-%d-%H:%M.%S') encrypt_cnt = 0 for dir_path in dir_list: if not os.path.exists(dir_path): logger.debug('Directory is not exists -> {0}'.format(dir_path)) continue for file_name in os.listdir(dir_path): try: if file_name.endswith(".enc") or file_name.endswith( ".tmp") or file_name.endswith("filepart"): continue file_path = os.path.join(dir_path, file_name) rename_file_path = os.path.join( dir_path, "encrypting_{0}".format(file_name)) encrypt_cnt += 1 logger.info("Rename [ {0} -> {1} ]".format( file_path, rename_file_path)) os.rename(file_path, rename_file_path) logger.info("Encrypt [ {0} ]".format(file_name)) openssl.encrypt_file([rename_file_path]) except Exception: exc_info = traceback.format_exc() logger.error( "Can't Rename or Encrypt record file. [ {0} ]".format( file_name)) logger.error(exc_info) continue dt = datetime.fromtimestamp(ts).strftime('%Y%m%d%H%M%S') logger.debug( "END.. Start time = {0}, The time required = {1}, Encrypt target file list count = {2}" .format(st, elapsed_time(dt), encrypt_cnt)) for handler in logger.handlers: handler.close() logger.removeHandler(handler)
def compression_and_delete_rec_file(logger, target_info_dict): """ Compression and delete file :param logger: Logger :param target_info_dict: Target information dictionary """ global DELETE_CNT global COMPRESSION_CNT # Compression and delete record file logger.info("1. Compression and delete record file") target_dir_path = target_info_dict.get('directory_path') logger.info("Target directory path : {0}".format(target_dir_path)) if target_dir_path[-1] == "/": target_dir_path = target_dir_path[:-1] compression_file_date = int(target_info_dict.get('compression_file_date')) delete_file_date = int(target_info_dict.get('delete_file_date')) enc = target_info_dict.get('enc') w_ob = os.walk(target_dir_path) date_time_now = datetime.now() logger.info('Date time now : {0}'.format(date_time_now)) for dir_path, sub_dirs, files in w_ob: for file_name in files: try: file_path = os.path.join(dir_path, file_name) if not os.path.exists(file_path): logger.debug("Not existed -> {0}".format(file_path)) continue m_time = os.path.getmtime(file_path) date_m_time = datetime.fromtimestamp(m_time) logger.info("Modified file date : {0}".format(date_m_time)) check_com_date = date_m_time + timedelta(days=compression_file_date) check_del_date = date_m_time + timedelta(days=delete_file_date - compression_file_date) ts = time.time() dt = datetime.fromtimestamp(ts).strftime('%Y%m%d%H%M%S') if date_time_now > check_com_date and not file_name.startswith('comp_'): logger.info('Compression file [{0}]'.format(file_path)) logger.info('Compression target date : {0}'.format(check_com_date)) if file_name.endswith('.enc'): decrypt_file([file_path]) tmp_file_name = file_name[:-4].replace('.', '_') else: tmp_file_name = file_name.replace('.', '_') tmp_file_path = file_path if not file_path.endswith('enc') else file_path[:-4] s16_file_path = "{0}.s16".format(os.path.join(dir_path, tmp_file_name)) wav_file_path = "{0}/comp_{1}.wav".format(dir_path, tmp_file_name) os.rename(tmp_file_path, s16_file_path) sox_cmd = 'sox -r 8000 -c 1 {0} -r 8000 -c 1 -e gsm {1}'.format(s16_file_path, wav_file_path) sub_process(logger, sox_cmd) COMPRESSION_CNT += 1 del_garbage(logger, s16_file_path) if enc: rename_file_path = "{0}/encrypting_comp_{1}.wav".format(dir_path, tmp_file_name) logger.debug("Rename [ {0} -> {1} ]".format(wav_file_path, rename_file_path)) os.rename(wav_file_path, rename_file_path) logger.debug("Encrypt [ {0} ]".format(rename_file_path)) encrypt_file([rename_file_path]) logger.info('Compression file [{0}], The time required = {1}'.format( file_path, elapsed_time(dt))) if date_time_now > check_del_date and file_name.startswith('comp_'): logger.info('Delete target date : {0}'.format(check_del_date)) logger.info('Delete file [{0}]'.format(file_path)) del_garbage(logger, file_path) DELETE_CNT += 1 except Exception: exc_info = traceback.format_exc() logger.error(exc_info) continue # Delete empty directory dir_name_list = list() w_ob = os.walk(target_dir_path) for dir_path, sub_dirs, files in w_ob: dir_name_list = sub_dirs break for dir_name in dir_name_list: if dir_name == 'incident_file': continue dir_path = "{0}/{1}".format(target_dir_path, dir_name) file_path_list = glob.glob("{0}/*".format(dir_path)) if len(file_path_list) == 0: shutil.rmtree(dir_path)
def processing(dir_list): """ Processing :param dir_list: List of directory """ # Add logging logger_args = { 'base_path': CONFIG['log_dir_path'], 'log_file_name': CONFIG['encrypt_log_file_name'], 'log_level': CONFIG['log_level'] } logger = set_logger_period_of_time(logger_args) ts = time.time() st = datetime.fromtimestamp(ts).strftime('%Y-%m-%d-%H:%M.%S') encrypt_cnt = 0 for dir_path in dir_list: if not os.path.exists(dir_path): logger.debug('Directory is not exists -> {0}'.format(dir_path)) continue w_ob = os.walk(dir_path) for dir_path2, sub_dirs, files in w_ob: for file_name in files: try: dir_name = os.path.basename(dir_path) encrypt_dir_path = os.path.join(CONFIG['encrypt_rec_dir_path'], dir_name) encrypt_file_path = "{0}/{1}".format(encrypt_dir_path, file_name) file_path = os.path.join(dir_path2, file_name) if file_name.endswith(".enc"): if not os.path.exists(encrypt_dir_path): os.makedirs(encrypt_dir_path) if os.path.exists(encrypt_file_path): os.remove(encrypt_file_path) logger.debug("Move [ {0} -> {1}.tmp ]".format(file_path, encrypt_file_path)) shutil.move(file_path, '{0}.tmp'.format(encrypt_file_path)) logger.debug("Rename [ {0}.tmp -> {0} ]".format(encrypt_file_path)) os.rename('{0}.tmp'.format(encrypt_file_path), encrypt_file_path) continue if file_name.endswith(".tmp") or file_name.endswith("filepart"): continue rename_file_path = os.path.join(dir_path2, "encrypting_{0}".format(file_name)) encrypt_cnt += 1 logger.debug("Rename [ {0} -> {1} ]".format(file_path, rename_file_path)) os.rename(file_path, rename_file_path) logger.debug("Encrypt [ {0} ]".format(file_name)) openssl.encrypt_file([rename_file_path]) except Exception: exc_info = traceback.format_exc() logger.error("Can't Rename or Encrypt record file. [ {0} ]".format(file_name)) logger.error(exc_info) continue try: encrypt_file_path += '.enc' enc_file_path = os.path.join(dir_path2, "{0}.enc".format(file_name)) if not os.path.exists(encrypt_dir_path): os.makedirs(encrypt_dir_path) if os.path.exists(encrypt_file_path): os.remove(encrypt_file_path) logger.debug("Move [ {0} -> {1}.tmp ]".format(enc_file_path, encrypt_file_path)) shutil.move(rename_file_path,'{0}.tmp'.format(encrypt_file_path)) logger.debug("Rename [ {0}.tmp -> {0} ]".format(encrypt_file_path)) os.rename('{0}.tmp'.format(encrypt_file_path), encrypt_file_path) except Exception: exc_info = traceback.format_exc() logger.error("Can't move encrypt file. [ {0} ]".format(file_name)) logger.error(exc_info) continue dt = datetime.fromtimestamp(ts).strftime('%Y%m%d%H%M%S') logger.info("END.. Start time = {0}, The time required = {1}, Encrypt target file list count = {2}".format( st, elapsed_time(dt), encrypt_cnt)) for handler in logger.handlers: handler.close() logger.removeHandler(handler)
def upload_data_to_db(logger, oracle, checked_json_data_dict): """ Upload data to database :param logger: Logger :param oracle: Oracle :param checked_json_data_dict: Checked json data dictionary """ global UPLOAD_CNT logger.debug("Json data upload to DB") for file_path, json_data in checked_json_data_dict.items(): try: logger.debug("Target json file = {0}".format(file_path)) project_code = 'PC0001' if json_data.get( 'project_cd') == 'CS' else 'PC0002' if json_data.get('call_type') == '1': call_type = 'CT0001' elif json_data.get('call_type') == '2': call_type = 'CT0002' else: call_type = 'CT0000' hour = json_data.get('duration')[:2] minute = json_data.get('duration')[2:4] sec = json_data.get('duration')[4:6] duration = int(hour) * 3600 + int(minute) * 60 + int(sec) record_key = '' if json_data.get( 'rec_id') == 'None' else json_data.get('rec_id') # CS BUSINESS_DCD if project_code == 'PC0001': user_id = json_data.get('r_user_id')[:7] user_name = json_data.get('r_user_name') if user_id == 'None': business_dcd = '' else: result = oracle.select_cs_business_dcd(user_id, user_name) if result: business_dcd = result[0] else: business_dcd = '' # TM BUSINESS_DCD else: user_id = json_data.get('r_user_id')[:7] user_name = json_data.get('r_user_name') if user_id == 'None': business_dcd = '' else: result = oracle.select_tm_business_dcd(user_id, user_name) if result: business_dcd = result[0] else: business_dcd = '' if business_dcd: meta_business_dcd = oracle.select_business_dcd_from_cm_cd_detail_tb( business_dcd) if meta_business_dcd: business_dcd = meta_business_dcd[0] #oracle.delete_cm_call_meta_tb(json_data.get('r_file_name')) oracle.insert_data_to_cm_call_meta_tb( project_code=project_code, file_name=json_data.get('r_file_name'), call_type_code=call_type, record_key=record_key, start_time=json_data.get('start_time'), end_time=json_data.get('end_time'), duration=duration, cti_call_id=json_data.get('cti_call_id'), stt_cmdtm='', stt_server_id=str(socket.gethostname()), ruser_id='' if json_data.get('r_user_id') == 'None' else json_data.get('r_user_id'), ruser_name='' if json_data.get('r_user_name') == 'None' else json_data.get('r_user_name'), ruser_number=json_data.get('r_user_number'), business_dcd=business_dcd, cu_id='' if json_data.get('cust_id') == 'None' else json_data.get('cust_id'), cu_name='' if json_data.get('cust_name') == 'None' else json_data.get('cust_name'), cu_number='' if json_data.get('cust_number') == 'None' else json_data.get('cust_number'), in_call_number=json_data.get('ani'), biz_cd=json_data.get('biz'), chn_tp=json_data.get('chn_tp'), file_sprt=json_data.get('sprt'), rec_ext=json_data.get('rec_ext'), creator_id='CS_REC', updator_id='CS_REC', ) json_output_dir_path = '{0}/{1}/{2}'.format( CONFIG['json_output_path'], json_data.get('start_time')[:10], json_data.get('biz')) json_output_file_path = os.path.join(json_output_dir_path, os.path.basename(file_path)) if not os.path.exists(json_output_dir_path): os.makedirs(json_output_dir_path) if os.path.exists(json_output_file_path): del_garbage(logger, json_output_file_path) logger.debug("Move json file to output directory.") shutil.move(file_path, json_output_dir_path) rec_dir_path = os.path.dirname(file_path) rec_move_dir_path = CONFIG['rec_move_dir_path'] chn_tp = json_data.get('chn_tp') sprt = json_data.get('sprt') file_name = json_data.get('r_file_name') extension = json_data.get('rec_ext') target_file_list = list() if chn_tp == 'S' and sprt == 'Y': target_file_list.append("{0}/{1}_rx.{2}".format( rec_dir_path, file_name, extension)) target_file_list.append("{0}/{1}_tx.{2}".format( rec_dir_path, file_name, extension)) elif (chn_tp == 'S' and sprt == 'N') or chn_tp == 'M': target_file_list.append("{0}/{1}.{2}".format( rec_dir_path, file_name, extension)) for target_file_path in target_file_list: rec_move_file_path = os.path.join( rec_move_dir_path, os.path.basename(target_file_path)) if not os.path.exists(rec_move_dir_path): os.makedirs(rec_move_dir_path) if os.path.exists(rec_move_file_path): del_garbage(logger, rec_move_file_path) logger.info( "Move rec file to collector directory. {0} -> {1}".format( target_file_path, rec_move_dir_path)) shutil.move(target_file_path, '{0}.tmp'.format(rec_move_file_path)) shutil.move('{0}.tmp'.format(rec_move_file_path), rec_move_file_path) encrypt_file([json_output_file_path]) oracle.conn.commit() UPLOAD_CNT += 1 except Exception: exc_info = traceback.format_exc() logger.error(exc_info) oracle.conn.rollback() error_process(logger, file_path) continue
def upload_data_to_db(logger, oracle, checked_json_data_dict): """ Upload data to database :param logger: Logger :param oracle: Oracle :param checked_json_data_dict: Checked json data dictionary """ global UPLOAD_CNT logger.debug("Json data upload to DB") for file_path, json_data in checked_json_data_dict.items(): try: logger.debug("Target json file = {0}".format(file_path)) if oracle.check_tb_cs_stt_rcdg_info(json_data.get('rec_id'), json_data.get('r_file_name')): oracle.update_data_to_rcdg_info( rec_id=json_data.get('rec_id'), rfile_name=json_data.get('r_file_name'), cs_stta_prgst_cd='00', stt_cmdtm='', stt_server_id=str(socket.gethostname()), project_cd=json_data.get('project_cd'), cti_call_id=json_data.get('cti_call_id'), call_type=json_data.get('call_type'), call_start_time=json_data.get('start_time'), call_end_time=json_data.get('end_time'), call_duration=json_data.get('duration'), ruser_id=json_data.get('r_user_id'), ruser_name=json_data.get('r_user_name'), ruser_number=json_data.get('r_user_number'), cu_id=json_data.get('cust_id'), cu_name=json_data.get('cust_name'), cu_number=json_data.get('cust_number'), in_call_number=json_data.get('ani'), biz_cd=json_data.get('biz'), chn_tp=json_data.get('chn_tp'), file_sprt=json_data.get('sprt'), rec_ext=json_data.get('rec_ext'), lst_chgp_cd='CS_REC', lst_chg_pgm_id='CS_REC' ) else: oracle.insert_data_to_rcdg_info( rec_id=json_data.get('rec_id'), rfile_name=json_data.get('r_file_name'), cs_stta_prgst_cd='00', stt_cmdtm='', stt_server_id=str(socket.gethostname()), project_cd=json_data.get('project_cd'), cti_call_id=json_data.get('cti_call_id'), call_type=json_data.get('call_type'), call_start_time=json_data.get('start_time'), call_end_time=json_data.get('end_time'), call_duration=json_data.get('duration'), ruser_id=json_data.get('r_user_id'), ruser_name=json_data.get('r_user_name'), ruser_number=json_data.get('r_user_number'), cu_id=json_data.get('cust_id'), cu_name=json_data.get('cust_name'), cu_number=json_data.get('cust_number'), in_call_number=json_data.get('ani'), biz_cd=json_data.get('biz'), chn_tp=json_data.get('chn_tp'), file_sprt=json_data.get('sprt'), rec_ext=json_data.get('rec_ext'), regp_cd='CS_REC', rgst_pgm_id='CS_REC', lst_chgp_cd='CS_REC', lst_chg_pgm_id='CS_REC' ) json_output_dir_path = '{0}/{1}/{2}'.format( CONFIG['json_output_path'], json_data.get('start_time')[:10], json_data.get('biz')) json_output_file_path = os.path.join(json_output_dir_path, os.path.basename(file_path)) if not os.path.exists(json_output_dir_path): os.makedirs(json_output_dir_path) if os.path.exists(json_output_file_path): del_garbage(logger, json_output_file_path) logger.debug("Move json file to output directory.") shutil.move(file_path, json_output_dir_path) encrypt_file([json_output_file_path]) oracle.conn.commit() UPLOAD_CNT += 1 except Exception: exc_info = traceback.format_exc() logger.error(exc_info) oracle.conn.rollback() error_process(logger, file_path) continue
def upload_data_to_db(logger, mysql, checked_json_data_dict): """ Upload data to database :param logger: Logger :param mysql: MySQL :param checked_json_data_dict: Checked json data dictionary """ global UPLOAD_CNT logger.debug("Json data upload to DB") for file_path, json_data in checked_json_data_dict.items(): try: logger.debug("Target json file = {0}".format(file_path)) if mysql.check_stt_rcdg_info(json_data.get('recordkey'), json_data.get('r_file_name')): mysql.update_data_to_stt_rcdg_info( con_id=json_data.get('con_id'), rfile_name=json_data.get('r_file_name'), channel=json_data.get('channel'), project_cd=json_data.get('project_cd'), stt_prgst_cd='00', stt_server_id=str(socket.gethostname()), call_type=json_data.get('call_type'), call_start_time=json_data.get('start_time'), call_end_time=json_data.get('end_time'), call_duration=json_data.get('duration'), ruser_id=json_data.get('r_user_id'), ruser_name=json_data.get('r_user_name'), ruser_number=json_data.get('r_user_number'), cu_number=json_data.get('cust_number'), biz_cd=json_data.get('biz'), chn_tp=json_data.get('chn_tp'), file_sprt=json_data.get('sprt'), rec_ext=json_data.get('rec_ext'), date=json_data.get('date'), recordkey=json_data.get('recordkey'), center=json_data.get('center') ) else: mysql.insert_data_to_stt_rcdg_info( con_id=json_data.get('con_id'), rfile_name=json_data.get('r_file_name'), channel=json_data.get('channel'), project_cd=json_data.get('project_cd'), stt_prgst_cd='00', stt_server_id=str(socket.gethostname()), call_type=json_data.get('call_type'), call_start_time=json_data.get('start_time'), call_end_time=json_data.get('end_time'), call_duration=json_data.get('duration'), ruser_id=json_data.get('r_user_id'), ruser_name=json_data.get('r_user_name'), ruser_number=json_data.get('r_user_number'), cu_number=json_data.get('cust_number'), biz_cd=json_data.get('biz'), chn_tp=json_data.get('chn_tp'), file_sprt=json_data.get('sprt'), rec_ext=json_data.get('rec_ext'), date=json_data.get('date'), recordkey=json_data.get('recordkey'), center=json_data.get('center') ) json_output_dir_path = '{0}/{1}/{2}'.format( JSON_CONFIG['json_output_path'], json_data.get('start_time')[:10], json_data.get('biz')) json_output_file_path = os.path.join(json_output_dir_path, os.path.basename(file_path)) if not os.path.exists(json_output_dir_path): os.makedirs(json_output_dir_path) if os.path.exists(json_output_file_path): del_garbage(logger, json_output_file_path) logger.debug("Move json file to output directory.") shutil.move(file_path, json_output_dir_path) encrypt_file([json_output_file_path]) mysql.conn.commit() UPLOAD_CNT += 1 except Exception: exc_info = traceback.format_exc() logger.error(exc_info) mysql.conn.rollback() error_process(logger, file_path) continue