def run_task(self, task_param): mysql = Custom_MySQL(using='etl_manage') mysql.begin() try: ''' 业务代码块放下方 ''' dir_param ={'game':task_param['game'], 'platform':task_param['platform'], 'log_date':task_param['log_date'], 'log_name':task_param['log_name']} filename_dict = {'log_name':task_param['log_name'],'log_time':task_param['log_time']} ''' 游戏\平台\日期\业务日志名\日志或者md5文件 ''' log_dir = "/%(game)s/%(platform)s/%(log_date)s/%(log_name)s/" % dir_param lzo_file_name = "%(log_name)s_%(log_time)s.txt"% filename_dict local_log_dir = '/tmp'+log_dir dump_sql = task_param['dump_sql'] dump_sql = dump_sql.replace('{table_name}',task_param['table_name']) dump_sql = dump_sql.replace('{partition_name}',task_param['partition_name']) dump_sql = dump_sql.replace('{db_name}',task_param['db_name']) print(dump_sql) result = mysql.dump(sql,local_log_dir+lzo_file_name) #print(result) ''' 将任务标识为加载文件完成:2 ''' datas = {'load_status':2} where = {} where['id'] = int(task_param['id']) mysql.update('etl_data_log', ' id = %(id)d' % where, **datas) mysql.commit() mysql.close() return True except Exception as exc: print (exc) mysql.rollback() raise self.retry(exc=exc, countdown=60)
def run_task(self, task_param): mysql = Custom_MySQL(using='etl_manage') mysql.begin() try: ''' 业务代码块放下方 ''' dir_param = { 'game': task_param['game'], 'platform': task_param['platform'], 'log_date': task_param['log_date'], 'log_name': task_param['log_name'] } filename_dict = { 'log_name': task_param['log_name'], 'log_time': task_param['log_time'] } ''' 游戏\平台\日期\业务日志名\日志或者md5文件 ''' log_dir = "/%(game)s/%(platform)s/%(log_date)s/%(log_name)s/" % dir_param lzo_file_name = "%(log_name)s_%(log_time)s.txt" % filename_dict local_log_dir = '/tmp' + log_dir dump_sql = task_param['dump_sql'] dump_sql = dump_sql.replace('{table_name}', task_param['table_name']) dump_sql = dump_sql.replace('{partition_name}', task_param['partition_name']) dump_sql = dump_sql.replace('{db_name}', task_param['db_name']) print(dump_sql) result = mysql.dump(sql, local_log_dir + lzo_file_name) #print(result) ''' 将任务标识为加载文件完成:2 ''' datas = {'load_status': 2} where = {} where['id'] = int(task_param['id']) mysql.update('etl_data_log', ' id = %(id)d' % where, **datas) mysql.commit() mysql.close() return True except Exception as exc: print(exc) mysql.rollback() raise self.retry(exc=exc, countdown=60)
def query_mysql(sql): mysql = Custom_MySQL(using='etl_manage') try: mysql.begin() result = mysql.query(sql) mysql.commit() mysql.close() return result except Exception as exc: #回滚 mysql.rollback() print(exc) mysql.close()
def query_mysql(sql): mysql = Custom_MySQL(using='etl_manage') try: mysql.begin() result = mysql.query(sql) mysql.commit() mysql.close() return result except Exception as exc: #回滚 mysql.rollback() print(exc) mysql.close()
def run_task(self, task_param): mysql = Custom_MySQL(using='hadoops2') mysql_etl = Custom_MySQL(using='etl_manage') redis = Custom_Redis(using='etl_task') mysql.begin() mysql_etl.begin() datas = {'status': 0} where = {'id': int(task_param['id'])} task_key_exc = "" try: hive = Custom_Hive(using='ares_dw') game = task_param['game'] platform = task_param['platform'] table_name = task_param['table_name'] log_date = task_param['log_date'] prefix_sql = task_param['prefix_sql'] exec_sql = task_param['exec_sql'] post_sql = task_param['post_sql'] date_cycle = task_param['date_cycle'] random_str = str(random.randint(0, 999999999)) stimes = str(int(time.time())) task_date = task_param['task_date'] task_name = task_param['task_name'] do_rate = task_param['do_rate'] #task_key_tmp = str(game)+str(platform)+str(task_name)+str(date_cycle)+str(do_rate)+str(log_date)+str(task_date) task_key_tmp = str(task_name) + str(date_cycle) + str(do_rate) + str( log_date) + str(task_date) task_key_md5 = hashlib.md5() task_key_md5.update(task_key_tmp) task_key_md5_result = task_key_md5.hexdigest() task_key = "dm2report_new_" + str(game) + "_" + str( platform) + "_" + str(task_key_md5_result) task_key_exc = task_key ''' celery 本身的机制存在漏洞,会将一个已经完成任务再次分配给其他的worker,致使同一个任务执行多次 为防止此种现象,在任务开始执行时,将任务的“唯一标示”写入redis中,标注已执行 ''' #如果task_key is None, 则表示该条任务没有执行过,正常执行即可 #如果task_key = 0, 则表示该条任务上次执行失败,允许重复执行 if redis.get(task_key) == "0" or redis.get(task_key) is None: tmp_file_dir = "/tmp/tmp/%s/%s/%s" % (game, platform, log_date) #创建本地目录 if not os.path.exists(tmp_file_dir): os.makedirs(tmp_file_dir) tmp_file = "%s/%s_%s_%s_%s.txt" % (tmp_file_dir, table_name, date_cycle, random_str, stimes) hql_conf = "SET hive.support.concurrency=false;" \ "SET hive.exec.compress.output=true;" \ "SET mapred.output.compression.codec=com.hadoop.compression.lzo.LzopCodec; " ''' 将任务标示为开始执行:1 ''' datas['status'] = 1 datas['start_time'] = str( datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')) mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where, **datas) mysql_etl.commit() #在redis标注 任务正在执行 redis.set(task_key, 1) #执行前置sql,将数据临时写入本地,用于mysql load数据 if prefix_sql is not None: result = mysql.delete_by_sql(prefix_sql) logger.info('exec prefix_sql: delete old data {0}'.format( result['output'])) if result['status'] != 0: logger.error( 'Error Code %s : %s Cmd: %s' % (result['status'], result['output'], prefix_sql)) ''' 执行失败,将其状态标为未执行:0 ''' datas['status'] = 0 #在redis标注 任务未开始执行 redis.set(task_key, 0) else: ''' 将任务标示为删除临时文件完成:2 ''' datas['status'] = 2 datas.pop('start_time') mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where, **datas) mysql_etl.commit() ''' 开始执行hive ql,将数据dump到本地 ''' result = hive.dump(hql_conf + exec_sql, tmp_file) logger.info('exec exec_sql: dump data {0}'.format( result['output'])) if result['status'] != 0 or False == os.path.exists( '%s' % tmp_file): logger.error( 'Error Code %s : %s Cmd: %s' % (result['status'], result['output'], exec_sql)) #在redis标注 任务未开始执行 redis.set(task_key, 0) else: ''' 将任务标示为dump hive数据完成:3 ''' datas['status'] = 3 datas['tmp_file_name'] = tmp_file mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where, **datas) mysql_etl.commit() #执行后置sql if post_sql is not None: post_sql = post_sql.replace('{dir_path}', tmp_file) post_sql = post_sql.replace( '{table_name}', task_param['table_name']) post_sql = post_sql.replace( '{db_name}', task_param['db_name']) result = mysql.load(post_sql) logger.info( 'exec post_sql: load data to hdfs {0}'.format( result['output'])) if result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], post_sql)) #在redis标注 任务未开始执行 redis.set(task_key, 0) else: ''' 将任务标识为录入mysql完成:4 ''' datas['status'] = 4 datas['end_time'] = str( datetime.datetime.now().strftime( '%Y-%m-%d %H:%M:%S')) datas.pop('tmp_file_name') #在redis标注 任务已成功执行 redis.set(task_key, 2) else: logger.error('prefix_sql is null') datas['status'] = 0 #在redis标注 任务未开始执行 redis.set(task_key, 0) #如果task_key=2, 则标示该条任务已经运行成功 elif redis.get(task_key) == "2": datas['status'] = 4 #该条任务正在运行中 else: return True ''' 将任务标示为:(模拟) 已从任务队列中移除 ''' datas['in_queue'] = 0 update_result = mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where, **datas) # 如果数据库更新失败,再调用一次。 如果还是失败,等待自动修复机制,但这样的概率应该很小了。 if update_result != 1: mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where, **datas) mysql_etl.commit() mysql.commit() mysql_etl.close() mysql.close() return True except Exception as exc: logger.error('dm2report error: %s' % exc) mysql_etl.rollback() mysql.rollback() redis.set(task_key_exc, 0) datas = {'in_queue': 0, 'status': 0} mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where, **datas) mysql_etl.commit() mysql_etl.close() mysql.close() raise self.retry(exc=exc, countdown=60)
if file2dw_result['status'] != 0: print(now+" restart file2dw task error") file2dw_result_1 = mysql.update_by_sql('update file2dw_log set load_status=0,exec_num=0,in_queue=0 ' 'where load_status!=5 and in_queue=1 and log_time <="%s" and task_date="%s"' '' % (log_time_before_1hour, task_date)) if file2dw_result_1['status'] != 0: print(now+" restart file2dw_1 task error") if '0500' < log_time < '1100': #重置dm2report_status dm2report_result = mysql.update_by_sql('update dm2report_log set status=0,exec_num=0,in_queue=0 ' 'where status=-1 and task_date="%s"' % task_date) if dm2report_result['status'] != 0: print(now+" restart dm2report task error") dm2report_result_1 = mysql.update_by_sql('update dm2report_log set status=0,exec_num=0,in_queue=0 ' 'where status!=4 and in_queue=1 and log_time <="%s" and task_date="%s"' '' % (log_time_before_1hour, task_date)) if dm2report_result_1['status'] != 0: print(now+" restart dm2report_1 task error") mysql.commit() mysql.close() except Exception as exc: #回滚 mysql.rollback() print(exc) mysql.close()
def run_task(self, task_param): mysql = Custom_MySQL(using='etl_manage') mysql.begin() where = {'id': int(task_param['id'])} try: dir_param = {'game': task_param['game'], 'platform': task_param['platform'], 'log_date': task_param['log_date'], 'log_name': task_param['log_name']} filename_dict = {'log_name': task_param['log_name'], 'log_time': task_param['log_time'], 'source_ip':task_param['source_ip']} index_dict = {'db_name': task_param['db_name'], 'table_name': task_param['table_name'], 'platform': task_param['platform'], 'log_date': datetime.datetime.strptime(task_param['log_date'], '%Y%m%d').strftime("%Y-%m-%d")} partition = {'platform': task_param['platform'], 'log_date': datetime.datetime.strptime(task_param['log_date'], '%Y%m%d').strftime("%Y-%m-%d")} log_dir = "/%(game)s/%(platform)s/%(log_date)s/%(log_name)s/" % dir_param lzo_file_name = "%(log_name)s_%(source_ip)s_%(log_time)s.txt.lzo" % filename_dict index_dir_name = "%(db_name)s.db/%(table_name)s/plat_form=%(platform)s/log_date=%(log_date)s/" % index_dict partition_name = "plat_form='%(platform)s',log_date='%(log_date)s'" % partition project_path = os.getcwd() local_log_dir = '/disk1/tmp_data'+log_dir logger.info('local_log_dir: {0}'.format(local_log_dir)) #判断要录入hive 中的文件知否存在,存在则执行 if os.path.exists('%s%s' % (local_log_dir, lzo_file_name)): ''' 将任务标识为开始执行:1 ''' datas = {'load_status': 1} mysql.update('file2dw_log', ' id = %(id)d' % where, **datas) #执行load之前,删除同名文件,防止同一个文件出现两次的可能 cmd_remove = '/bin/bash %s/file2dw/remove_damaged_file.sh %s %s' % (project_path, index_dir_name, lzo_file_name) logger.info('remove damaged files: {0}'.format(cmd_remove)) remove_result = cmd.run(cmd_remove) if remove_result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove)) ''' 文件加载到hive中 ''' hive = Custom_Hive(using='ares_dw') load_sql = task_param['load_sql'] load_sql = load_sql.replace('{dir_path}', local_log_dir+lzo_file_name) load_sql = load_sql.replace('{table_name}', task_param['table_name']) load_sql = load_sql.replace('{partition_name}', '%s' % partition_name) load_sql = load_sql.replace('{db_name}', task_param['db_name']) logger.info('hive load SQL: {0}'.format(load_sql)) result = hive.load(load_sql) logger.info('hive load result {0}'.format(result['output'])) if result['status'] == 0: ''' 将任务标识为加载文件完成:2 ''' datas = {'load_status': 2} mysql.update('file2dw_log', ' id = %(id)d' % where, **datas) ''' 建立索引,否则lzo将不支持split ''' #print index_dir_name cmd_index = '/bin/bash %s/file2dw/create_lzo_indexer.sh %s %s' % (project_path, index_dir_name, lzo_file_name) logger.info('create lzo index: {0}'.format(cmd_index)) index_result = cmd.run(cmd_index) if index_result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (index_result['status'], index_result['output'], cmd_index)) else: if "create index success" in index_result['output']: ''' 将任务标识为建立lzo索引完成:3 ''' datas = {'load_status': 3} mysql.update('file2dw_log', ' id = %(id)d' % where, **datas) else: ''' 如果load数据失败,则删除半途出现错误的文件,方式hive查询的时候报错 ''' cmd_remove = '/bin/bash %s/file2dw/remove_damaged_file.sh %s %s' % (project_path, index_dir_name, lzo_file_name) logger.info('remove damaged files: {0}'.format(cmd_remove)) remove_result = cmd.run(cmd_remove) if remove_result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove)) else: ''' 将任务标识为未启动,重新执行:0 ''' datas = {'load_status': 0} mysql.update('file2dw_log', ' id = %(id)d' % where, **datas) ''' 将任务标示为:(模拟) 已从任务队列中移除 ''' datas = {'in_queue': 0} update_result = mysql.update('file2dw_log', ' id = %(id)d' % where, **datas) # 如果更新失败,则再调用一次,如果还是失败,则等待自动修复机制,但这种概率很小了 if update_result != 1: mysql.update('file2dw_log', ' id = %(id)d' % where, **datas) mysql.commit() mysql.close() return True except Exception as exc: print (exc) mysql.rollback() datas = {'in_queue': 0} mysql.update('file2dw_log', ' id = %(id)d' % where, **datas) mysql.commit() mysql.close() raise self.retry(exc=exc, countdown=60)
class CenterApp(): def __init__(self): self.center_mysql = Custom_MySQL(using='center_app') # def get_server_list(self): # ''' # 获取执行任务 # ''' # try: # ip_list = [] # if param['flag'] == 'basic': # ip_list.append('120.26.1.250') # # if param['flag'] == 'most': # ip_list.append('120.26.13.150') # # if param['flag'] == 'log': # s_sql = "select t1.* from \ # (select a.public_ip as source_ip,a.platform_id from assets a inner join main_category m \ # on a.main_category_id = m.id \ # where a.is_del = 0 and (a.hostname like '%%%%web_balance%%%%' or a.hostname like '%%%%web_admin%%%%') ) t1 \ # left join platform t2 on t1.platform_id = t2.id \ # where t1.source_ip is not null group by source_ip" # # if param['game'] == 'kof': # s_sql = "select t1.* from \ # (select a.public_ip as source_ip,a.platform_id from assets a inner join main_category m \ # on a.main_category_id = m.id \ # where a.is_del = 0 and a.hostname like '%%%%gameserver%%%%' ) t1 \ # left join platform t2 on t1.platform_id = t2.id \ # where t1.source_ip is not null group by source_ip" # # ip_list = self.exec_sql(s_sql) # # if param['flag'] == 'snap': # s_sql = "select s.prefix as source_ip from sub_category s inner join main_category m \ # on s.main_category_id = m.id \ # where s.platform = '%s' and m.prefix = '%s'" % (param['platform'],param['game']) # ip_list = self.exec_sql(s_sql) # # return ip_list # except Exception as exc: # print exc # #异常回滚 # self.center_mysql.rollback() def get_log_ip(self): ''' 获取所有log的IP ''' ip_list = {} sql = "select t1.*,t2.prefix as platform from \ (select m.prefix as gamename,a.public_ip as source_ip,a.platform_id from assets a inner join main_category m \ on a.main_category_id = m.id \ where a.is_del = 0 and (a.hostname like '%%%%web_balance%%%%' or a.hostname like '%%%%web_admin%%%%') ) t1 \ left join platform t2 on t1.platform_id = t2.id \ where t1.source_ip is not null group by source_ip\ union all \ select t1.*,t2.prefix as platform from \ (select m.prefix as gamename,a.public_ip as source_ip,a.platform_id from assets a inner join main_category m \ on a.main_category_id = m.id \ where a.is_del = 0 and a.hostname like '%%%%gameserver%%%%' ) t1 \ left join platform t2 on t1.platform_id = t2.id \ where t1.source_ip is not null group by source_ip" result = self.center_mysql.query(sql) for info in result: if info['gamename'] is None or info['platform'] is None: continue if info['gamename'].encode('utf8') not in ip_list: ip_list[info['gamename'].encode('utf8')] = {} if info['platform'].encode('utf8') not in ip_list[info['gamename'].encode('utf8')]: ip_list[info['gamename'].encode('utf8')][info['platform'].encode('utf8')] = [] ip_list[info['gamename'].encode('utf8')][info['platform'].encode('utf8')].append(info['source_ip'].encode('utf8')) return ip_list def get_snap_ip(self): ''' 获取所有快照的IP ''' ip_list = {} s_sql = "select f.prefix as platform,s.prefix as source_ip,m.prefix as gamename from sub_category s inner join main_category m \ on s.main_category_id = m.id \ left join platform f on f.id = s.platform_id" result = self.center_mysql.query(s_sql) for info in result: if info['gamename'] is None or info['platform'] is None: continue if info['gamename'].encode('utf8') not in ip_list: ip_list[info['gamename'].encode('utf8')] = {} if info['platform'].encode('utf8') not in ip_list[info['gamename'].encode('utf8')]: ip_list[info['gamename'].encode('utf8')][info['platform'].encode('utf8')] = [] ip_list[info['gamename'].encode('utf8')][info['platform'].encode('utf8')].append(info['source_ip'].encode('utf8')) # print ip_list return ip_list def get_ip_list(self): ''' 整理IP列表 ''' try: ip_list = {} ip_list['basic'] = ['120.26.1.250'] ip_list['mostsdk'] = ['120.26.13.150'] ip_list['wanpay'] = ['112.124.116.44'] log_list = self.get_log_ip(); ip_list['log'] = log_list snap_list = self.get_snap_ip() ip_list['snap'] = snap_list return ip_list except Exception,e: print e print "异常" #异常回滚 self.center_mysql.rollback()
def run_task(self, task_param): redis = Custom_Redis(using='etl_task') mysql = Custom_MySQL(using='etl_manage') mysql.begin() datas = {'status': 0} where = {'id': int(task_param['id'])} hive = Custom_Hive(using='hadoops2') task_key_exc = "" try: game = task_param['game'] platform = task_param['platform'] task_dict = {'log_name': task_param['log_name'], 'do_rate': task_param['do_rate'], 'log_date': task_param['log_date'], 'task_date': task_param['task_date']} task_key_tmp = "%(log_name)s%(do_rate)s%(log_date)s%(task_date)s" % task_dict task_key_md5 = hashlib.md5() task_key_md5.update(task_key_tmp) task_key_md5_result = task_key_md5.hexdigest() task_key = "dw2dm_"+str(game)+"_"+str(platform)+"_"+str(task_key_md5_result) task_key_exc = task_key ''' celery 本身的机制存在漏洞,会将一个已经完成任务再次分配给其他的worker,致使同一个任务执行多次 为防止此种现象,在任务开始执行时,将任务的“唯一标示”写入redis中,标注已执行 ''' #如果task_key is None, 则表示该条任务没有执行过,正常执行即可 #如果task_key = 0, 则表示该条任务上次执行失败,允许重复执行 if redis.get(task_key) == "0" or redis.get(task_key) is None: exec_sql = task_param['exec_sql'] log_date = (datetime.datetime.strptime(task_param['log_date'], '%Y%m%d')).strftime("%Y-%m-%d") #prefix_sql = task_param['prefix_sql'] #post_sql = task_param['post_sql'] index_dict = {'db_name': task_param['db_name'], 'table_name': task_param['table_name'], 'platform': task_param['platform'], 'log_date': log_date} index_dir_name = "%(db_name)s.db/%(table_name)s/plat_form=%(platform)s/log_date=%(log_date)s/" % index_dict #用于删除索引 del_index_dir_name = "%(db_name)s.db/%(table_name)s/plat_form=%(platform)s/log_date=%(log_date)s/" % index_dict if platform == 'all': index_dict = {'db_name': task_param['db_name'], 'table_name': task_param['table_name'], 'log_date': log_date} #用于建立索引(建立索引时,不能使用*通配符,所以仅指定到表名) index_dir_name = "%(db_name)s.db/%(table_name)s/" % index_dict #用于删除索引,虽然建立的时候指定到表名,但删除的时候,仅删除当天的 del_index_dir_name = "%(db_name)s.db/%(table_name)s/*/log_date=%(log_date)s/*" % index_dict hql_conf = "set hive.exec.dynamic.partition.mode=nonstrict;" \ "set hive.exec.compress.output=true;" \ "set mapred.output.compress=true;" \ "set mapred.output.compression.codec=com.hadoop.compression.lzo.LzopCodec;" \ "set io.compression.codecs=com.hadoop.compression.lzo.LzopCodec; " #获取项目根路径:/data/etl_manage project_path = os.getcwd() ''' 将任务标示为开始执行:1 ''' datas['status'] = 1 datas['start_time'] = str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')) mysql.update('dw2dm_log', ' id = %(id)d' % where, **datas) mysql.commit() #在redis标注 任务正在执行 redis.set(task_key, 1) #执行正式的 select xx insert xx 语句 result = hive.select_insert(hql_conf+exec_sql) logger.info('exec exec_sql: select xxx insert xxx {0}'.format(result['output'])) if result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], exec_sql)) ''' 执行失败,将其状态标为未执行:0 ''' datas['status'] = 0 #在redis标注 任务未执行 redis.set(task_key, 0) else: ''' 将任务标示为执行hive ql, select xxx insert xxx完成:2 ''' datas['status'] = 2 datas.pop('start_time') mysql.update('dw2dm_log', ' id = %(id)d' % where, **datas) mysql.commit() ''' 建立索引,否则lzo将不支持split ''' cmd_index = '/bin/bash %s/dw2dm/create_lzo_indexer.sh %s' % (project_path, index_dir_name) logger.info('create lzo index: {0}'.format(cmd_index)) index_result = cmd.run(cmd_index) if index_result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (index_result['status'], index_result['output'], cmd_index)) #在redis标注 任务未执行 redis.set(task_key, 0) else: if "create index success" in index_result['output']: ''' 将任务标识为建立lzo索引完成:3 ''' datas['status'] = 3 datas['end_time'] = str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')) #在redis标注 任务已完成且成功 redis.set(task_key, 2) else: ''' 如果load数据失败,则删除半途出现错误的文件,防止hive查询的时候报错 ''' cmd_remove = '/bin/bash %s/dw2dm/remove_damaged_file.sh %s' % (project_path, del_index_dir_name) logger.info('remove damaged files: {0}'.format(cmd_remove)) remove_result = cmd.run(cmd_remove) if remove_result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove)) #在redis标注 任务未执行 redis.set(task_key, 0) #如果task_key=2, 则标示该条任务已经运行成功 elif redis.get(task_key) == "2": datas['status'] = 3 #该条任务正在运行中 else: return True ''' 执行完毕,模拟从队列中清除任务:0 ''' datas['in_queue'] = 0 update_result = mysql.update('dw2dm_log', ' id = %(id)d' % where, **datas) # 如果更新失败,则再调用一次,如果还是失败,则等待自动修复机制,但这种概率很小了 if update_result != 1: mysql.update('dw2dm_log', ' id = %(id)d' % where, **datas) mysql.commit() mysql.close() return True except Exception as exc: logger.error('dw2dm_log error: %s' % exc) redis.set(task_key_exc, 0) mysql.rollback() datas = {'in_queue': 0, 'status': 0} mysql.update('dw2dm_log', ' id = %(id)d' % where, **datas) mysql.commit() mysql.close() raise self.retry(exc=exc, countdown=60)
def run_task(self, task_param): mysql = Custom_MySQL(using='hadoops2') mysql_etl = Custom_MySQL(using='etl_manage') mysql.begin() mysql_etl.begin() where = {'id': int(task_param['id'])} try: hive = Custom_Hive(using='ares_dw') game = task_param['game'] platform = task_param['platform'] table_name = task_param['table_name'] log_date = task_param['log_date'] prefix_sql = task_param['prefix_sql'] exec_sql = task_param['exec_sql'] post_sql = task_param['post_sql'] date_cycle = task_param['date_cycle'] random_str = str(random.randint(0, 999999999)) stimes = str(int(time.time())) tmp_file_dir = "/tmp/tmp/%s/%s/%s" % (game, platform, log_date) #创建本地目录 if not os.path.exists(tmp_file_dir): os.makedirs(tmp_file_dir) tmp_file = "%s/%s_%s_%s_%s.txt" % (tmp_file_dir, table_name, date_cycle, random_str, stimes) hql_conf = "SET hive.support.concurrency=false;SET hive.exec.compress.output=true;" \ "SET mapred.output.compression.codec=com.hadoop.compression.lzo.LzopCodec; " ''' 将任务标示为开始执行:1 ''' datas = { 'status': 1, 'start_time': str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')) } mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas) mysql_etl.commit() #执行前置sql,将数据临时写入本地,用于mysql load数据 if prefix_sql is not None: result = mysql.delete_by_sql(prefix_sql) logger.info('exec prefix_sql: delete old data {0}'.format( result['output'])) if result['status'] == 0: ''' 将任务标示为删除临时文件完成:2 ''' datas = {'status': 2} mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas) mysql_etl.commit() ''' 开始执行hive ql,将数据dump到本地 ''' result = hive.dump(hql_conf + exec_sql, tmp_file) logger.info('exec exec_sql: dump data {0}'.format( result['output'])) if result['status'] == 0 and True == os.path.exists( '%s' % tmp_file): ''' 将任务标示为dump hive数据完成:3 ''' datas = {'status': 3, 'tmp_file_name': tmp_file} mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas) mysql_etl.commit() #执行后置sql if post_sql is not None: post_sql = post_sql.replace('{dir_path}', tmp_file) post_sql = post_sql.replace('{table_name}', task_param['table_name']) post_sql = post_sql.replace('{db_name}', task_param['db_name']) result = mysql.load(post_sql) logger.info( 'exec post_sql: load data to hdfs {0}'.format( result['output'])) if result['status'] == 0: ''' 将任务标识为录入mysql完成:4 ''' datas = { 'status': 4, 'end_time': str(datetime.datetime.now().strftime( '%Y-%m-%d %H:%M:%S')) } mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas) else: logger.error( 'Error Code %s : %s Cmd: %s' % (result['status'], result['output'], post_sql)) else: logger.error( 'Error Code %s : %s Cmd: %s' % (result['status'], result['output'], exec_sql)) else: logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], prefix_sql)) ''' 执行失败,将其状态标为未执行:0 ''' datas = {'status': 0} mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas) ''' 将任务标示为:(模拟) 已从任务队列中移除 ''' datas = {'in_queue': 0} update_result = mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas) # 如果数据库更新失败,再调用一次。 如果还是失败,等待自动修复机制,但这样的概率应该很小了。 if update_result != 1: mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas) mysql_etl.commit() mysql.commit() mysql_etl.close() mysql.close() return True except Exception as exc: logger.error('dm2report error: %s' % exc) mysql_etl.rollback() mysql.rollback() datas = {'in_queue': 0, 'status': 0} mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas) mysql_etl.commit() mysql_etl.close() mysql.close() raise self.retry(exc=exc, countdown=60)
def run_task(self, task_param): mysql = Custom_MySQL(using='hadoops2') mysql_etl = Custom_MySQL(using='etl_manage') mysql.begin() mysql_etl.begin() where = {'id': int(task_param['id'])} try: hive = Custom_Hive(using='ares_dw') game = task_param['game'] platform = task_param['platform'] table_name = task_param['table_name'] log_date = task_param['log_date'] prefix_sql = task_param['prefix_sql'] exec_sql = task_param['exec_sql'] post_sql = task_param['post_sql'] date_cycle = task_param['date_cycle'] random_str = str(random.randint(0, 999999999)) stimes = str(int(time.time())) tmp_file_dir = "/tmp/tmp/%s/%s/%s" % (game, platform, log_date) #创建本地目录 if not os.path.exists(tmp_file_dir): os.makedirs(tmp_file_dir) tmp_file = "%s/%s_%s_%s_%s.txt" % (tmp_file_dir, table_name, date_cycle, random_str, stimes) hql_conf = "SET hive.support.concurrency=false;SET hive.exec.compress.output=true;" \ "SET mapred.output.compression.codec=com.hadoop.compression.lzo.LzopCodec; " ''' 将任务标示为开始执行:1 ''' datas = {'status': 1, 'start_time': str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))} mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas) mysql_etl.commit() #执行前置sql,将数据临时写入本地,用于mysql load数据 if prefix_sql is not None: result = mysql.delete_by_sql(prefix_sql) logger.info('exec prefix_sql: delete old data {0}'.format(result['output'])) if result['status'] == 0: ''' 将任务标示为删除临时文件完成:2 ''' datas = {'status': 2} mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas) mysql_etl.commit() ''' 开始执行hive ql,将数据dump到本地 ''' result = hive.dump(hql_conf+exec_sql, tmp_file) logger.info('exec exec_sql: dump data {0}'.format(result['output'])) if result['status'] == 0 and True == os.path.exists('%s' % tmp_file): ''' 将任务标示为dump hive数据完成:3 ''' datas = {'status': 3, 'tmp_file_name': tmp_file} mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas) mysql_etl.commit() #执行后置sql if post_sql is not None: post_sql = post_sql.replace('{dir_path}', tmp_file) post_sql = post_sql.replace('{table_name}', task_param['table_name']) post_sql = post_sql.replace('{db_name}', task_param['db_name']) result = mysql.load(post_sql) logger.info('exec post_sql: load data to hdfs {0}'.format(result['output'])) if result['status'] == 0: ''' 将任务标识为录入mysql完成:4 ''' datas = {'status': 4, 'end_time': str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))} mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas) else: logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], post_sql)) else: logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], exec_sql)) else: logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], prefix_sql)) ''' 执行失败,将其状态标为未执行:0 ''' datas = {'status': 0} mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas) ''' 将任务标示为:(模拟) 已从任务队列中移除 ''' datas = {'in_queue': 0} update_result = mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas) # 如果数据库更新失败,再调用一次。 如果还是失败,等待自动修复机制,但这样的概率应该很小了。 if update_result != 1: mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas) mysql_etl.commit() mysql.commit() mysql_etl.close() mysql.close() return True except Exception as exc: logger.error('dm2report error: %s' % exc) mysql_etl.rollback() mysql.rollback() datas = {'in_queue': 0, 'status': 0} mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas) mysql_etl.commit() mysql_etl.close() mysql.close() raise self.retry(exc=exc, countdown=60)
def run_task(self, task_param): redis = Custom_Redis(using='etl_manage') mysql = Custom_MySQL(using='etl_manage') mysql.begin() datas = {'download_status': 0} where = {'id': int(task_param['id'])} local_log_dir = "" lzo_file_name = "" try: dir_param = { 'game': task_param['game'], 'platform': task_param['platform'], 'log_date': task_param['log_date'], 'log_name': task_param['log_name'] } filename_dict = { 'log_name': task_param['log_name'], 'log_time': task_param['log_time'], 'source_ip': task_param['source_ip'] } log_dir = "/%(game)s/%(platform)s/%(log_date)s/%(log_name)s/" % dir_param txt_file_name = "%(log_name)s_%(source_ip)s_%(log_time)s.txt" % filename_dict lzo_file_name = "%(log_name)s_%(source_ip)s_%(log_time)s.txt.lzo" % filename_dict md5_file_name = "%(log_name)s_%(source_ip)s_%(log_time)s.txt.lzo.md5" % filename_dict lzo_download_url = task_param['download_url'].rstrip( '/') + log_dir + lzo_file_name md5_download_url = task_param['download_url'].rstrip( '/') + log_dir + md5_file_name #从redis中,获取当前数据对应存储到哪块磁盘 if redis.get("disk_xml") is None: disk_tmp = open('/data/etl_manage/conf/disk_game.xml', 'r') redis.set("disk_xml", str(disk_tmp.read())) disk_list = str(redis.get("disk_xml")) root = ET.fromstring(disk_list) disk = "" for gameinfo in root.findall('game'): if gameinfo.get('name') == task_param['game']: disk = gameinfo.get('disk') continue #local_log_dir = '/disk1/tmp_data' + log_dir local_log_dir = '/' + disk + '/data' + log_dir #创建本地目录 if not os.path.exists(local_log_dir): os.makedirs(local_log_dir) #排除同名文件存在的可能,同时为修复执行提供方便 if os.path.exists('%s%s' % (local_log_dir, txt_file_name)): cmd_remove = 'rm -f %s%s*' % (local_log_dir, txt_file_name) logger.info('remove history file: {0}'.format(cmd_remove)) remove_result = cmd.run(cmd_remove) if remove_result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove)) ''' 下载md5文件,如果md5文件不存在则退出,不再继续执行程序,同时不向数据库写入任何标示 ''' md5_line = ' wget -o /tmp/log/wget_log -O %s%s %s' % ( local_log_dir, md5_file_name, md5_download_url) logger.info('md5 info: {0}'.format(md5_line)) md5_result = cmd.run(md5_line) if md5_result['status'] != 0: logger.error( 'Error Code %s : %s Cmd: %s' % (md5_result['status'], md5_result['output'], md5_line)) else: ''' 将任务标识为开始执行:1 ''' datas['download_status'] = 1 mysql.update('etl_data_log', ' id = %(id)d' % where, **datas) mysql.commit() ''' 下载数据文件 ''' lzo_line = ' wget -o /tmp/log/wget_log -O %s%s %s' % ( local_log_dir, lzo_file_name, lzo_download_url) logger.info('file info: {0}'.format(lzo_line)) lzo_result = cmd.run(lzo_line) if lzo_result['status'] != 0: logger.error( 'Error Code %s : %s Cmd: %s' % (lzo_result['status'], lzo_result['output'], lzo_line)) datas['download_status'] = 0 else: ''' 将任务标识为下载完成:2 ''' datas['download_status'] = 2 mysql.update('etl_data_log', ' id = %(id)d' % where, **datas) mysql.commit() ''' md5校验,如果未通过则不再继续执行程序 ''' check_line = "cat %s%s |grep `md5sum %s%s|cut -d ' ' -f 1`" % ( local_log_dir, md5_file_name, local_log_dir, lzo_file_name) logger.info('md5 or md5 info: {0}'.format(check_line)) check_result = cmd.run(check_line) if check_result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (check_result['status'], check_result['output'], check_line)) datas['download_status'] = 0 else: ''' lzop解压缩 ''' cmd_line = ' lzop -dP %s%s' % (local_log_dir, lzo_file_name) logger.info('file info: {0}'.format(cmd_line)) cmd_result = cmd.run(cmd_line) if cmd_result['status'] != 0: logger.error('Lzop Code %s : %s Cmd: %s' % (cmd_result['status'], cmd_result['output'], cmd_line)) datas['download_status'] = 0 else: ''' 将任务标识md5一致,完成下载任务:3 ''' datas['download_status'] = 3 ''' 将任务标示为:(模拟) 已从任务队列中移除 ''' datas['in_download_queue'] = 0 update_result = mysql.update('etl_data_log', ' id = %(id)d' % where, **datas) # 如果更新失败,则再调用一次,如果还是失败,则等待自动修复机制,但这种概率很小了 if update_result != 1: mysql.update('etl_data_log', ' id = %(id)d' % where, **datas) mysql.commit() mysql.close() return True except Exception as exc: print(exc) logger.error('download error : %s' % exc) mysql.rollback() kill_proces = "kill -9 `ps -ef |grep wget |grep -v grep |grep '%s%s'|awk '{print $2}'`" % ( local_log_dir, lzo_file_name) cmd.run(kill_proces) datas = {'in_download_queue': 0, 'download_status': 0} mysql.update('etl_data_log', ' id = %(id)d' % where, **datas) mysql.commit() mysql.close() raise self.retry(exc=exc, countdown=60)
def run_task(self, task_param): mysql = Custom_MySQL(using='etl_manage') mysql.begin() datas = {'etl_status': 0} where = {'id': int(task_param['id'])} try: log_param = { 'game': task_param['game'], 'platform': task_param['platform'], 'log_date': task_param['log_date'], 'log_name': task_param['log_name'], 'log_dir': task_param['log_dir'], 'col_num': task_param['col_num'] } log_name_param = { 'log_name': task_param['log_name'], 'source_ip': task_param['source_ip'], 'log_time': task_param['log_time'] } do_rate = task_param['do_rate'] flag = task_param['flag'] log_dir = '%(log_dir)s/%(game)s/%(platform)s/%(log_date)s/%(log_name)s' % log_param log_name = '%(log_name)s_%(source_ip)s_%(log_time)s' % log_name_param log_name_notime = '%(log_name)s_%(source_ip)s' % log_name_param col_num = task_param['col_num'] project_path = os.getcwd() #判断如果有md5文件和数据文件同时存在,则开始执行 if (do_rate == "1day" and flag == "log") or ( os.path.exists('%s/%s.log.md5' % (log_dir, log_name)) is True and os.path.exists('%s/%s.log' % (log_dir, log_name)) is True): #排除同名文件存在的可能,同时为修复执行提供方便 if os.path.exists('%s/%s.txt' % (log_dir, log_name)): cmd_remove = 'rm -f %s/%s.txt*' % (log_dir, log_name) logger.info('remove history file: {0}'.format(cmd_remove)) remove_result = cmd.run(cmd_remove) if remove_result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove)) ''' 将任务标识为开始执行:1 ''' datas['etl_status'] = 1 mysql.update('etl_data_log', ' id = %(id)d' % where, **datas) mysql.commit() ''' 校验数据 ''' #如果是“天”频次的日志数据,则特殊处理,直接将同一ip的合成一个文件,同时执行校验 if do_rate == "1day" and flag == "log": cmd_merge = '/bin/bash %s/etl_data/merge_data.sh %s %s %s' % ( project_path, log_dir, log_name_notime, col_num) logger.info('check data: {0}'.format(cmd_merge)) merge_result = cmd.run(cmd_merge) else: cmd_merge = '/bin/bash %s/etl_data/check_data.sh %s %s %s' % ( project_path, log_dir, log_name, col_num) logger.info('check data: {0}'.format(cmd_merge)) merge_result = cmd.run(cmd_merge) if merge_result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (merge_result['status'], merge_result['output'], cmd_merge)) datas['etl_status'] = 0 else: ''' 读取校验格式后的文件总条数 ''' row = open('%s/%s.txt.row' % (log_dir, log_name)).read() ''' 将文件总条数写入数据库,并将任务标识为为校验已完成:2 ''' datas['etl_status'] = 2 datas['row_num'] = int(row) #datas = {'etl_status': 2, 'row_num': int(row)} mysql.update('etl_data_log', ' id = %(id)d' % where, **datas) mysql.commit() ''' 压缩数据 ''' cmd_compress = '/bin/bash %s/etl_data/compress_data.sh %s %s' % ( project_path, log_dir, log_name) logger.info('compress data: {0}'.format(cmd_compress)) compress_result = cmd.run(cmd_compress) if compress_result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (compress_result['status'], compress_result['output'], cmd_compress)) datas['etl_status'] = 0 else: ''' 将任务标识为压缩完成:4 ''' datas['etl_status'] = 4 datas.pop('row_num') mysql.update('etl_data_log', ' id = %(id)d' % where, **datas) mysql.commit() ''' 生成MD5文件 ''' cmd_md5 = '/bin/bash %s/etl_data/md5_data.sh %s %s' % ( project_path, log_dir, log_name) logger.info('md5 data: {0}'.format(cmd_md5)) md5_result = cmd.run(cmd_md5) if md5_result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (md5_result['status'], md5_result['output'], cmd_md5)) datas['etl_status'] = 0 else: ''' 将任务标识为生成MD5完成(即为校验、合并、压缩均已完成):6 ''' datas['etl_status'] = 6 ''' 执行完毕,模拟从队列中清楚任务:0 ''' datas['in_etl_queue'] = 0 update_result = mysql.update('etl_data_log', ' id = %(id)d' % where, **datas) # 如果更新失败,则再调用一次,如果还是失败,则等待自动修复机制,但这种概率很小了 if update_result != 1: mysql.update('etl_data_log', ' id = %(id)d' % where, **datas) mysql.commit() mysql.close() return True except Exception as exc: logger.error('etl_data error: %s' % exc) mysql.rollback() datas = {'in_etl_queue': 0, 'etl_status': 0} mysql.update('etl_data_log', ' id = %(id)d' % where, **datas) mysql.commit() mysql.close() raise self.retry(exc=exc, countdown=30)
def run_task(self, task_param): redis = Custom_Redis(using='etl_manage') mysql = Custom_MySQL(using='etl_manage') mysql.begin() datas = {'download_status': 0} where = {'id': int(task_param['id'])} local_log_dir = "" lzo_file_name = "" try: dir_param = {'game': task_param['game'], 'platform': task_param['platform'], 'log_date': task_param['log_date'], 'log_name': task_param['log_name']} filename_dict = {'log_name': task_param['log_name'], 'log_time': task_param['log_time'], 'source_ip': task_param['source_ip']} log_dir = "/%(game)s/%(platform)s/%(log_date)s/%(log_name)s/" % dir_param txt_file_name = "%(log_name)s_%(source_ip)s_%(log_time)s.txt" % filename_dict lzo_file_name = "%(log_name)s_%(source_ip)s_%(log_time)s.txt.lzo" % filename_dict md5_file_name = "%(log_name)s_%(source_ip)s_%(log_time)s.txt.lzo.md5" % filename_dict lzo_download_url = task_param['download_url'].rstrip('/') + log_dir + lzo_file_name md5_download_url = task_param['download_url'].rstrip('/') + log_dir + md5_file_name #从redis中,获取当前数据对应存储到哪块磁盘 if redis.get("disk_xml") is None: disk_tmp = open('/data/etl_manage/conf/disk_game.xml', 'r') redis.set("disk_xml", str(disk_tmp.read())) disk_list = str(redis.get("disk_xml")) root = ET.fromstring(disk_list) disk = "" for gameinfo in root.findall('game'): if gameinfo.get('name') == task_param['game']: disk = gameinfo.get('disk') continue #local_log_dir = '/disk1/tmp_data' + log_dir local_log_dir = '/'+disk+'/data' + log_dir #创建本地目录 if not os.path.exists(local_log_dir): os.makedirs(local_log_dir) #排除同名文件存在的可能,同时为修复执行提供方便 if os.path.exists('%s%s' % (local_log_dir, txt_file_name)): cmd_remove = 'rm -f %s%s*' % (local_log_dir, txt_file_name) logger.info('remove history file: {0}'.format(cmd_remove)) remove_result = cmd.run(cmd_remove) if remove_result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove)) ''' 下载md5文件,如果md5文件不存在则退出,不再继续执行程序,同时不向数据库写入任何标示 ''' md5_line = ' wget -o /tmp/log/wget_log -O %s%s %s' % (local_log_dir, md5_file_name, md5_download_url) logger.info('md5 info: {0}'.format(md5_line)) md5_result = cmd.run(md5_line) if md5_result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (md5_result['status'], md5_result['output'], md5_line)) else: ''' 将任务标识为开始执行:1 ''' datas['download_status'] = 1 mysql.update('etl_data_log', ' id = %(id)d' % where, **datas) mysql.commit() ''' 下载数据文件 ''' lzo_line = ' wget -o /tmp/log/wget_log -O %s%s %s' % (local_log_dir, lzo_file_name, lzo_download_url) logger.info('file info: {0}'.format(lzo_line)) lzo_result = cmd.run(lzo_line) if lzo_result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (lzo_result['status'], lzo_result['output'], lzo_line)) datas['download_status'] = 0 else: ''' 将任务标识为下载完成:2 ''' datas['download_status'] = 2 mysql.update('etl_data_log', ' id = %(id)d' % where, **datas) mysql.commit() ''' md5校验,如果未通过则不再继续执行程序 ''' check_line = "cat %s%s |grep `md5sum %s%s|cut -d ' ' -f 1`" % (local_log_dir, md5_file_name, local_log_dir, lzo_file_name) logger.info('md5 or md5 info: {0}'.format(check_line)) check_result = cmd.run(check_line) if check_result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (check_result['status'], check_result['output'], check_line)) datas['download_status'] = 0 else: ''' lzop解压缩 ''' cmd_line = ' lzop -dP %s%s' % (local_log_dir, lzo_file_name) logger.info('file info: {0}'.format(cmd_line)) cmd_result = cmd.run(cmd_line) if cmd_result['status'] != 0: logger.error('Lzop Code %s : %s Cmd: %s' % (cmd_result['status'], cmd_result['output'], cmd_line)) datas['download_status'] = 0 else: ''' 将任务标识md5一致,完成下载任务:3 ''' datas['download_status'] = 3 ''' 将任务标示为:(模拟) 已从任务队列中移除 ''' datas['in_download_queue'] = 0 update_result = mysql.update('etl_data_log', ' id = %(id)d' % where, **datas) # 如果更新失败,则再调用一次,如果还是失败,则等待自动修复机制,但这种概率很小了 if update_result != 1: mysql.update('etl_data_log', ' id = %(id)d' % where, **datas) mysql.commit() mysql.close() return True except Exception as exc: print (exc) logger.error('download error : %s' % exc) mysql.rollback() kill_proces = "kill -9 `ps -ef |grep wget |grep -v grep |grep '%s%s'|awk '{print $2}'`" % (local_log_dir, lzo_file_name) cmd.run(kill_proces) datas = {'in_download_queue': 0, 'download_status': 0} mysql.update('etl_data_log', ' id = %(id)d' % where, **datas) mysql.commit() mysql.close() raise self.retry(exc=exc, countdown=60)
class CenterApp(): def __init__(self): self.center_mysql = Custom_MySQL(using='center_app') # def get_server_list(self): # ''' # 获取执行任务 # ''' # try: # ip_list = [] # if param['flag'] == 'basic': # ip_list.append('120.26.1.250') # # if param['flag'] == 'most': # ip_list.append('120.26.13.150') # # if param['flag'] == 'log': # s_sql = "select t1.* from \ # (select a.public_ip as source_ip,a.platform_id from assets a inner join main_category m \ # on a.main_category_id = m.id \ # where a.is_del = 0 and (a.hostname like '%%%%web_balance%%%%' or a.hostname like '%%%%web_admin%%%%') ) t1 \ # left join platform t2 on t1.platform_id = t2.id \ # where t1.source_ip is not null group by source_ip" # # if param['game'] == 'kof': # s_sql = "select t1.* from \ # (select a.public_ip as source_ip,a.platform_id from assets a inner join main_category m \ # on a.main_category_id = m.id \ # where a.is_del = 0 and a.hostname like '%%%%gameserver%%%%' ) t1 \ # left join platform t2 on t1.platform_id = t2.id \ # where t1.source_ip is not null group by source_ip" # # ip_list = self.exec_sql(s_sql) # # if param['flag'] == 'snap': # s_sql = "select s.prefix as source_ip from sub_category s inner join main_category m \ # on s.main_category_id = m.id \ # where s.platform = '%s' and m.prefix = '%s'" % (param['platform'],param['game']) # ip_list = self.exec_sql(s_sql) # # return ip_list # except Exception as exc: # print exc # #异常回滚 # self.center_mysql.rollback() def get_log_ip(self): ''' 获取所有log的IP ''' ip_list = {} sql = "select t1.*,t2.prefix as platform from \ (select m.prefix as gamename,a.public_ip as source_ip,a.platform_id from assets a inner join main_category m \ on a.main_category_id = m.id \ where a.is_del = 0 and (a.hostname like '%%%%web_balance%%%%' or a.hostname like '%%%%web_admin%%%%') ) t1 \ left join platform t2 on t1.platform_id = t2.id \ where t1.source_ip is not null group by source_ip\ union all \ select t1.*,t2.prefix as platform from \ (select m.prefix as gamename,a.public_ip as source_ip,a.platform_id from assets a inner join main_category m \ on a.main_category_id = m.id \ where a.is_del = 0 and a.hostname like '%%%%gameserver%%%%' ) t1 \ left join platform t2 on t1.platform_id = t2.id \ where t1.source_ip is not null group by source_ip" result = self.center_mysql.query(sql) for info in result: if info['gamename'] is None or info['platform'] is None: continue if info['gamename'].encode('utf8') not in ip_list: ip_list[info['gamename'].encode('utf8')] = {} if info['platform'].encode('utf8') not in ip_list[ info['gamename'].encode('utf8')]: ip_list[info['gamename'].encode('utf8')][ info['platform'].encode('utf8')] = [] ip_list[info['gamename'].encode('utf8')][info['platform'].encode( 'utf8')].append(info['source_ip'].encode('utf8')) return ip_list def get_snap_ip(self): ''' 获取所有快照的IP ''' ip_list = {} s_sql = "select f.prefix as platform,s.prefix as source_ip,m.prefix as gamename from sub_category s inner join main_category m \ on s.main_category_id = m.id \ left join platform f on f.id = s.platform_id" result = self.center_mysql.query(s_sql) for info in result: if info['gamename'] is None or info['platform'] is None: continue if info['gamename'].encode('utf8') not in ip_list: ip_list[info['gamename'].encode('utf8')] = {} if info['platform'].encode('utf8') not in ip_list[ info['gamename'].encode('utf8')]: ip_list[info['gamename'].encode('utf8')][ info['platform'].encode('utf8')] = [] ip_list[info['gamename'].encode('utf8')][info['platform'].encode( 'utf8')].append(info['source_ip'].encode('utf8')) # print ip_list return ip_list def get_ip_list(self): ''' 整理IP列表 ''' try: ip_list = {} ip_list['basic'] = ['120.26.1.250'] ip_list['mostsdk'] = ['120.26.13.150'] ip_list['wanpay'] = ['112.124.116.44'] log_list = self.get_log_ip() ip_list['log'] = log_list snap_list = self.get_snap_ip() ip_list['snap'] = snap_list return ip_list except Exception, e: print e print "异常" #异常回滚 self.center_mysql.rollback()
def run_task(self, task_param): mysql = Custom_MySQL(using='etl_manage') hive = Custom_Hive(using='ares_dw') redis = Custom_Redis(using='etl_manage') mysql.begin() where = {'id': int(task_param['id'])} try: log_date = datetime.datetime.strptime(task_param['log_date'], '%Y%m%d').strftime("%Y-%m-%d") log_date_1 = (datetime.datetime.strptime(task_param['log_date'], '%Y%m%d')-datetime.timedelta(days=1)).strftime("%Y-%m-%d") dir_param = {'game': task_param['game'], 'platform': task_param['platform'], 'log_date': task_param['log_date'], 'log_name': task_param['log_name']} filename_dict = {'log_name': task_param['log_name'], 'log_time': task_param['log_time']} index_dict = {'db_name': task_param['db_name'], 'table_name': task_param['table_name'], 'platform': task_param['platform'], 'log_date': log_date} partition = {'platform': task_param['platform'], 'log_date': log_date} index_dict_1 = {'db_name': task_param['db_name'], 'table_name': task_param['table_name'], 'platform': task_param['platform'], 'log_date': log_date_1} partition_1 = {'platform': task_param['platform'], 'log_date': log_date_1} log_dir = "/%(game)s/%(platform)s/%(log_date)s/%(log_name)s" % dir_param flag = task_param['flag'] file_name = "%(log_name)s" % filename_dict file_name_txt = "%(log_name)s_%(log_time)s_result.txt" % filename_dict file_name_lzo = "%(log_name)s_%(log_time)s_result.txt.lzo" % filename_dict file_name_row = "%(log_name)s_%(log_time)s_result.txt.row" % filename_dict index_dir_name = "%(db_name)s.db/%(table_name)s/plat_form=%(platform)s/log_date=%(log_date)s/" % index_dict partition_name = "plat_form='%(platform)s',log_date='%(log_date)s'" % partition project_path = os.getcwd() log_time = task_param['log_time'] do_rate = task_param['do_rate'] #if flag == "snap" or (do_rate == "1day" and flag == "log") if flag == "snap": index_dir_name = "%(db_name)s.db/%(table_name)s/plat_form=%(platform)s/log_date=%(log_date)s/" % index_dict_1 partition_name = "plat_form='%(platform)s',log_date='%(log_date)s'" % partition_1 #从redis中,获取当前数据对应存储到哪块磁盘 if redis.get("disk_xml") is None: disk_tmp = open('/data/etl_manage/conf/disk_game.xml', 'r') redis.set("disk_xml", str(disk_tmp.read())) disk_list = str(redis.get("disk_xml")) root = ET.fromstring(disk_list) disk = "" for gameinfo in root.findall('game'): if gameinfo.get('name') == task_param['game']: disk = gameinfo.get('disk') continue #local_log_dir = '/disk1/tmp_data'+log_dir local_log_dir = '/'+disk+'/data'+log_dir logger.info('local_log_dir: {0}'.format(local_log_dir)) #判断目录是否存在 if os.path.exists('%s' % local_log_dir): #排除同名文件存在的可能,同时为修复执行提供方便 if os.path.exists('%s/%s' % (local_log_dir, file_name_txt)): cmd_remove = 'rm -f %s/%s*' % (local_log_dir, file_name_txt) logger.info('remove history file: {0}'.format(cmd_remove)) remove_result = cmd.run(cmd_remove) if remove_result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove)) datas = {'load_status': 1} mysql.update('file2dw_log', ' id = %(id)d' % where, **datas) mysql.commit() cmd_merge = '/bin/bash %s/mergefile2dw/merge_data.sh %s %s %s %s %s' \ '' % (project_path, local_log_dir, local_log_dir, file_name, do_rate, log_time) logger.info('merge data: {0}'.format(cmd_merge)) merge_result = cmd.run(cmd_merge) logger.info('merge data result {0}'.format(merge_result['output'])) if merge_result['status'] == 0: #读取总条数 row = open('%s/%s' % (local_log_dir, file_name_row)).read() ''' 合并数据完成:2 ''' datas = {'load_status': 2, 'row_num': int(row)} mysql.update('file2dw_log', ' id = %(id)d' % where, **datas) mysql.commit() ''' 压缩数据 ''' cmd_compress = '/bin/bash %s/mergefile2dw/compress_data.sh %s %s' % (project_path, local_log_dir, file_name_txt) logger.info('compress data: {0}'.format(cmd_compress)) compress_result = cmd.run(cmd_compress) if compress_result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (compress_result['status'], compress_result['output'], cmd_compress)) else: ''' 压缩数据完成:3 ''' datas = {'load_status': 3} mysql.update('file2dw_log', ' id = %(id)d' % where, **datas) mysql.commit() #执行load之前,删除同名文件,防止同一个文件出现两次的可能 cmd_remove = '/bin/bash %s/mergefile2dw/remove_damaged_file.sh %s %s' % (project_path, index_dir_name, file_name_lzo) logger.info('remove damaged files: {0}'.format(cmd_remove)) remove_result = cmd.run(cmd_remove) if remove_result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove)) ''' 文件加载到hive中 ''' load_sql = task_param['load_sql'] load_sql = load_sql.replace('{dir_path}', local_log_dir+"/"+file_name_lzo) load_sql = load_sql.replace('{table_name}', task_param['table_name']) load_sql = load_sql.replace('{partition_name}', '%s' % partition_name) load_sql = load_sql.replace('{db_name}', task_param['db_name']) logger.info('hive load SQL: {0}'.format(load_sql)) result = hive.load(load_sql) logger.info('hive load result {0}'.format(result['output'])) if result['status'] == 0: ''' 将任务标识为加载文件完成:4 ''' datas = {'load_status': 4} mysql.update('file2dw_log', ' id = %(id)d' % where, **datas) mysql.commit() ''' 建立索引,否则lzo将不支持split ''' #print index_dir_name cmd_index = '/bin/bash %s/mergefile2dw/create_lzo_indexer.sh %s %s' % (project_path, index_dir_name, file_name_lzo) logger.info('create lzo index: {0}'.format(cmd_index)) index_result = cmd.run(cmd_index) if index_result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (index_result['status'], index_result['output'], cmd_index)) else: if "create index success" in index_result['output']: ''' 将任务标识为建立lzo索引完成:5 ''' datas = {'load_status': 5} mysql.update('file2dw_log', ' id = %(id)d' % where, **datas) else: ''' 如果load数据失败,则删除半途出现错误的文件,防止hive查询的时候报错 ''' cmd_remove = '/bin/bash %s/mergefile2dw/remove_damaged_file.sh %s %s' % (project_path, index_dir_name, file_name_lzo) logger.info('remove damaged files: {0}'.format(cmd_remove)) remove_result = cmd.run(cmd_remove) if remove_result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove)) else: logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], load_sql)) else: ''' 合并数据失败 ''' datas = {'load_status': 0} mysql.update('file2dw_log', ' id = %(id)d' % where, **datas) logger.error('Error Code %s : %s Cmd: %s' % (merge_result['status'], merge_result['output'], merge_result)) ''' 执行完毕,模拟从队列中清楚任务:0 ''' datas = {'in_queue': 0} update_result = mysql.update('file2dw_log', ' id = %(id)d' % where, **datas) # 如果更新失败,则再调用一次,如果还是失败,则等待自动修复机制,但这种概率很小了 if update_result != 1: mysql.update('file2dw_log', ' id = %(id)d' % where, **datas) mysql.commit() mysql.close() return True except Exception as exc: logger.error('mergefile2dw error: %s' % exc) mysql.rollback() datas = {'in_queue': 0, 'load_status': 0} mysql.update('file2dw_log', ' id = %(id)d' % where, **datas) mysql.commit() mysql.close() raise self.retry(exc=exc, countdown=30)
def run_task(self, task_param): redis = Custom_Redis(using='etl_manage') mysql = Custom_MySQL(using='hadoops2') mysql_etl = Custom_MySQL(using='etl_manage') mysql.begin() mysql_etl.begin() where = {'id': int(task_param['id'])} datas = {'load_status': 0} try: ''' 业务代码块放下方 ''' dir_param = {'game': task_param['game'], 'platform': task_param['platform'], 'log_date': task_param['log_date'], 'log_name': task_param['log_name']} filename_dict = {'log_name': task_param['log_name'], 'log_time': task_param['log_time'], 'source_ip': task_param['source_ip']} prefix_sql = task_param['prefix_sql'] post_sql = task_param['post_sql'] log_dir = "/%(game)s/%(platform)s/%(log_date)s/%(log_name)s/" % dir_param file_name = "%(log_name)s_%(source_ip)s_%(log_time)s.txt" % filename_dict #从redis中,获取当前数据对应存储到哪块磁盘 if redis.get("disk_xml") is None: disk_tmp = open('/data/etl_manage/conf/disk_game.xml', 'r') redis.set("disk_xml", str(disk_tmp.read())) disk_list = str(redis.get("disk_xml")) root = ET.fromstring(disk_list) disk = "" for gameinfo in root.findall('game'): if gameinfo.get('name') == task_param['game']: disk = gameinfo.get('disk') continue #local_log_dir = '/disk1/tmp_data'+log_dir local_log_dir = '/'+disk+'/data'+log_dir #判断要录入的文件是否存在,如果存在则执行,否则不执行 if os.path.exists('%s%s' % (local_log_dir, file_name)): ''' 将任务标识为开始执行:1 ''' datas['load_status'] = 1 mysql_etl.update('file2mysql_log', ' id = %(id)d' % where, **datas) mysql_etl.commit() logger.info('start load data') #执行前置sql if prefix_sql is not None and prefix_sql != '': mysql.query(prefix_sql) ''' 执行load数据进mysql ''' load_sql = task_param['load_sql'] load_sql = load_sql.replace('{dir_path}', local_log_dir+file_name) load_sql = load_sql.replace('{table_name}', task_param['table_name']) load_sql = load_sql.replace('{db_name}', task_param['db_name']) result = mysql.load(load_sql) logger.info('load data to mysql: {0}'.format(result['output'])) #判断录入mysql是否成功 if result['status'] == 0: #执行后置sql if post_sql is not None and post_sql != '': post_sql = post_sql.replace('{table_name}', task_param['table_name']) post_sql = post_sql.replace('{db_name}', task_param['db_name']) mysql.query(post_sql) ''' 将任务标识为录入mysql完成:3 ''' datas['load_status'] = 3 else: logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], load_sql)) ''' 录入mysql失败,将任务标示为未执行:0 ''' datas['load_status'] = 0 ''' 将任务标示为:(模拟) 已从任务队列中移除 ''' datas['in_queue'] = 0 update_result = mysql_etl.update('file2mysql_log', ' id = %(id)d' % where, **datas) # 如果更新失败,则再调用一次,如果还是失败,则等待自动修复机制,但这种概率很小了 if update_result != 1: mysql_etl.update('file2mysql_log', ' id = %(id)d' % where, **datas) mysql_etl.commit() mysql.commit() mysql_etl.close() mysql.close() return True except Exception as exc: logger.error('file2mysql error: %s' % exc) mysql_etl.rollback() mysql.rollback() datas = {'in_queue': 0, 'load_status': 0} mysql_etl.update('file2mysql_log', ' id = %(id)d' % where, **datas) mysql_etl.commit() mysql_etl.close() mysql.close() raise self.retry(exc=exc, countdown=60)
def run_task(self, task_param): mysql = Custom_MySQL(using='etl_manage') hive = Custom_Hive(using='ares_dw') redis = Custom_Redis(using='etl_manage') mysql.begin() where = {'id': int(task_param['id'])} try: log_date = datetime.datetime.strptime(task_param['log_date'], '%Y%m%d').strftime("%Y-%m-%d") log_date_1 = ( datetime.datetime.strptime(task_param['log_date'], '%Y%m%d') - datetime.timedelta(days=1)).strftime("%Y-%m-%d") dir_param = { 'game': task_param['game'], 'platform': task_param['platform'], 'log_date': task_param['log_date'], 'log_name': task_param['log_name'] } filename_dict = { 'log_name': task_param['log_name'], 'log_time': task_param['log_time'] } index_dict = { 'db_name': task_param['db_name'], 'table_name': task_param['table_name'], 'platform': task_param['platform'], 'log_date': log_date } partition = {'platform': task_param['platform'], 'log_date': log_date} index_dict_1 = { 'db_name': task_param['db_name'], 'table_name': task_param['table_name'], 'platform': task_param['platform'], 'log_date': log_date_1 } partition_1 = { 'platform': task_param['platform'], 'log_date': log_date_1 } log_dir = "/%(game)s/%(platform)s/%(log_date)s/%(log_name)s" % dir_param flag = task_param['flag'] file_name = "%(log_name)s" % filename_dict file_name_txt = "%(log_name)s_%(log_time)s_result.txt" % filename_dict file_name_lzo = "%(log_name)s_%(log_time)s_result.txt.lzo" % filename_dict file_name_row = "%(log_name)s_%(log_time)s_result.txt.row" % filename_dict index_dir_name = "%(db_name)s.db/%(table_name)s/plat_form=%(platform)s/log_date=%(log_date)s/" % index_dict partition_name = "plat_form='%(platform)s',log_date='%(log_date)s'" % partition project_path = os.getcwd() log_time = task_param['log_time'] do_rate = task_param['do_rate'] #if flag == "snap" or (do_rate == "1day" and flag == "log") if flag == "snap": index_dir_name = "%(db_name)s.db/%(table_name)s/plat_form=%(platform)s/log_date=%(log_date)s/" % index_dict_1 partition_name = "plat_form='%(platform)s',log_date='%(log_date)s'" % partition_1 #从redis中,获取当前数据对应存储到哪块磁盘 if redis.get("disk_xml") is None: disk_tmp = open('/data/etl_manage/conf/disk_game.xml', 'r') redis.set("disk_xml", str(disk_tmp.read())) disk_list = str(redis.get("disk_xml")) root = ET.fromstring(disk_list) disk = "" for gameinfo in root.findall('game'): if gameinfo.get('name') == task_param['game']: disk = gameinfo.get('disk') continue #local_log_dir = '/disk1/tmp_data'+log_dir local_log_dir = '/' + disk + '/data' + log_dir logger.info('local_log_dir: {0}'.format(local_log_dir)) #判断目录是否存在 if os.path.exists('%s' % local_log_dir): #排除同名文件存在的可能,同时为修复执行提供方便 if os.path.exists('%s/%s' % (local_log_dir, file_name_txt)): cmd_remove = 'rm -f %s/%s*' % (local_log_dir, file_name_txt) logger.info('remove history file: {0}'.format(cmd_remove)) remove_result = cmd.run(cmd_remove) if remove_result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove)) datas = {'load_status': 1} mysql.update('file2dw_log', ' id = %(id)d' % where, **datas) mysql.commit() cmd_merge = '/bin/bash %s/mergefile2dw/merge_data.sh %s %s %s %s %s' \ '' % (project_path, local_log_dir, local_log_dir, file_name, do_rate, log_time) logger.info('merge data: {0}'.format(cmd_merge)) merge_result = cmd.run(cmd_merge) logger.info('merge data result {0}'.format(merge_result['output'])) if merge_result['status'] == 0: #读取总条数 row = open('%s/%s' % (local_log_dir, file_name_row)).read() ''' 合并数据完成:2 ''' datas = {'load_status': 2, 'row_num': int(row)} mysql.update('file2dw_log', ' id = %(id)d' % where, **datas) mysql.commit() ''' 压缩数据 ''' cmd_compress = '/bin/bash %s/mergefile2dw/compress_data.sh %s %s' % ( project_path, local_log_dir, file_name_txt) logger.info('compress data: {0}'.format(cmd_compress)) compress_result = cmd.run(cmd_compress) if compress_result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (compress_result['status'], compress_result['output'], cmd_compress)) else: ''' 压缩数据完成:3 ''' datas = {'load_status': 3} mysql.update('file2dw_log', ' id = %(id)d' % where, **datas) mysql.commit() #执行load之前,删除同名文件,防止同一个文件出现两次的可能 cmd_remove = '/bin/bash %s/mergefile2dw/remove_damaged_file.sh %s %s' % ( project_path, index_dir_name, file_name_lzo) logger.info('remove damaged files: {0}'.format(cmd_remove)) remove_result = cmd.run(cmd_remove) if remove_result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove)) ''' 文件加载到hive中 ''' load_sql = task_param['load_sql'] load_sql = load_sql.replace( '{dir_path}', local_log_dir + "/" + file_name_lzo) load_sql = load_sql.replace('{table_name}', task_param['table_name']) load_sql = load_sql.replace('{partition_name}', '%s' % partition_name) load_sql = load_sql.replace('{db_name}', task_param['db_name']) logger.info('hive load SQL: {0}'.format(load_sql)) result = hive.load(load_sql) logger.info('hive load result {0}'.format( result['output'])) if result['status'] == 0: ''' 将任务标识为加载文件完成:4 ''' datas = {'load_status': 4} mysql.update('file2dw_log', ' id = %(id)d' % where, **datas) mysql.commit() ''' 建立索引,否则lzo将不支持split ''' #print index_dir_name cmd_index = '/bin/bash %s/mergefile2dw/create_lzo_indexer.sh %s %s' % ( project_path, index_dir_name, file_name_lzo) logger.info('create lzo index: {0}'.format(cmd_index)) index_result = cmd.run(cmd_index) if index_result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (index_result['status'], index_result['output'], cmd_index)) else: if "create index success" in index_result[ 'output']: ''' 将任务标识为建立lzo索引完成:5 ''' datas = {'load_status': 5} mysql.update('file2dw_log', ' id = %(id)d' % where, **datas) else: ''' 如果load数据失败,则删除半途出现错误的文件,防止hive查询的时候报错 ''' cmd_remove = '/bin/bash %s/mergefile2dw/remove_damaged_file.sh %s %s' % ( project_path, index_dir_name, file_name_lzo) logger.info('remove damaged files: {0}'.format( cmd_remove)) remove_result = cmd.run(cmd_remove) if remove_result['status'] != 0: logger.error( 'Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove)) else: logger.error( 'Error Code %s : %s Cmd: %s' % (result['status'], result['output'], load_sql)) else: ''' 合并数据失败 ''' datas = {'load_status': 0} mysql.update('file2dw_log', ' id = %(id)d' % where, **datas) logger.error('Error Code %s : %s Cmd: %s' % (merge_result['status'], merge_result['output'], merge_result)) ''' 执行完毕,模拟从队列中清楚任务:0 ''' datas = {'in_queue': 0} update_result = mysql.update('file2dw_log', ' id = %(id)d' % where, **datas) # 如果更新失败,则再调用一次,如果还是失败,则等待自动修复机制,但这种概率很小了 if update_result != 1: mysql.update('file2dw_log', ' id = %(id)d' % where, **datas) mysql.commit() mysql.close() return True except Exception as exc: logger.error('mergefile2dw error: %s' % exc) mysql.rollback() datas = {'in_queue': 0, 'load_status': 0} mysql.update('file2dw_log', ' id = %(id)d' % where, **datas) mysql.commit() mysql.close() raise self.retry(exc=exc, countdown=30)
def run_task(self, task_param): mysql = Custom_MySQL(using='hadoops2') mysql_etl = Custom_MySQL(using='etl_manage') redis = Custom_Redis(using='etl_task') mysql.begin() mysql_etl.begin() datas = {'status': 0} where = {'id': int(task_param['id'])} task_key_exc = "" try: hive = Custom_Hive(using='ares_dw') game = task_param['game'] platform = task_param['platform'] table_name = task_param['table_name'] log_date = task_param['log_date'] prefix_sql = task_param['prefix_sql'] exec_sql = task_param['exec_sql'] post_sql = task_param['post_sql'] date_cycle = task_param['date_cycle'] random_str = str(random.randint(0, 999999999)) stimes = str(int(time.time())) task_date = task_param['task_date'] task_name = task_param['task_name'] do_rate = task_param['do_rate'] #task_key_tmp = str(game)+str(platform)+str(task_name)+str(date_cycle)+str(do_rate)+str(log_date)+str(task_date) task_key_tmp = str(task_name)+str(date_cycle)+str(do_rate)+str(log_date)+str(task_date) task_key_md5 = hashlib.md5() task_key_md5.update(task_key_tmp) task_key_md5_result = task_key_md5.hexdigest() task_key = "dm2report_new_"+str(game)+"_"+str(platform)+"_"+str(task_key_md5_result) task_key_exc = task_key ''' celery 本身的机制存在漏洞,会将一个已经完成任务再次分配给其他的worker,致使同一个任务执行多次 为防止此种现象,在任务开始执行时,将任务的“唯一标示”写入redis中,标注已执行 ''' #如果task_key is None, 则表示该条任务没有执行过,正常执行即可 #如果task_key = 0, 则表示该条任务上次执行失败,允许重复执行 if redis.get(task_key) == "0" or redis.get(task_key) is None: tmp_file_dir = "/tmp/tmp/%s/%s/%s" % (game, platform, log_date) #创建本地目录 if not os.path.exists(tmp_file_dir): os.makedirs(tmp_file_dir) tmp_file = "%s/%s_%s_%s_%s.txt" % (tmp_file_dir, table_name, date_cycle, random_str, stimes) hql_conf = "SET hive.support.concurrency=false;" \ "SET hive.exec.compress.output=true;" \ "SET mapred.output.compression.codec=com.hadoop.compression.lzo.LzopCodec; " ''' 将任务标示为开始执行:1 ''' datas['status'] = 1 datas['start_time'] = str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')) mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where, **datas) mysql_etl.commit() #在redis标注 任务正在执行 redis.set(task_key, 1) #执行前置sql,将数据临时写入本地,用于mysql load数据 if prefix_sql is not None: result = mysql.delete_by_sql(prefix_sql) logger.info('exec prefix_sql: delete old data {0}'.format(result['output'])) if result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], prefix_sql)) ''' 执行失败,将其状态标为未执行:0 ''' datas['status'] = 0 #在redis标注 任务未开始执行 redis.set(task_key, 0) else: ''' 将任务标示为删除临时文件完成:2 ''' datas['status'] = 2 datas.pop('start_time') mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where, **datas) mysql_etl.commit() ''' 开始执行hive ql,将数据dump到本地 ''' result = hive.dump(hql_conf+exec_sql, tmp_file) logger.info('exec exec_sql: dump data {0}'.format(result['output'])) if result['status'] != 0 or False == os.path.exists('%s' % tmp_file): logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], exec_sql)) #在redis标注 任务未开始执行 redis.set(task_key, 0) else: ''' 将任务标示为dump hive数据完成:3 ''' datas['status'] = 3 datas['tmp_file_name'] = tmp_file mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where, **datas) mysql_etl.commit() #执行后置sql if post_sql is not None: post_sql = post_sql.replace('{dir_path}', tmp_file) post_sql = post_sql.replace('{table_name}', task_param['table_name']) post_sql = post_sql.replace('{db_name}', task_param['db_name']) result = mysql.load(post_sql) logger.info('exec post_sql: load data to hdfs {0}'.format(result['output'])) if result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], post_sql)) #在redis标注 任务未开始执行 redis.set(task_key, 0) else: ''' 将任务标识为录入mysql完成:4 ''' datas['status'] = 4 datas['end_time'] = str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')) datas.pop('tmp_file_name') #在redis标注 任务已成功执行 redis.set(task_key, 2) else: logger.error('prefix_sql is null') datas['status'] = 0 #在redis标注 任务未开始执行 redis.set(task_key, 0) #如果task_key=2, 则标示该条任务已经运行成功 elif redis.get(task_key) == "2": datas['status'] = 4 #该条任务正在运行中 else: return True ''' 将任务标示为:(模拟) 已从任务队列中移除 ''' datas['in_queue'] = 0 update_result = mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where, **datas) # 如果数据库更新失败,再调用一次。 如果还是失败,等待自动修复机制,但这样的概率应该很小了。 if update_result != 1: mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where, **datas) mysql_etl.commit() mysql.commit() mysql_etl.close() mysql.close() return True except Exception as exc: logger.error('dm2report error: %s' % exc) mysql_etl.rollback() mysql.rollback() redis.set(task_key_exc, 0) datas = {'in_queue': 0, 'status': 0} mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where, **datas) mysql_etl.commit() mysql_etl.close() mysql.close() raise self.retry(exc=exc, countdown=60)
'exec_num': exec_num, 'in_queue': in_queue, 'retry_num': retry_num } update_result = mysql.update('file2mysql_log', ' id = %(id)d' % where, **datas) # 为防止修改数据库时出现异常,调整为:确认修改字段状态成功后,再加入队列 if update_result == 1: run_task.apply_async((result, ), queue='file2mysql') else: print "update data{'exec_num', 'in_queue', 'retry_num'} error" else: datas = {'load_status': -1} mysql.update('file2mysql_log', ' id = %(id)d' % where, **datas) else: datas = {'load_status': -2} mysql.update('file2mysql_log', ' id = %(id)d' % where, **datas) mysql.commit() #mysql.close() except Exception as exc: #回滚 print(exc) mysql.rollback() print "db rollback success" mysql.close()
def run_task(self, task_param): mysql = Custom_MySQL(using='etl_manage') mysql.begin() datas = {'etl_status': 0} where = {'id': int(task_param['id'])} try: log_param = {'game': task_param['game'], 'platform': task_param['platform'], 'log_date': task_param['log_date'], 'log_name': task_param['log_name'], 'log_dir': task_param['log_dir'], 'col_num': task_param['col_num']} log_name_param = {'log_name': task_param['log_name'], 'source_ip': task_param['source_ip'], 'log_time': task_param['log_time']} do_rate = task_param['do_rate'] flag = task_param['flag'] log_dir = '%(log_dir)s/%(game)s/%(platform)s/%(log_date)s/%(log_name)s' % log_param log_name = '%(log_name)s_%(source_ip)s_%(log_time)s' % log_name_param log_name_notime = '%(log_name)s_%(source_ip)s' % log_name_param col_num = task_param['col_num'] project_path = os.getcwd() #判断如果有md5文件和数据文件同时存在,则开始执行 if (do_rate == "1day" and flag == "log") or (os.path.exists('%s/%s.log.md5' % (log_dir, log_name)) is True and os.path.exists('%s/%s.log' % (log_dir, log_name)) is True): #排除同名文件存在的可能,同时为修复执行提供方便 if os.path.exists('%s/%s.txt' % (log_dir, log_name)): cmd_remove = 'rm -f %s/%s.txt*' % (log_dir, log_name) logger.info('remove history file: {0}'.format(cmd_remove)) remove_result = cmd.run(cmd_remove) if remove_result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove)) ''' 将任务标识为开始执行:1 ''' datas['etl_status'] = 1 mysql.update('etl_data_log', ' id = %(id)d' % where, **datas) mysql.commit() ''' 校验数据 ''' #如果是“天”频次的日志数据,则特殊处理,直接将同一ip的合成一个文件,同时执行校验 if do_rate == "1day" and flag == "log": cmd_merge = '/bin/bash %s/etl_data/merge_data.sh %s %s %s' % (project_path, log_dir, log_name_notime, col_num) logger.info('check data: {0}'.format(cmd_merge)) merge_result = cmd.run(cmd_merge) else: cmd_merge = '/bin/bash %s/etl_data/check_data.sh %s %s %s' % (project_path, log_dir, log_name, col_num) logger.info('check data: {0}'.format(cmd_merge)) merge_result = cmd.run(cmd_merge) if merge_result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (merge_result['status'], merge_result['output'], cmd_merge)) datas['etl_status'] = 0 else: ''' 读取校验格式后的文件总条数 ''' row = open('%s/%s.txt.row' % (log_dir, log_name)).read() ''' 将文件总条数写入数据库,并将任务标识为为校验已完成:2 ''' datas['etl_status'] = 2 datas['row_num'] = int(row) #datas = {'etl_status': 2, 'row_num': int(row)} mysql.update('etl_data_log', ' id = %(id)d' % where, **datas) mysql.commit() ''' 压缩数据 ''' cmd_compress = '/bin/bash %s/etl_data/compress_data.sh %s %s' % (project_path, log_dir, log_name) logger.info('compress data: {0}'.format(cmd_compress)) compress_result = cmd.run(cmd_compress) if compress_result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (compress_result['status'], compress_result['output'], cmd_compress)) datas['etl_status'] = 0 else: ''' 将任务标识为压缩完成:4 ''' datas['etl_status'] = 4 datas.pop('row_num') mysql.update('etl_data_log', ' id = %(id)d' % where, **datas) mysql.commit() ''' 生成MD5文件 ''' cmd_md5 = '/bin/bash %s/etl_data/md5_data.sh %s %s' % (project_path, log_dir, log_name) logger.info('md5 data: {0}'.format(cmd_md5)) md5_result = cmd.run(cmd_md5) if md5_result['status'] != 0: logger.error('Error Code %s : %s Cmd: %s' % (md5_result['status'], md5_result['output'], cmd_md5)) datas['etl_status'] = 0 else: ''' 将任务标识为生成MD5完成(即为校验、合并、压缩均已完成):6 ''' datas['etl_status'] = 6 ''' 执行完毕,模拟从队列中清楚任务:0 ''' datas['in_etl_queue'] = 0 update_result = mysql.update('etl_data_log', ' id = %(id)d' % where, **datas) # 如果更新失败,则再调用一次,如果还是失败,则等待自动修复机制,但这种概率很小了 if update_result != 1: mysql.update('etl_data_log', ' id = %(id)d' % where, **datas) mysql.commit() mysql.close() return True except Exception as exc: logger.error('etl_data error: %s' % exc) mysql.rollback() datas = {'in_etl_queue': 0, 'etl_status': 0} mysql.update('etl_data_log', ' id = %(id)d' % where, **datas) mysql.commit() mysql.close() raise self.retry(exc=exc, countdown=30)