def run_task(self, task_param):
mysql = Custom_MySQL(using='etl_manage')
mysql.begin()
try:
'''
业务代码块放下方
'''
dir_param ={'game':task_param['game'],
'platform':task_param['platform'],
'log_date':task_param['log_date'],
'log_name':task_param['log_name']}
filename_dict = {'log_name':task_param['log_name'],'log_time':task_param['log_time']}
'''
游戏＼平台＼日期＼业务日志名＼日志或者md5文件
'''
log_dir = "/%(game)s/%(platform)s/%(log_date)s/%(log_name)s/" % dir_param
lzo_file_name = "%(log_name)s_%(log_time)s.txt"% filename_dict
local_log_dir = '/tmp'+log_dir
dump_sql = task_param['dump_sql']
dump_sql = dump_sql.replace('{table_name}',task_param['table_name'])
dump_sql = dump_sql.replace('{partition_name}',task_param['partition_name'])
dump_sql = dump_sql.replace('{db_name}',task_param['db_name'])
print(dump_sql)
result = mysql.dump(sql,local_log_dir+lzo_file_name)
#print(result)
'''
将任务标识为加载文件完成：2
'''
datas = {'load_status':2}
where = {}
where['id'] = int(task_param['id'])
mysql.update('etl_data_log',
' id = %(id)d' % where,
**datas)
mysql.commit()
mysql.close()
return True
except Exception as exc:
print (exc)
mysql.rollback()
raise self.retry(exc=exc, countdown=60)
def run_task(self, task_param):
mysql = Custom_MySQL(using='etl_manage')
mysql.begin()
try:
'''
业务代码块放下方
'''
dir_param = {
'game': task_param['game'],
'platform': task_param['platform'],
'log_date': task_param['log_date'],
'log_name': task_param['log_name']
}
filename_dict = {
'log_name': task_param['log_name'],
'log_time': task_param['log_time']
}
'''
游戏＼平台＼日期＼业务日志名＼日志或者md5文件
'''
log_dir = "/%(game)s/%(platform)s/%(log_date)s/%(log_name)s/" % dir_param
lzo_file_name = "%(log_name)s_%(log_time)s.txt" % filename_dict
local_log_dir = '/tmp' + log_dir
dump_sql = task_param['dump_sql']
dump_sql = dump_sql.replace('{table_name}', task_param['table_name'])
dump_sql = dump_sql.replace('{partition_name}',
task_param['partition_name'])
dump_sql = dump_sql.replace('{db_name}', task_param['db_name'])
print(dump_sql)
result = mysql.dump(sql, local_log_dir + lzo_file_name)
#print(result)
'''
将任务标识为加载文件完成：2
'''
datas = {'load_status': 2}
where = {}
where['id'] = int(task_param['id'])
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
return True
except Exception as exc:
print(exc)
mysql.rollback()
raise self.retry(exc=exc, countdown=60)
def query_mysql(sql):
mysql = Custom_MySQL(using='etl_manage')
try:
mysql.begin()
result = mysql.query(sql)
mysql.commit()
mysql.close()
return result
except Exception as exc:
#回滚
mysql.rollback()
print(exc)
mysql.close()
def query_mysql(sql):
mysql = Custom_MySQL(using='etl_manage')
try:
mysql.begin()
result = mysql.query(sql)
mysql.commit()
mysql.close()
return result
except Exception as exc:
#回滚
mysql.rollback()
print(exc)
mysql.close()
def run_task(self, task_param):
mysql = Custom_MySQL(using='hadoops2')
mysql_etl = Custom_MySQL(using='etl_manage')
redis = Custom_Redis(using='etl_task')
mysql.begin()
mysql_etl.begin()
datas = {'status': 0}
where = {'id': int(task_param['id'])}
task_key_exc = ""
try:
hive = Custom_Hive(using='ares_dw')
game = task_param['game']
platform = task_param['platform']
table_name = task_param['table_name']
log_date = task_param['log_date']
prefix_sql = task_param['prefix_sql']
exec_sql = task_param['exec_sql']
post_sql = task_param['post_sql']
date_cycle = task_param['date_cycle']
random_str = str(random.randint(0, 999999999))
stimes = str(int(time.time()))
task_date = task_param['task_date']
task_name = task_param['task_name']
do_rate = task_param['do_rate']
#task_key_tmp = str(game)+str(platform)+str(task_name)+str(date_cycle)+str(do_rate)+str(log_date)+str(task_date)
task_key_tmp = str(task_name) + str(date_cycle) + str(do_rate) + str(
log_date) + str(task_date)
task_key_md5 = hashlib.md5()
task_key_md5.update(task_key_tmp)
task_key_md5_result = task_key_md5.hexdigest()
task_key = "dm2report_new_" + str(game) + "_" + str(
platform) + "_" + str(task_key_md5_result)
task_key_exc = task_key
'''
celery 本身的机制存在漏洞，会将一个已经完成任务再次分配给其他的worker，致使同一个任务执行多次
为防止此种现象，在任务开始执行时，将任务的“唯一标示”写入redis中，标注已执行
'''
#如果task_key is None, 则表示该条任务没有执行过，正常执行即可
#如果task_key = 0, 则表示该条任务上次执行失败，允许重复执行
if redis.get(task_key) == "0" or redis.get(task_key) is None:
tmp_file_dir = "/tmp/tmp/%s/%s/%s" % (game, platform, log_date)
#创建本地目录
if not os.path.exists(tmp_file_dir):
os.makedirs(tmp_file_dir)
tmp_file = "%s/%s_%s_%s_%s.txt" % (tmp_file_dir, table_name,
date_cycle, random_str, stimes)
hql_conf = "SET hive.support.concurrency=false;" \
"SET hive.exec.compress.output=true;" \
"SET mapred.output.compression.codec=com.hadoop.compression.lzo.LzopCodec; "
'''
将任务标示为开始执行：1
'''
datas['status'] = 1
datas['start_time'] = str(
datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where,
**datas)
mysql_etl.commit()
#在redis标注 任务正在执行
redis.set(task_key, 1)
#执行前置sql，将数据临时写入本地，用于mysql load数据
if prefix_sql is not None:
result = mysql.delete_by_sql(prefix_sql)
logger.info('exec prefix_sql: delete old data {0}'.format(
result['output']))
if result['status'] != 0:
logger.error(
'Error Code %s : %s Cmd: %s' %
(result['status'], result['output'], prefix_sql))
'''
执行失败，将其状态标为未执行：0
'''
datas['status'] = 0
#在redis标注 任务未开始执行
redis.set(task_key, 0)
else:
'''
将任务标示为删除临时文件完成：2
'''
datas['status'] = 2
datas.pop('start_time')
mysql_etl.update('dm2report_new_log',
' id = %(id)d' % where, **datas)
mysql_etl.commit()
'''
开始执行hive ql,将数据dump到本地
'''
result = hive.dump(hql_conf + exec_sql, tmp_file)
logger.info('exec exec_sql: dump data {0}'.format(
result['output']))
if result['status'] != 0 or False == os.path.exists(
'%s' % tmp_file):
logger.error(
'Error Code %s : %s Cmd: %s' %
(result['status'], result['output'], exec_sql))
#在redis标注 任务未开始执行
redis.set(task_key, 0)
else:
'''
将任务标示为dump hive数据完成：3
'''
datas['status'] = 3
datas['tmp_file_name'] = tmp_file
mysql_etl.update('dm2report_new_log',
' id = %(id)d' % where, **datas)
mysql_etl.commit()
#执行后置sql
if post_sql is not None:
post_sql = post_sql.replace('{dir_path}', tmp_file)
post_sql = post_sql.replace(
'{table_name}', task_param['table_name'])
post_sql = post_sql.replace(
'{db_name}', task_param['db_name'])
result = mysql.load(post_sql)
logger.info(
'exec post_sql: load data to hdfs {0}'.format(
result['output']))
if result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' %
(result['status'],
result['output'], post_sql))
#在redis标注 任务未开始执行
redis.set(task_key, 0)
else:
'''
将任务标识为录入mysql完成：4
'''
datas['status'] = 4
datas['end_time'] = str(
datetime.datetime.now().strftime(
'%Y-%m-%d %H:%M:%S'))
datas.pop('tmp_file_name')
#在redis标注 任务已成功执行
redis.set(task_key, 2)
else:
logger.error('prefix_sql is null')
datas['status'] = 0
#在redis标注 任务未开始执行
redis.set(task_key, 0)
#如果task_key=2, 则标示该条任务已经运行成功
elif redis.get(task_key) == "2":
datas['status'] = 4
#该条任务正在运行中
else:
return True
'''
将任务标示为：(模拟) 已从任务队列中移除
'''
datas['in_queue'] = 0
update_result = mysql_etl.update('dm2report_new_log',
' id = %(id)d' % where, **datas)
# 如果数据库更新失败，再调用一次。 如果还是失败，等待自动修复机制，但这样的概率应该很小了。
if update_result != 1:
mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where,
**datas)
mysql_etl.commit()
mysql.commit()
mysql_etl.close()
mysql.close()
return True
except Exception as exc:
logger.error('dm2report error: %s' % exc)
mysql_etl.rollback()
mysql.rollback()
redis.set(task_key_exc, 0)
datas = {'in_queue': 0, 'status': 0}
mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
mysql_etl.close()
mysql.close()
raise self.retry(exc=exc, countdown=60)
if file2dw_result['status'] != 0:
print(now+" restart file2dw task error")
file2dw_result_1 = mysql.update_by_sql('update file2dw_log set load_status=0,exec_num=0,in_queue=0 '
'where load_status!=5 and in_queue=1 and log_time <="%s" and task_date="%s"'
'' % (log_time_before_1hour, task_date))
if file2dw_result_1['status'] != 0:
print(now+" restart file2dw_1 task error")
if '0500' < log_time < '1100':
#重置dm2report_status
dm2report_result = mysql.update_by_sql('update dm2report_log set status=0,exec_num=0,in_queue=0 '
'where status=-1 and task_date="%s"' % task_date)
if dm2report_result['status'] != 0:
print(now+" restart dm2report task error")
dm2report_result_1 = mysql.update_by_sql('update dm2report_log set status=0,exec_num=0,in_queue=0 '
'where status!=4 and in_queue=1 and log_time <="%s" and task_date="%s"'
'' % (log_time_before_1hour, task_date))
if dm2report_result_1['status'] != 0:
print(now+" restart dm2report_1 task error")
mysql.commit()
mysql.close()
except Exception as exc:
#回滚
mysql.rollback()
print(exc)
mysql.close()
def run_task(self, task_param):
mysql = Custom_MySQL(using='etl_manage')
mysql.begin()
where = {'id': int(task_param['id'])}
try:
dir_param = {'game': task_param['game'], 'platform': task_param['platform'],
'log_date': task_param['log_date'], 'log_name': task_param['log_name']}
filename_dict = {'log_name': task_param['log_name'], 'log_time': task_param['log_time'], 'source_ip':task_param['source_ip']}
index_dict = {'db_name': task_param['db_name'], 'table_name': task_param['table_name'], 'platform': task_param['platform'],
'log_date': datetime.datetime.strptime(task_param['log_date'], '%Y%m%d').strftime("%Y-%m-%d")}
partition = {'platform': task_param['platform'], 'log_date': datetime.datetime.strptime(task_param['log_date'], '%Y%m%d').strftime("%Y-%m-%d")}
log_dir = "/%(game)s/%(platform)s/%(log_date)s/%(log_name)s/" % dir_param
lzo_file_name = "%(log_name)s_%(source_ip)s_%(log_time)s.txt.lzo" % filename_dict
index_dir_name = "%(db_name)s.db/%(table_name)s/plat_form=%(platform)s/log_date=%(log_date)s/" % index_dict
partition_name = "plat_form='%(platform)s',log_date='%(log_date)s'" % partition
project_path = os.getcwd()
local_log_dir = '/disk1/tmp_data'+log_dir
logger.info('local_log_dir: {0}'.format(local_log_dir))
#判断要录入hive 中的文件知否存在，存在则执行
if os.path.exists('%s%s' % (local_log_dir, lzo_file_name)):
'''
将任务标识为开始执行：1
'''
datas = {'load_status': 1}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
#执行load之前，删除同名文件，防止同一个文件出现两次的可能
cmd_remove = '/bin/bash %s/file2dw/remove_damaged_file.sh %s %s' % (project_path, index_dir_name, lzo_file_name)
logger.info('remove damaged files: {0}'.format(cmd_remove))
remove_result = cmd.run(cmd_remove)
if remove_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove))
'''
文件加载到hive中
'''
hive = Custom_Hive(using='ares_dw')
load_sql = task_param['load_sql']
load_sql = load_sql.replace('{dir_path}', local_log_dir+lzo_file_name)
load_sql = load_sql.replace('{table_name}', task_param['table_name'])
load_sql = load_sql.replace('{partition_name}', '%s' % partition_name)
load_sql = load_sql.replace('{db_name}', task_param['db_name'])
logger.info('hive load SQL: {0}'.format(load_sql))
result = hive.load(load_sql)
logger.info('hive load result {0}'.format(result['output']))
if result['status'] == 0:
'''
将任务标识为加载文件完成：2
'''
datas = {'load_status': 2}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
'''
建立索引，否则lzo将不支持split
'''
#print index_dir_name
cmd_index = '/bin/bash %s/file2dw/create_lzo_indexer.sh %s %s' % (project_path, index_dir_name, lzo_file_name)
logger.info('create lzo index: {0}'.format(cmd_index))
index_result = cmd.run(cmd_index)
if index_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (index_result['status'], index_result['output'], cmd_index))
else:
if "create index success" in index_result['output']:
'''
将任务标识为建立lzo索引完成：3
'''
datas = {'load_status': 3}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
else:
'''
如果load数据失败，则删除半途出现错误的文件，方式hive查询的时候报错
'''
cmd_remove = '/bin/bash %s/file2dw/remove_damaged_file.sh %s %s' % (project_path, index_dir_name, lzo_file_name)
logger.info('remove damaged files: {0}'.format(cmd_remove))
remove_result = cmd.run(cmd_remove)
if remove_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove))
else:
'''
将任务标识为未启动，重新执行：0
'''
datas = {'load_status': 0}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
'''
将任务标示为：(模拟) 已从任务队列中移除
'''
datas = {'in_queue': 0}
update_result = mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
# 如果更新失败，则再调用一次，如果还是失败，则等待自动修复机制，但这种概率很小了
if update_result != 1:
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
return True
except Exception as exc:
print (exc)
mysql.rollback()
datas = {'in_queue': 0}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
raise self.retry(exc=exc, countdown=60)
class CenterApp():
def __init__(self):
self.center_mysql = Custom_MySQL(using='center_app')
# def get_server_list(self):
# '''
# 获取执行任务
# '''
# try:
# ip_list = []
# if param['flag'] == 'basic':
# ip_list.append('120.26.1.250')
#
# if param['flag'] == 'most':
# ip_list.append('120.26.13.150')
#
# if param['flag'] == 'log':
# s_sql = "select t1.* from \
# (select a.public_ip as source_ip,a.platform_id from assets a inner join main_category m \
# on a.main_category_id = m.id \
# where a.is_del = 0 and (a.hostname like '%%%%web_balance%%%%' or a.hostname like '%%%%web_admin%%%%') ) t1 \
# left join platform t2 on t1.platform_id = t2.id \
# where t1.source_ip is not null group by source_ip"
#
# if param['game'] == 'kof':
# s_sql = "select t1.* from \
# (select a.public_ip as source_ip,a.platform_id from assets a inner join main_category m \
# on a.main_category_id = m.id \
# where a.is_del = 0 and a.hostname like '%%%%gameserver%%%%' ) t1 \
# left join platform t2 on t1.platform_id = t2.id \
# where t1.source_ip is not null group by source_ip"
#
# ip_list = self.exec_sql(s_sql)
#
# if param['flag'] == 'snap':
# s_sql = "select s.prefix as source_ip from sub_category s inner join main_category m \
# on s.main_category_id = m.id \
# where s.platform = '%s' and m.prefix = '%s'" % (param['platform'],param['game'])
# ip_list = self.exec_sql(s_sql)
#
# return ip_list
# except Exception as exc:
# print exc
# #异常回滚
# self.center_mysql.rollback()
def get_log_ip(self):
'''
获取所有log的IP
'''
ip_list = {}
sql = "select t1.*,t2.prefix as platform from \
(select m.prefix as gamename,a.public_ip as source_ip,a.platform_id from assets a inner join main_category m \
on a.main_category_id = m.id \
where a.is_del = 0 and (a.hostname like '%%%%web_balance%%%%' or a.hostname like '%%%%web_admin%%%%') ) t1 \
left join platform t2 on t1.platform_id = t2.id \
where t1.source_ip is not null group by source_ip\
union all \
select t1.*,t2.prefix as platform from \
(select m.prefix as gamename,a.public_ip as source_ip,a.platform_id from assets a inner join main_category m \
on a.main_category_id = m.id \
where a.is_del = 0 and a.hostname like '%%%%gameserver%%%%' ) t1 \
left join platform t2 on t1.platform_id = t2.id \
where t1.source_ip is not null group by source_ip"
result = self.center_mysql.query(sql)
for info in result:
if info['gamename'] is None or info['platform'] is None:
continue
if info['gamename'].encode('utf8') not in ip_list:
ip_list[info['gamename'].encode('utf8')] = {}
if info['platform'].encode('utf8') not in ip_list[info['gamename'].encode('utf8')]:
ip_list[info['gamename'].encode('utf8')][info['platform'].encode('utf8')] = []
ip_list[info['gamename'].encode('utf8')][info['platform'].encode('utf8')].append(info['source_ip'].encode('utf8'))
return ip_list
def get_snap_ip(self):
'''
获取所有快照的IP
'''
ip_list = {}
s_sql = "select f.prefix as platform,s.prefix as source_ip,m.prefix as gamename from sub_category s inner join main_category m \
on s.main_category_id = m.id \
left join platform f on f.id = s.platform_id"
result = self.center_mysql.query(s_sql)
for info in result:
if info['gamename'] is None or info['platform'] is None:
continue
if info['gamename'].encode('utf8') not in ip_list:
ip_list[info['gamename'].encode('utf8')] = {}
if info['platform'].encode('utf8') not in ip_list[info['gamename'].encode('utf8')]:
ip_list[info['gamename'].encode('utf8')][info['platform'].encode('utf8')] = []
ip_list[info['gamename'].encode('utf8')][info['platform'].encode('utf8')].append(info['source_ip'].encode('utf8'))
# print ip_list
return ip_list
def get_ip_list(self):
'''
整理IP列表
'''
try:
ip_list = {}
ip_list['basic'] = ['120.26.1.250']
ip_list['mostsdk'] = ['120.26.13.150']
ip_list['wanpay'] = ['112.124.116.44']
log_list = self.get_log_ip();
ip_list['log'] = log_list
snap_list = self.get_snap_ip()
ip_list['snap'] = snap_list
return ip_list
except Exception,e:
print e
print "异常"
#异常回滚
self.center_mysql.rollback()
def run_task(self, task_param):
redis = Custom_Redis(using='etl_task')
mysql = Custom_MySQL(using='etl_manage')
mysql.begin()
datas = {'status': 0}
where = {'id': int(task_param['id'])}
hive = Custom_Hive(using='hadoops2')
task_key_exc = ""
try:
game = task_param['game']
platform = task_param['platform']
task_dict = {'log_name': task_param['log_name'], 'do_rate': task_param['do_rate'],
'log_date': task_param['log_date'], 'task_date': task_param['task_date']}
task_key_tmp = "%(log_name)s%(do_rate)s%(log_date)s%(task_date)s" % task_dict
task_key_md5 = hashlib.md5()
task_key_md5.update(task_key_tmp)
task_key_md5_result = task_key_md5.hexdigest()
task_key = "dw2dm_"+str(game)+"_"+str(platform)+"_"+str(task_key_md5_result)
task_key_exc = task_key
'''
celery 本身的机制存在漏洞，会将一个已经完成任务再次分配给其他的worker，致使同一个任务执行多次
为防止此种现象，在任务开始执行时，将任务的“唯一标示”写入redis中，标注已执行
'''
#如果task_key is None, 则表示该条任务没有执行过，正常执行即可
#如果task_key = 0, 则表示该条任务上次执行失败，允许重复执行
if redis.get(task_key) == "0" or redis.get(task_key) is None:
exec_sql = task_param['exec_sql']
log_date = (datetime.datetime.strptime(task_param['log_date'], '%Y%m%d')).strftime("%Y-%m-%d")
#prefix_sql = task_param['prefix_sql']
#post_sql = task_param['post_sql']
index_dict = {'db_name': task_param['db_name'], 'table_name': task_param['table_name'],
'platform': task_param['platform'], 'log_date': log_date}
index_dir_name = "%(db_name)s.db/%(table_name)s/plat_form=%(platform)s/log_date=%(log_date)s/" % index_dict
#用于删除索引
del_index_dir_name = "%(db_name)s.db/%(table_name)s/plat_form=%(platform)s/log_date=%(log_date)s/" % index_dict
if platform == 'all':
index_dict = {'db_name': task_param['db_name'], 'table_name': task_param['table_name'],
'log_date': log_date}
#用于建立索引(建立索引时，不能使用*通配符，所以仅指定到表名)
index_dir_name = "%(db_name)s.db/%(table_name)s/" % index_dict
#用于删除索引，虽然建立的时候指定到表名，但删除的时候，仅删除当天的
del_index_dir_name = "%(db_name)s.db/%(table_name)s/*/log_date=%(log_date)s/*" % index_dict
hql_conf = "set hive.exec.dynamic.partition.mode=nonstrict;" \
"set hive.exec.compress.output=true;" \
"set mapred.output.compress=true;" \
"set mapred.output.compression.codec=com.hadoop.compression.lzo.LzopCodec;" \
"set io.compression.codecs=com.hadoop.compression.lzo.LzopCodec; "
#获取项目根路径：/data/etl_manage
project_path = os.getcwd()
'''
将任务标示为开始执行：1
'''
datas['status'] = 1
datas['start_time'] = str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
mysql.update('dw2dm_log', ' id = %(id)d' % where, **datas)
mysql.commit()
#在redis标注 任务正在执行
redis.set(task_key, 1)
#执行正式的 select xx insert xx 语句
result = hive.select_insert(hql_conf+exec_sql)
logger.info('exec exec_sql: select xxx insert xxx {0}'.format(result['output']))
if result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], exec_sql))
'''
执行失败，将其状态标为未执行：0
'''
datas['status'] = 0
#在redis标注 任务未执行
redis.set(task_key, 0)
else:
'''
将任务标示为执行hive ql, select xxx insert xxx完成：2
'''
datas['status'] = 2
datas.pop('start_time')
mysql.update('dw2dm_log', ' id = %(id)d' % where, **datas)
mysql.commit()
'''
建立索引，否则lzo将不支持split
'''
cmd_index = '/bin/bash %s/dw2dm/create_lzo_indexer.sh %s' % (project_path, index_dir_name)
logger.info('create lzo index: {0}'.format(cmd_index))
index_result = cmd.run(cmd_index)
if index_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (index_result['status'], index_result['output'], cmd_index))
#在redis标注 任务未执行
redis.set(task_key, 0)
else:
if "create index success" in index_result['output']:
'''
将任务标识为建立lzo索引完成：3
'''
datas['status'] = 3
datas['end_time'] = str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
#在redis标注 任务已完成且成功
redis.set(task_key, 2)
else:
'''
如果load数据失败，则删除半途出现错误的文件，防止hive查询的时候报错
'''
cmd_remove = '/bin/bash %s/dw2dm/remove_damaged_file.sh %s' % (project_path, del_index_dir_name)
logger.info('remove damaged files: {0}'.format(cmd_remove))
remove_result = cmd.run(cmd_remove)
if remove_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove))
#在redis标注 任务未执行
redis.set(task_key, 0)
#如果task_key=2, 则标示该条任务已经运行成功
elif redis.get(task_key) == "2":
datas['status'] = 3
#该条任务正在运行中
else:
return True
'''
执行完毕，模拟从队列中清除任务:0
'''
datas['in_queue'] = 0
update_result = mysql.update('dw2dm_log', ' id = %(id)d' % where, **datas)
# 如果更新失败，则再调用一次，如果还是失败，则等待自动修复机制，但这种概率很小了
if update_result != 1:
mysql.update('dw2dm_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
return True
except Exception as exc:
logger.error('dw2dm_log error: %s' % exc)
redis.set(task_key_exc, 0)
mysql.rollback()
datas = {'in_queue': 0, 'status': 0}
mysql.update('dw2dm_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
raise self.retry(exc=exc, countdown=60)
def run_task(self, task_param):
mysql = Custom_MySQL(using='hadoops2')
mysql_etl = Custom_MySQL(using='etl_manage')
mysql.begin()
mysql_etl.begin()
where = {'id': int(task_param['id'])}
try:
hive = Custom_Hive(using='ares_dw')
game = task_param['game']
platform = task_param['platform']
table_name = task_param['table_name']
log_date = task_param['log_date']
prefix_sql = task_param['prefix_sql']
exec_sql = task_param['exec_sql']
post_sql = task_param['post_sql']
date_cycle = task_param['date_cycle']
random_str = str(random.randint(0, 999999999))
stimes = str(int(time.time()))
tmp_file_dir = "/tmp/tmp/%s/%s/%s" % (game, platform, log_date)
#创建本地目录
if not os.path.exists(tmp_file_dir):
os.makedirs(tmp_file_dir)
tmp_file = "%s/%s_%s_%s_%s.txt" % (tmp_file_dir, table_name,
date_cycle, random_str, stimes)
hql_conf = "SET hive.support.concurrency=false;SET hive.exec.compress.output=true;" \
"SET mapred.output.compression.codec=com.hadoop.compression.lzo.LzopCodec; "
'''
将任务标示为开始执行：1
'''
datas = {
'status':
1,
'start_time':
str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
}
mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
#执行前置sql，将数据临时写入本地，用于mysql load数据
if prefix_sql is not None:
result = mysql.delete_by_sql(prefix_sql)
logger.info('exec prefix_sql: delete old data {0}'.format(
result['output']))
if result['status'] == 0:
'''
将任务标示为删除临时文件完成：2
'''
datas = {'status': 2}
mysql_etl.update('dm2report_log', ' id = %(id)d' % where,
**datas)
mysql_etl.commit()
'''
开始执行hive ql,将数据dump到本地
'''
result = hive.dump(hql_conf + exec_sql, tmp_file)
logger.info('exec exec_sql: dump data {0}'.format(
result['output']))
if result['status'] == 0 and True == os.path.exists(
'%s' % tmp_file):
'''
将任务标示为dump hive数据完成：3
'''
datas = {'status': 3, 'tmp_file_name': tmp_file}
mysql_etl.update('dm2report_log', ' id = %(id)d' % where,
**datas)
mysql_etl.commit()
#执行后置sql
if post_sql is not None:
post_sql = post_sql.replace('{dir_path}', tmp_file)
post_sql = post_sql.replace('{table_name}',
task_param['table_name'])
post_sql = post_sql.replace('{db_name}',
task_param['db_name'])
result = mysql.load(post_sql)
logger.info(
'exec post_sql: load data to hdfs {0}'.format(
result['output']))
if result['status'] == 0:
'''
将任务标识为录入mysql完成：4
'''
datas = {
'status':
4,
'end_time':
str(datetime.datetime.now().strftime(
'%Y-%m-%d %H:%M:%S'))
}
mysql_etl.update('dm2report_log',
' id = %(id)d' % where, **datas)
else:
logger.error(
'Error Code %s : %s Cmd: %s' %
(result['status'], result['output'], post_sql))
else:
logger.error(
'Error Code %s : %s Cmd: %s' %
(result['status'], result['output'], exec_sql))
else:
logger.error('Error Code %s : %s Cmd: %s' %
(result['status'], result['output'], prefix_sql))
'''
执行失败，将其状态标为未执行：0
'''
datas = {'status': 0}
mysql_etl.update('dm2report_log', ' id = %(id)d' % where,
**datas)
'''
将任务标示为：(模拟) 已从任务队列中移除
'''
datas = {'in_queue': 0}
update_result = mysql_etl.update('dm2report_log',
' id = %(id)d' % where, **datas)
# 如果数据库更新失败，再调用一次。 如果还是失败，等待自动修复机制，但这样的概率应该很小了。
if update_result != 1:
mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
mysql.commit()
mysql_etl.close()
mysql.close()
return True
except Exception as exc:
logger.error('dm2report error: %s' % exc)
mysql_etl.rollback()
mysql.rollback()
datas = {'in_queue': 0, 'status': 0}
mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
mysql_etl.close()
mysql.close()
raise self.retry(exc=exc, countdown=60)
def run_task(self, task_param):
mysql = Custom_MySQL(using='hadoops2')
mysql_etl = Custom_MySQL(using='etl_manage')
mysql.begin()
mysql_etl.begin()
where = {'id': int(task_param['id'])}
try:
hive = Custom_Hive(using='ares_dw')
game = task_param['game']
platform = task_param['platform']
table_name = task_param['table_name']
log_date = task_param['log_date']
prefix_sql = task_param['prefix_sql']
exec_sql = task_param['exec_sql']
post_sql = task_param['post_sql']
date_cycle = task_param['date_cycle']
random_str = str(random.randint(0, 999999999))
stimes = str(int(time.time()))
tmp_file_dir = "/tmp/tmp/%s/%s/%s" % (game, platform, log_date)
#创建本地目录
if not os.path.exists(tmp_file_dir):
os.makedirs(tmp_file_dir)
tmp_file = "%s/%s_%s_%s_%s.txt" % (tmp_file_dir, table_name, date_cycle, random_str, stimes)
hql_conf = "SET hive.support.concurrency=false;SET hive.exec.compress.output=true;" \
"SET mapred.output.compression.codec=com.hadoop.compression.lzo.LzopCodec; "
'''
将任务标示为开始执行：1
'''
datas = {'status': 1, 'start_time': str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))}
mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
#执行前置sql，将数据临时写入本地，用于mysql load数据
if prefix_sql is not None:
result = mysql.delete_by_sql(prefix_sql)
logger.info('exec prefix_sql: delete old data {0}'.format(result['output']))
if result['status'] == 0:
'''
将任务标示为删除临时文件完成：2
'''
datas = {'status': 2}
mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
'''
开始执行hive ql,将数据dump到本地
'''
result = hive.dump(hql_conf+exec_sql, tmp_file)
logger.info('exec exec_sql: dump data {0}'.format(result['output']))
if result['status'] == 0 and True == os.path.exists('%s' % tmp_file):
'''
将任务标示为dump hive数据完成：3
'''
datas = {'status': 3, 'tmp_file_name': tmp_file}
mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
#执行后置sql
if post_sql is not None:
post_sql = post_sql.replace('{dir_path}', tmp_file)
post_sql = post_sql.replace('{table_name}', task_param['table_name'])
post_sql = post_sql.replace('{db_name}', task_param['db_name'])
result = mysql.load(post_sql)
logger.info('exec post_sql: load data to hdfs {0}'.format(result['output']))
if result['status'] == 0:
'''
将任务标识为录入mysql完成：4
'''
datas = {'status': 4, 'end_time': str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))}
mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas)
else:
logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], post_sql))
else:
logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], exec_sql))
else:
logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], prefix_sql))
'''
执行失败，将其状态标为未执行：0
'''
datas = {'status': 0}
mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas)
'''
将任务标示为：(模拟) 已从任务队列中移除
'''
datas = {'in_queue': 0}
update_result = mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas)
# 如果数据库更新失败，再调用一次。 如果还是失败，等待自动修复机制，但这样的概率应该很小了。
if update_result != 1:
mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
mysql.commit()
mysql_etl.close()
mysql.close()
return True
except Exception as exc:
logger.error('dm2report error: %s' % exc)
mysql_etl.rollback()
mysql.rollback()
datas = {'in_queue': 0, 'status': 0}
mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
mysql_etl.close()
mysql.close()
raise self.retry(exc=exc, countdown=60)
def run_task(self, task_param):
redis = Custom_Redis(using='etl_manage')
mysql = Custom_MySQL(using='etl_manage')
mysql.begin()
datas = {'download_status': 0}
where = {'id': int(task_param['id'])}
local_log_dir = ""
lzo_file_name = ""
try:
dir_param = {
'game': task_param['game'],
'platform': task_param['platform'],
'log_date': task_param['log_date'],
'log_name': task_param['log_name']
}
filename_dict = {
'log_name': task_param['log_name'],
'log_time': task_param['log_time'],
'source_ip': task_param['source_ip']
}
log_dir = "/%(game)s/%(platform)s/%(log_date)s/%(log_name)s/" % dir_param
txt_file_name = "%(log_name)s_%(source_ip)s_%(log_time)s.txt" % filename_dict
lzo_file_name = "%(log_name)s_%(source_ip)s_%(log_time)s.txt.lzo" % filename_dict
md5_file_name = "%(log_name)s_%(source_ip)s_%(log_time)s.txt.lzo.md5" % filename_dict
lzo_download_url = task_param['download_url'].rstrip(
'/') + log_dir + lzo_file_name
md5_download_url = task_param['download_url'].rstrip(
'/') + log_dir + md5_file_name
#从redis中，获取当前数据对应存储到哪块磁盘
if redis.get("disk_xml") is None:
disk_tmp = open('/data/etl_manage/conf/disk_game.xml', 'r')
redis.set("disk_xml", str(disk_tmp.read()))
disk_list = str(redis.get("disk_xml"))
root = ET.fromstring(disk_list)
disk = ""
for gameinfo in root.findall('game'):
if gameinfo.get('name') == task_param['game']:
disk = gameinfo.get('disk')
continue
#local_log_dir = '/disk1/tmp_data' + log_dir
local_log_dir = '/' + disk + '/data' + log_dir
#创建本地目录
if not os.path.exists(local_log_dir):
os.makedirs(local_log_dir)
#排除同名文件存在的可能，同时为修复执行提供方便
if os.path.exists('%s%s' % (local_log_dir, txt_file_name)):
cmd_remove = 'rm -f %s%s*' % (local_log_dir, txt_file_name)
logger.info('remove history file: {0}'.format(cmd_remove))
remove_result = cmd.run(cmd_remove)
if remove_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' %
(remove_result['status'], remove_result['output'],
cmd_remove))
'''
下载md5文件,如果md5文件不存在则退出，不再继续执行程序，同时不向数据库写入任何标示
'''
md5_line = ' wget -o /tmp/log/wget_log -O %s%s %s' % (
local_log_dir, md5_file_name, md5_download_url)
logger.info('md5 info: {0}'.format(md5_line))
md5_result = cmd.run(md5_line)
if md5_result['status'] != 0:
logger.error(
'Error Code %s : %s Cmd: %s' %
(md5_result['status'], md5_result['output'], md5_line))
else:
'''
将任务标识为开始执行：1
'''
datas['download_status'] = 1
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
'''
下载数据文件
'''
lzo_line = ' wget -o /tmp/log/wget_log -O %s%s %s' % (
local_log_dir, lzo_file_name, lzo_download_url)
logger.info('file info: {0}'.format(lzo_line))
lzo_result = cmd.run(lzo_line)
if lzo_result['status'] != 0:
logger.error(
'Error Code %s : %s Cmd: %s' %
(lzo_result['status'], lzo_result['output'], lzo_line))
datas['download_status'] = 0
else:
'''
将任务标识为下载完成：2
'''
datas['download_status'] = 2
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
'''
md5校验，如果未通过则不再继续执行程序
'''
check_line = "cat %s%s |grep `md5sum %s%s|cut -d ' ' -f 1`" % (
local_log_dir, md5_file_name, local_log_dir, lzo_file_name)
logger.info('md5 or md5 info: {0}'.format(check_line))
check_result = cmd.run(check_line)
if check_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' %
(check_result['status'],
check_result['output'], check_line))
datas['download_status'] = 0
else:
'''
lzop解压缩
'''
cmd_line = ' lzop -dP %s%s' % (local_log_dir,
lzo_file_name)
logger.info('file info: {0}'.format(cmd_line))
cmd_result = cmd.run(cmd_line)
if cmd_result['status'] != 0:
logger.error('Lzop Code %s : %s Cmd: %s' %
(cmd_result['status'],
cmd_result['output'], cmd_line))
datas['download_status'] = 0
else:
'''
将任务标识md5一致，完成下载任务：3
'''
datas['download_status'] = 3
'''
将任务标示为：(模拟) 已从任务队列中移除
'''
datas['in_download_queue'] = 0
update_result = mysql.update('etl_data_log', ' id = %(id)d' % where,
**datas)
# 如果更新失败，则再调用一次，如果还是失败，则等待自动修复机制，但这种概率很小了
if update_result != 1:
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
return True
except Exception as exc:
print(exc)
logger.error('download error : %s' % exc)
mysql.rollback()
kill_proces = "kill -9 `ps -ef |grep wget |grep -v grep |grep '%s%s'|awk '{print $2}'`" % (
local_log_dir, lzo_file_name)
cmd.run(kill_proces)
datas = {'in_download_queue': 0, 'download_status': 0}
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
raise self.retry(exc=exc, countdown=60)
def run_task(self, task_param):
mysql = Custom_MySQL(using='etl_manage')
mysql.begin()
datas = {'etl_status': 0}
where = {'id': int(task_param['id'])}
try:
log_param = {
'game': task_param['game'],
'platform': task_param['platform'],
'log_date': task_param['log_date'],
'log_name': task_param['log_name'],
'log_dir': task_param['log_dir'],
'col_num': task_param['col_num']
}
log_name_param = {
'log_name': task_param['log_name'],
'source_ip': task_param['source_ip'],
'log_time': task_param['log_time']
}
do_rate = task_param['do_rate']
flag = task_param['flag']
log_dir = '%(log_dir)s/%(game)s/%(platform)s/%(log_date)s/%(log_name)s' % log_param
log_name = '%(log_name)s_%(source_ip)s_%(log_time)s' % log_name_param
log_name_notime = '%(log_name)s_%(source_ip)s' % log_name_param
col_num = task_param['col_num']
project_path = os.getcwd()
#判断如果有md5文件和数据文件同时存在，则开始执行
if (do_rate == "1day" and flag == "log") or (
os.path.exists('%s/%s.log.md5' % (log_dir, log_name)) is True
and os.path.exists('%s/%s.log' % (log_dir, log_name)) is True):
#排除同名文件存在的可能，同时为修复执行提供方便
if os.path.exists('%s/%s.txt' % (log_dir, log_name)):
cmd_remove = 'rm -f %s/%s.txt*' % (log_dir, log_name)
logger.info('remove history file: {0}'.format(cmd_remove))
remove_result = cmd.run(cmd_remove)
if remove_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' %
(remove_result['status'],
remove_result['output'], cmd_remove))
'''
将任务标识为开始执行：1
'''
datas['etl_status'] = 1
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
'''
校验数据
'''
#如果是“天”频次的日志数据，则特殊处理，直接将同一ip的合成一个文件，同时执行校验
if do_rate == "1day" and flag == "log":
cmd_merge = '/bin/bash %s/etl_data/merge_data.sh %s %s %s' % (
project_path, log_dir, log_name_notime, col_num)
logger.info('check data: {0}'.format(cmd_merge))
merge_result = cmd.run(cmd_merge)
else:
cmd_merge = '/bin/bash %s/etl_data/check_data.sh %s %s %s' % (
project_path, log_dir, log_name, col_num)
logger.info('check data: {0}'.format(cmd_merge))
merge_result = cmd.run(cmd_merge)
if merge_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' %
(merge_result['status'], merge_result['output'],
cmd_merge))
datas['etl_status'] = 0
else:
'''
读取校验格式后的文件总条数
'''
row = open('%s/%s.txt.row' % (log_dir, log_name)).read()
'''
将文件总条数写入数据库，并将任务标识为为校验已完成：2
'''
datas['etl_status'] = 2
datas['row_num'] = int(row)
#datas = {'etl_status': 2, 'row_num': int(row)}
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
'''
压缩数据
'''
cmd_compress = '/bin/bash %s/etl_data/compress_data.sh %s %s' % (
project_path, log_dir, log_name)
logger.info('compress data: {0}'.format(cmd_compress))
compress_result = cmd.run(cmd_compress)
if compress_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' %
(compress_result['status'],
compress_result['output'], cmd_compress))
datas['etl_status'] = 0
else:
'''
将任务标识为压缩完成：4
'''
datas['etl_status'] = 4
datas.pop('row_num')
mysql.update('etl_data_log', ' id = %(id)d' % where,
**datas)
mysql.commit()
'''
生成MD5文件
'''
cmd_md5 = '/bin/bash %s/etl_data/md5_data.sh %s %s' % (
project_path, log_dir, log_name)
logger.info('md5 data: {0}'.format(cmd_md5))
md5_result = cmd.run(cmd_md5)
if md5_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' %
(md5_result['status'],
md5_result['output'], cmd_md5))
datas['etl_status'] = 0
else:
'''
将任务标识为生成MD5完成(即为校验、合并、压缩均已完成)：6
'''
datas['etl_status'] = 6
'''
执行完毕，模拟从队列中清楚任务:0
'''
datas['in_etl_queue'] = 0
update_result = mysql.update('etl_data_log', ' id = %(id)d' % where,
**datas)
# 如果更新失败，则再调用一次，如果还是失败，则等待自动修复机制，但这种概率很小了
if update_result != 1:
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
return True
except Exception as exc:
logger.error('etl_data error: %s' % exc)
mysql.rollback()
datas = {'in_etl_queue': 0, 'etl_status': 0}
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
raise self.retry(exc=exc, countdown=30)
def run_task(self, task_param):
redis = Custom_Redis(using='etl_manage')
mysql = Custom_MySQL(using='etl_manage')
mysql.begin()
datas = {'download_status': 0}
where = {'id': int(task_param['id'])}
local_log_dir = ""
lzo_file_name = ""
try:
dir_param = {'game': task_param['game'], 'platform': task_param['platform'],
'log_date': task_param['log_date'], 'log_name': task_param['log_name']}
filename_dict = {'log_name': task_param['log_name'], 'log_time': task_param['log_time'], 'source_ip': task_param['source_ip']}
log_dir = "/%(game)s/%(platform)s/%(log_date)s/%(log_name)s/" % dir_param
txt_file_name = "%(log_name)s_%(source_ip)s_%(log_time)s.txt" % filename_dict
lzo_file_name = "%(log_name)s_%(source_ip)s_%(log_time)s.txt.lzo" % filename_dict
md5_file_name = "%(log_name)s_%(source_ip)s_%(log_time)s.txt.lzo.md5" % filename_dict
lzo_download_url = task_param['download_url'].rstrip('/') + log_dir + lzo_file_name
md5_download_url = task_param['download_url'].rstrip('/') + log_dir + md5_file_name
#从redis中，获取当前数据对应存储到哪块磁盘
if redis.get("disk_xml") is None:
disk_tmp = open('/data/etl_manage/conf/disk_game.xml', 'r')
redis.set("disk_xml", str(disk_tmp.read()))
disk_list = str(redis.get("disk_xml"))
root = ET.fromstring(disk_list)
disk = ""
for gameinfo in root.findall('game'):
if gameinfo.get('name') == task_param['game']:
disk = gameinfo.get('disk')
continue
#local_log_dir = '/disk1/tmp_data' + log_dir
local_log_dir = '/'+disk+'/data' + log_dir
#创建本地目录
if not os.path.exists(local_log_dir):
os.makedirs(local_log_dir)
#排除同名文件存在的可能，同时为修复执行提供方便
if os.path.exists('%s%s' % (local_log_dir, txt_file_name)):
cmd_remove = 'rm -f %s%s*' % (local_log_dir, txt_file_name)
logger.info('remove history file: {0}'.format(cmd_remove))
remove_result = cmd.run(cmd_remove)
if remove_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove))
'''
下载md5文件,如果md5文件不存在则退出，不再继续执行程序，同时不向数据库写入任何标示
'''
md5_line = ' wget -o /tmp/log/wget_log -O %s%s %s' % (local_log_dir, md5_file_name, md5_download_url)
logger.info('md5 info: {0}'.format(md5_line))
md5_result = cmd.run(md5_line)
if md5_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (md5_result['status'], md5_result['output'], md5_line))
else:
'''
将任务标识为开始执行：1
'''
datas['download_status'] = 1
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
'''
下载数据文件
'''
lzo_line = ' wget -o /tmp/log/wget_log -O %s%s %s' % (local_log_dir, lzo_file_name, lzo_download_url)
logger.info('file info: {0}'.format(lzo_line))
lzo_result = cmd.run(lzo_line)
if lzo_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (lzo_result['status'], lzo_result['output'], lzo_line))
datas['download_status'] = 0
else:
'''
将任务标识为下载完成：2
'''
datas['download_status'] = 2
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
'''
md5校验，如果未通过则不再继续执行程序
'''
check_line = "cat %s%s |grep `md5sum %s%s|cut -d ' ' -f 1`" % (local_log_dir, md5_file_name, local_log_dir, lzo_file_name)
logger.info('md5 or md5 info: {0}'.format(check_line))
check_result = cmd.run(check_line)
if check_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (check_result['status'], check_result['output'], check_line))
datas['download_status'] = 0
else:
'''
lzop解压缩
'''
cmd_line = ' lzop -dP %s%s' % (local_log_dir, lzo_file_name)
logger.info('file info: {0}'.format(cmd_line))
cmd_result = cmd.run(cmd_line)
if cmd_result['status'] != 0:
logger.error('Lzop Code %s : %s Cmd: %s' % (cmd_result['status'], cmd_result['output'], cmd_line))
datas['download_status'] = 0
else:
'''
将任务标识md5一致，完成下载任务：3
'''
datas['download_status'] = 3
'''
将任务标示为：(模拟) 已从任务队列中移除
'''
datas['in_download_queue'] = 0
update_result = mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
# 如果更新失败，则再调用一次，如果还是失败，则等待自动修复机制，但这种概率很小了
if update_result != 1:
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
return True
except Exception as exc:
print (exc)
logger.error('download error : %s' % exc)
mysql.rollback()
kill_proces = "kill -9 `ps -ef |grep wget |grep -v grep |grep '%s%s'|awk '{print $2}'`" % (local_log_dir, lzo_file_name)
cmd.run(kill_proces)
datas = {'in_download_queue': 0, 'download_status': 0}
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
raise self.retry(exc=exc, countdown=60)
class CenterApp():
def __init__(self):
self.center_mysql = Custom_MySQL(using='center_app')
# def get_server_list(self):
# '''
# 获取执行任务
# '''
# try:
# ip_list = []
# if param['flag'] == 'basic':
# ip_list.append('120.26.1.250')
#
# if param['flag'] == 'most':
# ip_list.append('120.26.13.150')
#
# if param['flag'] == 'log':
# s_sql = "select t1.* from \
# (select a.public_ip as source_ip,a.platform_id from assets a inner join main_category m \
# on a.main_category_id = m.id \
# where a.is_del = 0 and (a.hostname like '%%%%web_balance%%%%' or a.hostname like '%%%%web_admin%%%%') ) t1 \
# left join platform t2 on t1.platform_id = t2.id \
# where t1.source_ip is not null group by source_ip"
#
# if param['game'] == 'kof':
# s_sql = "select t1.* from \
# (select a.public_ip as source_ip,a.platform_id from assets a inner join main_category m \
# on a.main_category_id = m.id \
# where a.is_del = 0 and a.hostname like '%%%%gameserver%%%%' ) t1 \
# left join platform t2 on t1.platform_id = t2.id \
# where t1.source_ip is not null group by source_ip"
#
# ip_list = self.exec_sql(s_sql)
#
# if param['flag'] == 'snap':
# s_sql = "select s.prefix as source_ip from sub_category s inner join main_category m \
# on s.main_category_id = m.id \
# where s.platform = '%s' and m.prefix = '%s'" % (param['platform'],param['game'])
# ip_list = self.exec_sql(s_sql)
#
# return ip_list
# except Exception as exc:
# print exc
# #异常回滚
# self.center_mysql.rollback()
def get_log_ip(self):
'''
获取所有log的IP
'''
ip_list = {}
sql = "select t1.*,t2.prefix as platform from \
(select m.prefix as gamename,a.public_ip as source_ip,a.platform_id from assets a inner join main_category m \
on a.main_category_id = m.id \
where a.is_del = 0 and (a.hostname like '%%%%web_balance%%%%' or a.hostname like '%%%%web_admin%%%%') ) t1 \
left join platform t2 on t1.platform_id = t2.id \
where t1.source_ip is not null group by source_ip\
union all \
select t1.*,t2.prefix as platform from \
(select m.prefix as gamename,a.public_ip as source_ip,a.platform_id from assets a inner join main_category m \
on a.main_category_id = m.id \
where a.is_del = 0 and a.hostname like '%%%%gameserver%%%%' ) t1 \
left join platform t2 on t1.platform_id = t2.id \
where t1.source_ip is not null group by source_ip"
result = self.center_mysql.query(sql)
for info in result:
if info['gamename'] is None or info['platform'] is None:
continue
if info['gamename'].encode('utf8') not in ip_list:
ip_list[info['gamename'].encode('utf8')] = {}
if info['platform'].encode('utf8') not in ip_list[
info['gamename'].encode('utf8')]:
ip_list[info['gamename'].encode('utf8')][
info['platform'].encode('utf8')] = []
ip_list[info['gamename'].encode('utf8')][info['platform'].encode(
'utf8')].append(info['source_ip'].encode('utf8'))
return ip_list
def get_snap_ip(self):
'''
获取所有快照的IP
'''
ip_list = {}
s_sql = "select f.prefix as platform,s.prefix as source_ip,m.prefix as gamename from sub_category s inner join main_category m \
on s.main_category_id = m.id \
left join platform f on f.id = s.platform_id"
result = self.center_mysql.query(s_sql)
for info in result:
if info['gamename'] is None or info['platform'] is None:
continue
if info['gamename'].encode('utf8') not in ip_list:
ip_list[info['gamename'].encode('utf8')] = {}
if info['platform'].encode('utf8') not in ip_list[
info['gamename'].encode('utf8')]:
ip_list[info['gamename'].encode('utf8')][
info['platform'].encode('utf8')] = []
ip_list[info['gamename'].encode('utf8')][info['platform'].encode(
'utf8')].append(info['source_ip'].encode('utf8'))
# print ip_list
return ip_list
def get_ip_list(self):
'''
整理IP列表
'''
try:
ip_list = {}
ip_list['basic'] = ['120.26.1.250']
ip_list['mostsdk'] = ['120.26.13.150']
ip_list['wanpay'] = ['112.124.116.44']
log_list = self.get_log_ip()
ip_list['log'] = log_list
snap_list = self.get_snap_ip()
ip_list['snap'] = snap_list
return ip_list
except Exception, e:
print e
print "异常"
#异常回滚
self.center_mysql.rollback()
def run_task(self, task_param):
mysql = Custom_MySQL(using='etl_manage')
hive = Custom_Hive(using='ares_dw')
redis = Custom_Redis(using='etl_manage')
mysql.begin()
where = {'id': int(task_param['id'])}
try:
log_date = datetime.datetime.strptime(task_param['log_date'], '%Y%m%d').strftime("%Y-%m-%d")
log_date_1 = (datetime.datetime.strptime(task_param['log_date'], '%Y%m%d')-datetime.timedelta(days=1)).strftime("%Y-%m-%d")
dir_param = {'game': task_param['game'], 'platform': task_param['platform'],
'log_date': task_param['log_date'], 'log_name': task_param['log_name']}
filename_dict = {'log_name': task_param['log_name'], 'log_time': task_param['log_time']}
index_dict = {'db_name': task_param['db_name'], 'table_name': task_param['table_name'],
'platform': task_param['platform'], 'log_date': log_date}
partition = {'platform': task_param['platform'], 'log_date': log_date}
index_dict_1 = {'db_name': task_param['db_name'], 'table_name': task_param['table_name'],
'platform': task_param['platform'], 'log_date': log_date_1}
partition_1 = {'platform': task_param['platform'], 'log_date': log_date_1}
log_dir = "/%(game)s/%(platform)s/%(log_date)s/%(log_name)s" % dir_param
flag = task_param['flag']
file_name = "%(log_name)s" % filename_dict
file_name_txt = "%(log_name)s_%(log_time)s_result.txt" % filename_dict
file_name_lzo = "%(log_name)s_%(log_time)s_result.txt.lzo" % filename_dict
file_name_row = "%(log_name)s_%(log_time)s_result.txt.row" % filename_dict
index_dir_name = "%(db_name)s.db/%(table_name)s/plat_form=%(platform)s/log_date=%(log_date)s/" % index_dict
partition_name = "plat_form='%(platform)s',log_date='%(log_date)s'" % partition
project_path = os.getcwd()
log_time = task_param['log_time']
do_rate = task_param['do_rate']
#if flag == "snap" or (do_rate == "1day" and flag == "log")
if flag == "snap":
index_dir_name = "%(db_name)s.db/%(table_name)s/plat_form=%(platform)s/log_date=%(log_date)s/" % index_dict_1
partition_name = "plat_form='%(platform)s',log_date='%(log_date)s'" % partition_1
#从redis中，获取当前数据对应存储到哪块磁盘
if redis.get("disk_xml") is None:
disk_tmp = open('/data/etl_manage/conf/disk_game.xml', 'r')
redis.set("disk_xml", str(disk_tmp.read()))
disk_list = str(redis.get("disk_xml"))
root = ET.fromstring(disk_list)
disk = ""
for gameinfo in root.findall('game'):
if gameinfo.get('name') == task_param['game']:
disk = gameinfo.get('disk')
continue
#local_log_dir = '/disk1/tmp_data'+log_dir
local_log_dir = '/'+disk+'/data'+log_dir
logger.info('local_log_dir: {0}'.format(local_log_dir))
#判断目录是否存在
if os.path.exists('%s' % local_log_dir):
#排除同名文件存在的可能，同时为修复执行提供方便
if os.path.exists('%s/%s' % (local_log_dir, file_name_txt)):
cmd_remove = 'rm -f %s/%s*' % (local_log_dir, file_name_txt)
logger.info('remove history file: {0}'.format(cmd_remove))
remove_result = cmd.run(cmd_remove)
if remove_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove))
datas = {'load_status': 1}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
mysql.commit()
cmd_merge = '/bin/bash %s/mergefile2dw/merge_data.sh %s %s %s %s %s' \
'' % (project_path, local_log_dir, local_log_dir, file_name, do_rate, log_time)
logger.info('merge data: {0}'.format(cmd_merge))
merge_result = cmd.run(cmd_merge)
logger.info('merge data result {0}'.format(merge_result['output']))
if merge_result['status'] == 0:
#读取总条数
row = open('%s/%s' % (local_log_dir, file_name_row)).read()
'''
合并数据完成：2
'''
datas = {'load_status': 2, 'row_num': int(row)}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
mysql.commit()
'''
压缩数据
'''
cmd_compress = '/bin/bash %s/mergefile2dw/compress_data.sh %s %s' % (project_path, local_log_dir, file_name_txt)
logger.info('compress data: {0}'.format(cmd_compress))
compress_result = cmd.run(cmd_compress)
if compress_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (compress_result['status'], compress_result['output'], cmd_compress))
else:
'''
压缩数据完成：3
'''
datas = {'load_status': 3}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
mysql.commit()
#执行load之前，删除同名文件，防止同一个文件出现两次的可能
cmd_remove = '/bin/bash %s/mergefile2dw/remove_damaged_file.sh %s %s' % (project_path, index_dir_name, file_name_lzo)
logger.info('remove damaged files: {0}'.format(cmd_remove))
remove_result = cmd.run(cmd_remove)
if remove_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove))
'''
文件加载到hive中
'''
load_sql = task_param['load_sql']
load_sql = load_sql.replace('{dir_path}', local_log_dir+"/"+file_name_lzo)
load_sql = load_sql.replace('{table_name}', task_param['table_name'])
load_sql = load_sql.replace('{partition_name}', '%s' % partition_name)
load_sql = load_sql.replace('{db_name}', task_param['db_name'])
logger.info('hive load SQL: {0}'.format(load_sql))
result = hive.load(load_sql)
logger.info('hive load result {0}'.format(result['output']))
if result['status'] == 0:
'''
将任务标识为加载文件完成：4
'''
datas = {'load_status': 4}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
mysql.commit()
'''
建立索引，否则lzo将不支持split
'''
#print index_dir_name
cmd_index = '/bin/bash %s/mergefile2dw/create_lzo_indexer.sh %s %s' % (project_path, index_dir_name, file_name_lzo)
logger.info('create lzo index: {0}'.format(cmd_index))
index_result = cmd.run(cmd_index)
if index_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (index_result['status'], index_result['output'], cmd_index))
else:
if "create index success" in index_result['output']:
'''
将任务标识为建立lzo索引完成：5
'''
datas = {'load_status': 5}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
else:
'''
如果load数据失败，则删除半途出现错误的文件，防止hive查询的时候报错
'''
cmd_remove = '/bin/bash %s/mergefile2dw/remove_damaged_file.sh %s %s' % (project_path, index_dir_name, file_name_lzo)
logger.info('remove damaged files: {0}'.format(cmd_remove))
remove_result = cmd.run(cmd_remove)
if remove_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove))
else:
logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], load_sql))
else:
'''
合并数据失败
'''
datas = {'load_status': 0}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
logger.error('Error Code %s : %s Cmd: %s' % (merge_result['status'], merge_result['output'], merge_result))
'''
执行完毕，模拟从队列中清楚任务:0
'''
datas = {'in_queue': 0}
update_result = mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
# 如果更新失败，则再调用一次，如果还是失败，则等待自动修复机制，但这种概率很小了
if update_result != 1:
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
return True
except Exception as exc:
logger.error('mergefile2dw error: %s' % exc)
mysql.rollback()
datas = {'in_queue': 0, 'load_status': 0}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
raise self.retry(exc=exc, countdown=30)
def run_task(self, task_param):
redis = Custom_Redis(using='etl_manage')
mysql = Custom_MySQL(using='hadoops2')
mysql_etl = Custom_MySQL(using='etl_manage')
mysql.begin()
mysql_etl.begin()
where = {'id': int(task_param['id'])}
datas = {'load_status': 0}
try:
'''
业务代码块放下方
'''
dir_param = {'game': task_param['game'], 'platform': task_param['platform'],
'log_date': task_param['log_date'], 'log_name': task_param['log_name']}
filename_dict = {'log_name': task_param['log_name'], 'log_time': task_param['log_time'], 'source_ip': task_param['source_ip']}
prefix_sql = task_param['prefix_sql']
post_sql = task_param['post_sql']
log_dir = "/%(game)s/%(platform)s/%(log_date)s/%(log_name)s/" % dir_param
file_name = "%(log_name)s_%(source_ip)s_%(log_time)s.txt" % filename_dict
#从redis中，获取当前数据对应存储到哪块磁盘
if redis.get("disk_xml") is None:
disk_tmp = open('/data/etl_manage/conf/disk_game.xml', 'r')
redis.set("disk_xml", str(disk_tmp.read()))
disk_list = str(redis.get("disk_xml"))
root = ET.fromstring(disk_list)
disk = ""
for gameinfo in root.findall('game'):
if gameinfo.get('name') == task_param['game']:
disk = gameinfo.get('disk')
continue
#local_log_dir = '/disk1/tmp_data'+log_dir
local_log_dir = '/'+disk+'/data'+log_dir
#判断要录入的文件是否存在，如果存在则执行，否则不执行
if os.path.exists('%s%s' % (local_log_dir, file_name)):
'''
将任务标识为开始执行：1
'''
datas['load_status'] = 1
mysql_etl.update('file2mysql_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
logger.info('start load data')
#执行前置sql
if prefix_sql is not None and prefix_sql != '':
mysql.query(prefix_sql)
'''
执行load数据进mysql
'''
load_sql = task_param['load_sql']
load_sql = load_sql.replace('{dir_path}', local_log_dir+file_name)
load_sql = load_sql.replace('{table_name}', task_param['table_name'])
load_sql = load_sql.replace('{db_name}', task_param['db_name'])
result = mysql.load(load_sql)
logger.info('load data to mysql: {0}'.format(result['output']))
#判断录入mysql是否成功
if result['status'] == 0:
#执行后置sql
if post_sql is not None and post_sql != '':
post_sql = post_sql.replace('{table_name}', task_param['table_name'])
post_sql = post_sql.replace('{db_name}', task_param['db_name'])
mysql.query(post_sql)
'''
将任务标识为录入mysql完成：3
'''
datas['load_status'] = 3
else:
logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], load_sql))
'''
录入mysql失败，将任务标示为未执行：0
'''
datas['load_status'] = 0
'''
将任务标示为：(模拟) 已从任务队列中移除
'''
datas['in_queue'] = 0
update_result = mysql_etl.update('file2mysql_log', ' id = %(id)d' % where, **datas)
# 如果更新失败，则再调用一次，如果还是失败，则等待自动修复机制，但这种概率很小了
if update_result != 1:
mysql_etl.update('file2mysql_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
mysql.commit()
mysql_etl.close()
mysql.close()
return True
except Exception as exc:
logger.error('file2mysql error: %s' % exc)
mysql_etl.rollback()
mysql.rollback()
datas = {'in_queue': 0, 'load_status': 0}
mysql_etl.update('file2mysql_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
mysql_etl.close()
mysql.close()
raise self.retry(exc=exc, countdown=60)
def run_task(self, task_param):
mysql = Custom_MySQL(using='etl_manage')
hive = Custom_Hive(using='ares_dw')
redis = Custom_Redis(using='etl_manage')
mysql.begin()
where = {'id': int(task_param['id'])}
try:
log_date = datetime.datetime.strptime(task_param['log_date'],
'%Y%m%d').strftime("%Y-%m-%d")
log_date_1 = (
datetime.datetime.strptime(task_param['log_date'], '%Y%m%d') -
datetime.timedelta(days=1)).strftime("%Y-%m-%d")
dir_param = {
'game': task_param['game'],
'platform': task_param['platform'],
'log_date': task_param['log_date'],
'log_name': task_param['log_name']
}
filename_dict = {
'log_name': task_param['log_name'],
'log_time': task_param['log_time']
}
index_dict = {
'db_name': task_param['db_name'],
'table_name': task_param['table_name'],
'platform': task_param['platform'],
'log_date': log_date
}
partition = {'platform': task_param['platform'], 'log_date': log_date}
index_dict_1 = {
'db_name': task_param['db_name'],
'table_name': task_param['table_name'],
'platform': task_param['platform'],
'log_date': log_date_1
}
partition_1 = {
'platform': task_param['platform'],
'log_date': log_date_1
}
log_dir = "/%(game)s/%(platform)s/%(log_date)s/%(log_name)s" % dir_param
flag = task_param['flag']
file_name = "%(log_name)s" % filename_dict
file_name_txt = "%(log_name)s_%(log_time)s_result.txt" % filename_dict
file_name_lzo = "%(log_name)s_%(log_time)s_result.txt.lzo" % filename_dict
file_name_row = "%(log_name)s_%(log_time)s_result.txt.row" % filename_dict
index_dir_name = "%(db_name)s.db/%(table_name)s/plat_form=%(platform)s/log_date=%(log_date)s/" % index_dict
partition_name = "plat_form='%(platform)s',log_date='%(log_date)s'" % partition
project_path = os.getcwd()
log_time = task_param['log_time']
do_rate = task_param['do_rate']
#if flag == "snap" or (do_rate == "1day" and flag == "log")
if flag == "snap":
index_dir_name = "%(db_name)s.db/%(table_name)s/plat_form=%(platform)s/log_date=%(log_date)s/" % index_dict_1
partition_name = "plat_form='%(platform)s',log_date='%(log_date)s'" % partition_1
#从redis中，获取当前数据对应存储到哪块磁盘
if redis.get("disk_xml") is None:
disk_tmp = open('/data/etl_manage/conf/disk_game.xml', 'r')
redis.set("disk_xml", str(disk_tmp.read()))
disk_list = str(redis.get("disk_xml"))
root = ET.fromstring(disk_list)
disk = ""
for gameinfo in root.findall('game'):
if gameinfo.get('name') == task_param['game']:
disk = gameinfo.get('disk')
continue
#local_log_dir = '/disk1/tmp_data'+log_dir
local_log_dir = '/' + disk + '/data' + log_dir
logger.info('local_log_dir: {0}'.format(local_log_dir))
#判断目录是否存在
if os.path.exists('%s' % local_log_dir):
#排除同名文件存在的可能，同时为修复执行提供方便
if os.path.exists('%s/%s' % (local_log_dir, file_name_txt)):
cmd_remove = 'rm -f %s/%s*' % (local_log_dir, file_name_txt)
logger.info('remove history file: {0}'.format(cmd_remove))
remove_result = cmd.run(cmd_remove)
if remove_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' %
(remove_result['status'],
remove_result['output'], cmd_remove))
datas = {'load_status': 1}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
mysql.commit()
cmd_merge = '/bin/bash %s/mergefile2dw/merge_data.sh %s %s %s %s %s' \
'' % (project_path, local_log_dir, local_log_dir, file_name, do_rate, log_time)
logger.info('merge data: {0}'.format(cmd_merge))
merge_result = cmd.run(cmd_merge)
logger.info('merge data result {0}'.format(merge_result['output']))
if merge_result['status'] == 0:
#读取总条数
row = open('%s/%s' % (local_log_dir, file_name_row)).read()
'''
合并数据完成：2
'''
datas = {'load_status': 2, 'row_num': int(row)}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
mysql.commit()
'''
压缩数据
'''
cmd_compress = '/bin/bash %s/mergefile2dw/compress_data.sh %s %s' % (
project_path, local_log_dir, file_name_txt)
logger.info('compress data: {0}'.format(cmd_compress))
compress_result = cmd.run(cmd_compress)
if compress_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' %
(compress_result['status'],
compress_result['output'], cmd_compress))
else:
'''
压缩数据完成：3
'''
datas = {'load_status': 3}
mysql.update('file2dw_log', ' id = %(id)d' % where,
**datas)
mysql.commit()
#执行load之前，删除同名文件，防止同一个文件出现两次的可能
cmd_remove = '/bin/bash %s/mergefile2dw/remove_damaged_file.sh %s %s' % (
project_path, index_dir_name, file_name_lzo)
logger.info('remove damaged files: {0}'.format(cmd_remove))
remove_result = cmd.run(cmd_remove)
if remove_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' %
(remove_result['status'],
remove_result['output'], cmd_remove))
'''
文件加载到hive中
'''
load_sql = task_param['load_sql']
load_sql = load_sql.replace(
'{dir_path}', local_log_dir + "/" + file_name_lzo)
load_sql = load_sql.replace('{table_name}',
task_param['table_name'])
load_sql = load_sql.replace('{partition_name}',
'%s' % partition_name)
load_sql = load_sql.replace('{db_name}',
task_param['db_name'])
logger.info('hive load SQL: {0}'.format(load_sql))
result = hive.load(load_sql)
logger.info('hive load result {0}'.format(
result['output']))
if result['status'] == 0:
'''
将任务标识为加载文件完成：4
'''
datas = {'load_status': 4}
mysql.update('file2dw_log', ' id = %(id)d' % where,
**datas)
mysql.commit()
'''
建立索引，否则lzo将不支持split
'''
#print index_dir_name
cmd_index = '/bin/bash %s/mergefile2dw/create_lzo_indexer.sh %s %s' % (
project_path, index_dir_name, file_name_lzo)
logger.info('create lzo index: {0}'.format(cmd_index))
index_result = cmd.run(cmd_index)
if index_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' %
(index_result['status'],
index_result['output'], cmd_index))
else:
if "create index success" in index_result[
'output']:
'''
将任务标识为建立lzo索引完成：5
'''
datas = {'load_status': 5}
mysql.update('file2dw_log',
' id = %(id)d' % where, **datas)
else:
'''
如果load数据失败，则删除半途出现错误的文件，防止hive查询的时候报错
'''
cmd_remove = '/bin/bash %s/mergefile2dw/remove_damaged_file.sh %s %s' % (
project_path, index_dir_name,
file_name_lzo)
logger.info('remove damaged files: {0}'.format(
cmd_remove))
remove_result = cmd.run(cmd_remove)
if remove_result['status'] != 0:
logger.error(
'Error Code %s : %s Cmd: %s' %
(remove_result['status'],
remove_result['output'], cmd_remove))
else:
logger.error(
'Error Code %s : %s Cmd: %s' %
(result['status'], result['output'], load_sql))
else:
'''
合并数据失败
'''
datas = {'load_status': 0}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
logger.error('Error Code %s : %s Cmd: %s' %
(merge_result['status'], merge_result['output'],
merge_result))
'''
执行完毕，模拟从队列中清楚任务:0
'''
datas = {'in_queue': 0}
update_result = mysql.update('file2dw_log', ' id = %(id)d' % where,
**datas)
# 如果更新失败，则再调用一次，如果还是失败，则等待自动修复机制，但这种概率很小了
if update_result != 1:
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
return True
except Exception as exc:
logger.error('mergefile2dw error: %s' % exc)
mysql.rollback()
datas = {'in_queue': 0, 'load_status': 0}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
raise self.retry(exc=exc, countdown=30)
def run_task(self, task_param):
mysql = Custom_MySQL(using='hadoops2')
mysql_etl = Custom_MySQL(using='etl_manage')
redis = Custom_Redis(using='etl_task')
mysql.begin()
mysql_etl.begin()
datas = {'status': 0}
where = {'id': int(task_param['id'])}
task_key_exc = ""
try:
hive = Custom_Hive(using='ares_dw')
game = task_param['game']
platform = task_param['platform']
table_name = task_param['table_name']
log_date = task_param['log_date']
prefix_sql = task_param['prefix_sql']
exec_sql = task_param['exec_sql']
post_sql = task_param['post_sql']
date_cycle = task_param['date_cycle']
random_str = str(random.randint(0, 999999999))
stimes = str(int(time.time()))
task_date = task_param['task_date']
task_name = task_param['task_name']
do_rate = task_param['do_rate']
#task_key_tmp = str(game)+str(platform)+str(task_name)+str(date_cycle)+str(do_rate)+str(log_date)+str(task_date)
task_key_tmp = str(task_name)+str(date_cycle)+str(do_rate)+str(log_date)+str(task_date)
task_key_md5 = hashlib.md5()
task_key_md5.update(task_key_tmp)
task_key_md5_result = task_key_md5.hexdigest()
task_key = "dm2report_new_"+str(game)+"_"+str(platform)+"_"+str(task_key_md5_result)
task_key_exc = task_key
'''
celery 本身的机制存在漏洞，会将一个已经完成任务再次分配给其他的worker，致使同一个任务执行多次
为防止此种现象，在任务开始执行时，将任务的“唯一标示”写入redis中，标注已执行
'''
#如果task_key is None, 则表示该条任务没有执行过，正常执行即可
#如果task_key = 0, 则表示该条任务上次执行失败，允许重复执行
if redis.get(task_key) == "0" or redis.get(task_key) is None:
tmp_file_dir = "/tmp/tmp/%s/%s/%s" % (game, platform, log_date)
#创建本地目录
if not os.path.exists(tmp_file_dir):
os.makedirs(tmp_file_dir)
tmp_file = "%s/%s_%s_%s_%s.txt" % (tmp_file_dir, table_name, date_cycle, random_str, stimes)
hql_conf = "SET hive.support.concurrency=false;" \
"SET hive.exec.compress.output=true;" \
"SET mapred.output.compression.codec=com.hadoop.compression.lzo.LzopCodec; "
'''
将任务标示为开始执行：1
'''
datas['status'] = 1
datas['start_time'] = str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
#在redis标注 任务正在执行
redis.set(task_key, 1)
#执行前置sql，将数据临时写入本地，用于mysql load数据
if prefix_sql is not None:
result = mysql.delete_by_sql(prefix_sql)
logger.info('exec prefix_sql: delete old data {0}'.format(result['output']))
if result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], prefix_sql))
'''
执行失败，将其状态标为未执行：0
'''
datas['status'] = 0
#在redis标注 任务未开始执行
redis.set(task_key, 0)
else:
'''
将任务标示为删除临时文件完成：2
'''
datas['status'] = 2
datas.pop('start_time')
mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
'''
开始执行hive ql,将数据dump到本地
'''
result = hive.dump(hql_conf+exec_sql, tmp_file)
logger.info('exec exec_sql: dump data {0}'.format(result['output']))
if result['status'] != 0 or False == os.path.exists('%s' % tmp_file):
logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], exec_sql))
#在redis标注 任务未开始执行
redis.set(task_key, 0)
else:
'''
将任务标示为dump hive数据完成：3
'''
datas['status'] = 3
datas['tmp_file_name'] = tmp_file
mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
#执行后置sql
if post_sql is not None:
post_sql = post_sql.replace('{dir_path}', tmp_file)
post_sql = post_sql.replace('{table_name}', task_param['table_name'])
post_sql = post_sql.replace('{db_name}', task_param['db_name'])
result = mysql.load(post_sql)
logger.info('exec post_sql: load data to hdfs {0}'.format(result['output']))
if result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], post_sql))
#在redis标注 任务未开始执行
redis.set(task_key, 0)
else:
'''
将任务标识为录入mysql完成：4
'''
datas['status'] = 4
datas['end_time'] = str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
datas.pop('tmp_file_name')
#在redis标注 任务已成功执行
redis.set(task_key, 2)
else:
logger.error('prefix_sql is null')
datas['status'] = 0
#在redis标注 任务未开始执行
redis.set(task_key, 0)
#如果task_key=2, 则标示该条任务已经运行成功
elif redis.get(task_key) == "2":
datas['status'] = 4
#该条任务正在运行中
else:
return True
'''
将任务标示为：(模拟) 已从任务队列中移除
'''
datas['in_queue'] = 0
update_result = mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where, **datas)
# 如果数据库更新失败，再调用一次。 如果还是失败，等待自动修复机制，但这样的概率应该很小了。
if update_result != 1:
mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
mysql.commit()
mysql_etl.close()
mysql.close()
return True
except Exception as exc:
logger.error('dm2report error: %s' % exc)
mysql_etl.rollback()
mysql.rollback()
redis.set(task_key_exc, 0)
datas = {'in_queue': 0, 'status': 0}
mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
mysql_etl.close()
mysql.close()
raise self.retry(exc=exc, countdown=60)
'exec_num': exec_num,
'in_queue': in_queue,
'retry_num': retry_num
}
update_result = mysql.update('file2mysql_log',
' id = %(id)d' % where,
**datas)
# 为防止修改数据库时出现异常，调整为：确认修改字段状态成功后，再加入队列
if update_result == 1:
run_task.apply_async((result, ),
queue='file2mysql')
else:
print "update data{'exec_num', 'in_queue', 'retry_num'} error"
else:
datas = {'load_status': -1}
mysql.update('file2mysql_log', ' id = %(id)d' % where,
**datas)
else:
datas = {'load_status': -2}
mysql.update('file2mysql_log', ' id = %(id)d' % where, **datas)
mysql.commit()
#mysql.close()
except Exception as exc:
#回滚
print(exc)
mysql.rollback()
print "db rollback success"
mysql.close()
def run_task(self, task_param):
mysql = Custom_MySQL(using='etl_manage')
mysql.begin()
datas = {'etl_status': 0}
where = {'id': int(task_param['id'])}
try:
log_param = {'game': task_param['game'], 'platform': task_param['platform'],
'log_date': task_param['log_date'], 'log_name': task_param['log_name'],
'log_dir': task_param['log_dir'], 'col_num': task_param['col_num']}
log_name_param = {'log_name': task_param['log_name'], 'source_ip': task_param['source_ip'], 'log_time': task_param['log_time']}
do_rate = task_param['do_rate']
flag = task_param['flag']
log_dir = '%(log_dir)s/%(game)s/%(platform)s/%(log_date)s/%(log_name)s' % log_param
log_name = '%(log_name)s_%(source_ip)s_%(log_time)s' % log_name_param
log_name_notime = '%(log_name)s_%(source_ip)s' % log_name_param
col_num = task_param['col_num']
project_path = os.getcwd()
#判断如果有md5文件和数据文件同时存在，则开始执行
if (do_rate == "1day" and flag == "log") or (os.path.exists('%s/%s.log.md5' % (log_dir, log_name)) is True and os.path.exists('%s/%s.log' % (log_dir, log_name)) is True):
#排除同名文件存在的可能，同时为修复执行提供方便
if os.path.exists('%s/%s.txt' % (log_dir, log_name)):
cmd_remove = 'rm -f %s/%s.txt*' % (log_dir, log_name)
logger.info('remove history file: {0}'.format(cmd_remove))
remove_result = cmd.run(cmd_remove)
if remove_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove))
'''
将任务标识为开始执行：1
'''
datas['etl_status'] = 1
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
'''
校验数据
'''
#如果是“天”频次的日志数据，则特殊处理，直接将同一ip的合成一个文件，同时执行校验
if do_rate == "1day" and flag == "log":
cmd_merge = '/bin/bash %s/etl_data/merge_data.sh %s %s %s' % (project_path, log_dir, log_name_notime, col_num)
logger.info('check data: {0}'.format(cmd_merge))
merge_result = cmd.run(cmd_merge)
else:
cmd_merge = '/bin/bash %s/etl_data/check_data.sh %s %s %s' % (project_path, log_dir, log_name, col_num)
logger.info('check data: {0}'.format(cmd_merge))
merge_result = cmd.run(cmd_merge)
if merge_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (merge_result['status'], merge_result['output'], cmd_merge))
datas['etl_status'] = 0
else:
'''
读取校验格式后的文件总条数
'''
row = open('%s/%s.txt.row' % (log_dir, log_name)).read()
'''
将文件总条数写入数据库，并将任务标识为为校验已完成：2
'''
datas['etl_status'] = 2
datas['row_num'] = int(row)
#datas = {'etl_status': 2, 'row_num': int(row)}
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
'''
压缩数据
'''
cmd_compress = '/bin/bash %s/etl_data/compress_data.sh %s %s' % (project_path, log_dir, log_name)
logger.info('compress data: {0}'.format(cmd_compress))
compress_result = cmd.run(cmd_compress)
if compress_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (compress_result['status'], compress_result['output'], cmd_compress))
datas['etl_status'] = 0
else:
'''
将任务标识为压缩完成：4
'''
datas['etl_status'] = 4
datas.pop('row_num')
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
'''
生成MD5文件
'''
cmd_md5 = '/bin/bash %s/etl_data/md5_data.sh %s %s' % (project_path, log_dir, log_name)
logger.info('md5 data: {0}'.format(cmd_md5))
md5_result = cmd.run(cmd_md5)
if md5_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (md5_result['status'], md5_result['output'], cmd_md5))
datas['etl_status'] = 0
else:
'''
将任务标识为生成MD5完成(即为校验、合并、压缩均已完成)：6
'''
datas['etl_status'] = 6
'''
执行完毕，模拟从队列中清楚任务:0
'''
datas['in_etl_queue'] = 0
update_result = mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
# 如果更新失败，则再调用一次，如果还是失败，则等待自动修复机制，但这种概率很小了
if update_result != 1:
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
return True
except Exception as exc:
logger.error('etl_data error: %s' % exc)
mysql.rollback()
datas = {'in_etl_queue': 0, 'etl_status': 0}
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
raise self.retry(exc=exc, countdown=30)