def run_task(self, task_param):
mysql = Custom_MySQL(using='etl_manage')
mysql.begin()
try:
'''
业务代码块放下方
'''
dir_param ={'game':task_param['game'],
'platform':task_param['platform'],
'log_date':task_param['log_date'],
'log_name':task_param['log_name']}
filename_dict = {'log_name':task_param['log_name'],'log_time':task_param['log_time']}
'''
游戏＼平台＼日期＼业务日志名＼日志或者md5文件
'''
log_dir = "/%(game)s/%(platform)s/%(log_date)s/%(log_name)s/" % dir_param
lzo_file_name = "%(log_name)s_%(log_time)s.txt"% filename_dict
local_log_dir = '/tmp'+log_dir
dump_sql = task_param['dump_sql']
dump_sql = dump_sql.replace('{table_name}',task_param['table_name'])
dump_sql = dump_sql.replace('{partition_name}',task_param['partition_name'])
dump_sql = dump_sql.replace('{db_name}',task_param['db_name'])
print(dump_sql)
result = mysql.dump(sql,local_log_dir+lzo_file_name)
#print(result)
'''
将任务标识为加载文件完成：2
'''
datas = {'load_status':2}
where = {}
where['id'] = int(task_param['id'])
mysql.update('etl_data_log',
' id = %(id)d' % where,
**datas)
mysql.commit()
mysql.close()
return True
except Exception as exc:
print (exc)
mysql.rollback()
raise self.retry(exc=exc, countdown=60)
def main():
tables = excel_table_byindex()
db = Custom_MySQL(using='center_app')
for row in tables:
if row['pub'] != "":
param ={}
param['pwd'] = row['pwd']
db.update('assets','public_ip="%s"'%row['pub'],**param)
def run_task(self, task_param):
mysql = Custom_MySQL(using='etl_manage')
mysql.begin()
try:
'''
业务代码块放下方
'''
dir_param = {
'game': task_param['game'],
'platform': task_param['platform'],
'log_date': task_param['log_date'],
'log_name': task_param['log_name']
}
filename_dict = {
'log_name': task_param['log_name'],
'log_time': task_param['log_time']
}
'''
游戏＼平台＼日期＼业务日志名＼日志或者md5文件
'''
log_dir = "/%(game)s/%(platform)s/%(log_date)s/%(log_name)s/" % dir_param
lzo_file_name = "%(log_name)s_%(log_time)s.txt" % filename_dict
local_log_dir = '/tmp' + log_dir
dump_sql = task_param['dump_sql']
dump_sql = dump_sql.replace('{table_name}', task_param['table_name'])
dump_sql = dump_sql.replace('{partition_name}',
task_param['partition_name'])
dump_sql = dump_sql.replace('{db_name}', task_param['db_name'])
print(dump_sql)
result = mysql.dump(sql, local_log_dir + lzo_file_name)
#print(result)
'''
将任务标识为加载文件完成：2
'''
datas = {'load_status': 2}
where = {}
where['id'] = int(task_param['id'])
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
return True
except Exception as exc:
print(exc)
mysql.rollback()
raise self.retry(exc=exc, countdown=60)
def save_hosts(self):
if self.data == None:
return False
db = Custom_MySQL(using='center_app')
for param in self.data:
sql ='select count(*) as count from assets where wxsn= %s'
p =(param['wxsn'],)
count = db.count(sql,*p)
if count['count'] == 0:
db.insert('assets',**param)
else:
db.update('assets','wxsn="%s"'%param['wxsn'],**param)
def save_idcs(self):
if self.data == None:
return False
db = Custom_MySQL(using='center_app')
for param in self.data:
sql = 'select count(*) as count from idc where prefix = %s'
p = (param['prefix'],)
count = db.count(sql, *p)
# 检查是否存在
if count['count'] == 0:
db.insert('idc', **param)
else:
db.update('idc', 'prefix="%s"' % param['prefix'], **param)
def call_back(self,data=[]):
db = Custom_MySQL(using='center_app')
for result in data:
if result !=[] and result['flag'] == '1':
param = {}
param['is_manage'] = 1
db.update('assets', 'public_ip="%s"' % result['ip'], **param)
db.commit()
else:
print result
param = {}
param['is_manage'] = 0
db.update('assets', 'public_ip="%s"' % result['ip'], **param)
db.commit()
def run(self):
'''
业务入口
'''
api_data = self.get_api_data()
if api_data == None:return
db = Custom_MySQL(using='center_app')
#更新大类中的应用类型
app_type ={}
app_type ={'app_type':','.join(api_data['type'])}
db.update('main_category',' prefix="%s" ' % self.game_code,**app_type)
game = db.get('select id from main_category where prefix="%s"' % self.game_code)
main_category_id = game['id']
#获取区组信息
for dist in api_data['dists']:
print '========'+dist['name']+'+'+dist['code']
sql ='select count(id) as count from sub_category where main_category_id ='+str(main_category_id)+ ' and name="'+dist['name']+'"'
count = db.count(sql)
if count==None:
print 'SQL Error:%s'% sql
return False
#区组更新内容
dist_data ={}
dist_data ={'prefix':dist['code'],
'main_category_id':main_category_id,
'name':dist['name'],
'platform':self.platform}
#如果没有区组信息则保存
if count['count'] == 0:
db.insert('sub_category',**dist_data)
else:
db.update('sub_category',' main_category_id ='+str(main_category_id)+ ' and name="'+dist['name']+'"',**dist_data)
def main():
tables = excel_table_byindex()
db = Custom_MySQL(using='center_app')
for row in tables:
print row
game = row['name'].split('_')[0]
sql ='select id from main_category where prefix= %s'
p =(game,)
result= db.get(sql,*p)
if result:
print result['id']
param={}
param['main_category_id']= result['id']
if row['inner'] !='' and row['pub']!="":
sql='select count(*) as count from assets where inner_ip="%s" or public_ip ="%s"'%(row['inner'],row['pub'])
count = db.count(sql)['count']
if count == 0:
param['inner_ip'] = row['inner']
param['public_ip'] = row['pub']
param['hostname'] = row['name']
param['wxsn'] = row['name']
db.insert('assets',**param)
else:
param['hostname'] = row['name']
db.update('assets','inner_ip="%s"'%row['inner'],**param)
elif row['inner'] !='':
sql='select count(*) as count from assets where inner_ip="%s"'%row['inner']
count = db.count(sql)['count']
if count == 0:
param['inner_ip'] = row['inner']
param['hostname'] = row['name']
param['wxsn'] = row['name']
db.insert('assets',**param)
else:
param['hostname'] = row['name']
db.update('assets','inner_ip="%s"'%row['inner'],**param)
elif row['pub']!="":
sql='select count(*) as count from assets where public_ip="%s"'%row['pub']
count = db.count(sql)['count']
if count ==0:
param['public_ip'] = row['pub']
param['hostname'] = row['name']
param['wxsn'] = row['name']
db.insert('assets',**param)
else:
param['hostname'] = row['name']
db.update('assets','public_ip="%s"'%row['pub'],**param)
else:
print 'pub and inner are both empty'
"" % (pid, log_date, game, platform)
gpresult = mysql.query(gpsql)
if not gpresult:
formal_result = mysql.get(formal_sql)
# 如果为空，则全部完成
if formal_result is not None and formal_result != "":
exec_num = int(formal_result['exec_num'])
in_queue = int(formal_result['in_queue'])
if in_queue == 0:
if exec_num < 3:
exec_num += 1
in_queue = 1
datas = {'exec_num': exec_num, 'in_queue': in_queue}
update_result = mysql.update('dw2dm_log',
' id = %s' % pid, **datas)
if update_result == 1:
run_task.apply_async((formal_result, ),
queue='dw2dm')
else:
print "update data{'exec_num', 'in_queue'} error"
else:
datas = {'status': -1}
mysql.update('dw2dm_log', ' id = %s' % pid, **datas)
mysql.commit()
except Exception as exc:
print(exc)
mysql.rollback()
print "db rollback success"
mysql.close()
def run_task(self, task_param):
mysql = Custom_MySQL(using='hadoops2')
mysql_etl = Custom_MySQL(using='etl_manage')
redis = Custom_Redis(using='etl_task')
mysql.begin()
mysql_etl.begin()
datas = {'status': 0}
where = {'id': int(task_param['id'])}
task_key_exc = ""
try:
hive = Custom_Hive(using='ares_dw')
game = task_param['game']
platform = task_param['platform']
table_name = task_param['table_name']
log_date = task_param['log_date']
prefix_sql = task_param['prefix_sql']
exec_sql = task_param['exec_sql']
post_sql = task_param['post_sql']
date_cycle = task_param['date_cycle']
random_str = str(random.randint(0, 999999999))
stimes = str(int(time.time()))
task_date = task_param['task_date']
task_name = task_param['task_name']
do_rate = task_param['do_rate']
#task_key_tmp = str(game)+str(platform)+str(task_name)+str(date_cycle)+str(do_rate)+str(log_date)+str(task_date)
task_key_tmp = str(task_name) + str(date_cycle) + str(do_rate) + str(
log_date) + str(task_date)
task_key_md5 = hashlib.md5()
task_key_md5.update(task_key_tmp)
task_key_md5_result = task_key_md5.hexdigest()
task_key = "dm2report_new_" + str(game) + "_" + str(
platform) + "_" + str(task_key_md5_result)
task_key_exc = task_key
'''
celery 本身的机制存在漏洞，会将一个已经完成任务再次分配给其他的worker，致使同一个任务执行多次
为防止此种现象，在任务开始执行时，将任务的“唯一标示”写入redis中，标注已执行
'''
#如果task_key is None, 则表示该条任务没有执行过，正常执行即可
#如果task_key = 0, 则表示该条任务上次执行失败，允许重复执行
if redis.get(task_key) == "0" or redis.get(task_key) is None:
tmp_file_dir = "/tmp/tmp/%s/%s/%s" % (game, platform, log_date)
#创建本地目录
if not os.path.exists(tmp_file_dir):
os.makedirs(tmp_file_dir)
tmp_file = "%s/%s_%s_%s_%s.txt" % (tmp_file_dir, table_name,
date_cycle, random_str, stimes)
hql_conf = "SET hive.support.concurrency=false;" \
"SET hive.exec.compress.output=true;" \
"SET mapred.output.compression.codec=com.hadoop.compression.lzo.LzopCodec; "
'''
将任务标示为开始执行：1
'''
datas['status'] = 1
datas['start_time'] = str(
datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where,
**datas)
mysql_etl.commit()
#在redis标注 任务正在执行
redis.set(task_key, 1)
#执行前置sql，将数据临时写入本地，用于mysql load数据
if prefix_sql is not None:
result = mysql.delete_by_sql(prefix_sql)
logger.info('exec prefix_sql: delete old data {0}'.format(
result['output']))
if result['status'] != 0:
logger.error(
'Error Code %s : %s Cmd: %s' %
(result['status'], result['output'], prefix_sql))
'''
执行失败，将其状态标为未执行：0
'''
datas['status'] = 0
#在redis标注 任务未开始执行
redis.set(task_key, 0)
else:
'''
将任务标示为删除临时文件完成：2
'''
datas['status'] = 2
datas.pop('start_time')
mysql_etl.update('dm2report_new_log',
' id = %(id)d' % where, **datas)
mysql_etl.commit()
'''
开始执行hive ql,将数据dump到本地
'''
result = hive.dump(hql_conf + exec_sql, tmp_file)
logger.info('exec exec_sql: dump data {0}'.format(
result['output']))
if result['status'] != 0 or False == os.path.exists(
'%s' % tmp_file):
logger.error(
'Error Code %s : %s Cmd: %s' %
(result['status'], result['output'], exec_sql))
#在redis标注 任务未开始执行
redis.set(task_key, 0)
else:
'''
将任务标示为dump hive数据完成：3
'''
datas['status'] = 3
datas['tmp_file_name'] = tmp_file
mysql_etl.update('dm2report_new_log',
' id = %(id)d' % where, **datas)
mysql_etl.commit()
#执行后置sql
if post_sql is not None:
post_sql = post_sql.replace('{dir_path}', tmp_file)
post_sql = post_sql.replace(
'{table_name}', task_param['table_name'])
post_sql = post_sql.replace(
'{db_name}', task_param['db_name'])
result = mysql.load(post_sql)
logger.info(
'exec post_sql: load data to hdfs {0}'.format(
result['output']))
if result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' %
(result['status'],
result['output'], post_sql))
#在redis标注 任务未开始执行
redis.set(task_key, 0)
else:
'''
将任务标识为录入mysql完成：4
'''
datas['status'] = 4
datas['end_time'] = str(
datetime.datetime.now().strftime(
'%Y-%m-%d %H:%M:%S'))
datas.pop('tmp_file_name')
#在redis标注 任务已成功执行
redis.set(task_key, 2)
else:
logger.error('prefix_sql is null')
datas['status'] = 0
#在redis标注 任务未开始执行
redis.set(task_key, 0)
#如果task_key=2, 则标示该条任务已经运行成功
elif redis.get(task_key) == "2":
datas['status'] = 4
#该条任务正在运行中
else:
return True
'''
将任务标示为：(模拟) 已从任务队列中移除
'''
datas['in_queue'] = 0
update_result = mysql_etl.update('dm2report_new_log',
' id = %(id)d' % where, **datas)
# 如果数据库更新失败，再调用一次。 如果还是失败，等待自动修复机制，但这样的概率应该很小了。
if update_result != 1:
mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where,
**datas)
mysql_etl.commit()
mysql.commit()
mysql_etl.close()
mysql.close()
return True
except Exception as exc:
logger.error('dm2report error: %s' % exc)
mysql_etl.rollback()
mysql.rollback()
redis.set(task_key_exc, 0)
datas = {'in_queue': 0, 'status': 0}
mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
mysql_etl.close()
mysql.close()
raise self.retry(exc=exc, countdown=60)
'''
照理说for里面不会有人抢快照数据，以防万一起动排他锁(使用主键启动行锁)，兄弟们最好别瞎用
'''
#result = mysql.get('select f.*,s.db_name,s.table_name from file2dw_log as f left join structure as s '
result = mysql.get('select f.*,s.db_name,s.table_name,s.flag from file2dw_log as f left join structure as s '
'on f.target_id=s.id where f.id = %(id)s ' % where)
if result is not None:
exec_num = int(result['exec_num'])
in_queue = int(result['in_queue'])
if in_queue == 0:
if exec_num < 3:
exec_num += 1
in_queue = 1
datas = {'exec_num': exec_num, 'in_queue': in_queue}
update_result = mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
# 为防止修改数据库时出现异常，调整为：确认修改字段状态成功后，再加入队列
if update_result == 1:
run_task.apply_async((result,), queue='mergefile2dw')
else:
print "update data{'exec_num', 'in_queue'} error"
else:
datas = {'load_status': -1}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
mysql.commit()
except Exception as exc:
print(exc)
mysql.rollback()
print "db rollback success"
def run_task(self, task_param):
mysql = Custom_MySQL(using='hadoops2')
mysql_etl = Custom_MySQL(using='etl_manage')
redis = Custom_Redis(using='etl_task')
mysql.begin()
mysql_etl.begin()
datas = {'status': 0}
where = {'id': int(task_param['id'])}
task_key_exc = ""
try:
hive = Custom_Hive(using='ares_dw')
game = task_param['game']
platform = task_param['platform']
table_name = task_param['table_name']
log_date = task_param['log_date']
prefix_sql = task_param['prefix_sql']
exec_sql = task_param['exec_sql']
post_sql = task_param['post_sql']
date_cycle = task_param['date_cycle']
random_str = str(random.randint(0, 999999999))
stimes = str(int(time.time()))
task_date = task_param['task_date']
task_name = task_param['task_name']
do_rate = task_param['do_rate']
#task_key_tmp = str(game)+str(platform)+str(task_name)+str(date_cycle)+str(do_rate)+str(log_date)+str(task_date)
task_key_tmp = str(task_name)+str(date_cycle)+str(do_rate)+str(log_date)+str(task_date)
task_key_md5 = hashlib.md5()
task_key_md5.update(task_key_tmp)
task_key_md5_result = task_key_md5.hexdigest()
task_key = "dm2report_new_"+str(game)+"_"+str(platform)+"_"+str(task_key_md5_result)
task_key_exc = task_key
'''
celery 本身的机制存在漏洞，会将一个已经完成任务再次分配给其他的worker，致使同一个任务执行多次
为防止此种现象，在任务开始执行时，将任务的“唯一标示”写入redis中，标注已执行
'''
#如果task_key is None, 则表示该条任务没有执行过，正常执行即可
#如果task_key = 0, 则表示该条任务上次执行失败，允许重复执行
if redis.get(task_key) == "0" or redis.get(task_key) is None:
tmp_file_dir = "/tmp/tmp/%s/%s/%s" % (game, platform, log_date)
#创建本地目录
if not os.path.exists(tmp_file_dir):
os.makedirs(tmp_file_dir)
tmp_file = "%s/%s_%s_%s_%s.txt" % (tmp_file_dir, table_name, date_cycle, random_str, stimes)
hql_conf = "SET hive.support.concurrency=false;" \
"SET hive.exec.compress.output=true;" \
"SET mapred.output.compression.codec=com.hadoop.compression.lzo.LzopCodec; "
'''
将任务标示为开始执行：1
'''
datas['status'] = 1
datas['start_time'] = str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
#在redis标注 任务正在执行
redis.set(task_key, 1)
#执行前置sql，将数据临时写入本地，用于mysql load数据
if prefix_sql is not None:
result = mysql.delete_by_sql(prefix_sql)
logger.info('exec prefix_sql: delete old data {0}'.format(result['output']))
if result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], prefix_sql))
'''
执行失败，将其状态标为未执行：0
'''
datas['status'] = 0
#在redis标注 任务未开始执行
redis.set(task_key, 0)
else:
'''
将任务标示为删除临时文件完成：2
'''
datas['status'] = 2
datas.pop('start_time')
mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
'''
开始执行hive ql,将数据dump到本地
'''
result = hive.dump(hql_conf+exec_sql, tmp_file)
logger.info('exec exec_sql: dump data {0}'.format(result['output']))
if result['status'] != 0 or False == os.path.exists('%s' % tmp_file):
logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], exec_sql))
#在redis标注 任务未开始执行
redis.set(task_key, 0)
else:
'''
将任务标示为dump hive数据完成：3
'''
datas['status'] = 3
datas['tmp_file_name'] = tmp_file
mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
#执行后置sql
if post_sql is not None:
post_sql = post_sql.replace('{dir_path}', tmp_file)
post_sql = post_sql.replace('{table_name}', task_param['table_name'])
post_sql = post_sql.replace('{db_name}', task_param['db_name'])
result = mysql.load(post_sql)
logger.info('exec post_sql: load data to hdfs {0}'.format(result['output']))
if result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], post_sql))
#在redis标注 任务未开始执行
redis.set(task_key, 0)
else:
'''
将任务标识为录入mysql完成：4
'''
datas['status'] = 4
datas['end_time'] = str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
datas.pop('tmp_file_name')
#在redis标注 任务已成功执行
redis.set(task_key, 2)
else:
logger.error('prefix_sql is null')
datas['status'] = 0
#在redis标注 任务未开始执行
redis.set(task_key, 0)
#如果task_key=2, 则标示该条任务已经运行成功
elif redis.get(task_key) == "2":
datas['status'] = 4
#该条任务正在运行中
else:
return True
'''
将任务标示为：(模拟) 已从任务队列中移除
'''
datas['in_queue'] = 0
update_result = mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where, **datas)
# 如果数据库更新失败，再调用一次。 如果还是失败，等待自动修复机制，但这样的概率应该很小了。
if update_result != 1:
mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
mysql.commit()
mysql_etl.close()
mysql.close()
return True
except Exception as exc:
logger.error('dm2report error: %s' % exc)
mysql_etl.rollback()
mysql.rollback()
redis.set(task_key_exc, 0)
datas = {'in_queue': 0, 'status': 0}
mysql_etl.update('dm2report_new_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
mysql_etl.close()
mysql.close()
raise self.retry(exc=exc, countdown=60)
def run_task(self, task_param):
mysql = Custom_MySQL(using='etl_manage')
mysql.begin()
where = {'id': int(task_param['id'])}
try:
dir_param = {'game': task_param['game'], 'platform': task_param['platform'],
'log_date': task_param['log_date'], 'log_name': task_param['log_name']}
filename_dict = {'log_name': task_param['log_name'], 'log_time': task_param['log_time'], 'source_ip':task_param['source_ip']}
index_dict = {'db_name': task_param['db_name'], 'table_name': task_param['table_name'], 'platform': task_param['platform'],
'log_date': datetime.datetime.strptime(task_param['log_date'], '%Y%m%d').strftime("%Y-%m-%d")}
partition = {'platform': task_param['platform'], 'log_date': datetime.datetime.strptime(task_param['log_date'], '%Y%m%d').strftime("%Y-%m-%d")}
log_dir = "/%(game)s/%(platform)s/%(log_date)s/%(log_name)s/" % dir_param
lzo_file_name = "%(log_name)s_%(source_ip)s_%(log_time)s.txt.lzo" % filename_dict
index_dir_name = "%(db_name)s.db/%(table_name)s/plat_form=%(platform)s/log_date=%(log_date)s/" % index_dict
partition_name = "plat_form='%(platform)s',log_date='%(log_date)s'" % partition
project_path = os.getcwd()
local_log_dir = '/disk1/tmp_data'+log_dir
logger.info('local_log_dir: {0}'.format(local_log_dir))
#判断要录入hive 中的文件知否存在，存在则执行
if os.path.exists('%s%s' % (local_log_dir, lzo_file_name)):
'''
将任务标识为开始执行：1
'''
datas = {'load_status': 1}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
#执行load之前，删除同名文件，防止同一个文件出现两次的可能
cmd_remove = '/bin/bash %s/file2dw/remove_damaged_file.sh %s %s' % (project_path, index_dir_name, lzo_file_name)
logger.info('remove damaged files: {0}'.format(cmd_remove))
remove_result = cmd.run(cmd_remove)
if remove_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove))
'''
文件加载到hive中
'''
hive = Custom_Hive(using='ares_dw')
load_sql = task_param['load_sql']
load_sql = load_sql.replace('{dir_path}', local_log_dir+lzo_file_name)
load_sql = load_sql.replace('{table_name}', task_param['table_name'])
load_sql = load_sql.replace('{partition_name}', '%s' % partition_name)
load_sql = load_sql.replace('{db_name}', task_param['db_name'])
logger.info('hive load SQL: {0}'.format(load_sql))
result = hive.load(load_sql)
logger.info('hive load result {0}'.format(result['output']))
if result['status'] == 0:
'''
将任务标识为加载文件完成：2
'''
datas = {'load_status': 2}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
'''
建立索引，否则lzo将不支持split
'''
#print index_dir_name
cmd_index = '/bin/bash %s/file2dw/create_lzo_indexer.sh %s %s' % (project_path, index_dir_name, lzo_file_name)
logger.info('create lzo index: {0}'.format(cmd_index))
index_result = cmd.run(cmd_index)
if index_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (index_result['status'], index_result['output'], cmd_index))
else:
if "create index success" in index_result['output']:
'''
将任务标识为建立lzo索引完成：3
'''
datas = {'load_status': 3}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
else:
'''
如果load数据失败，则删除半途出现错误的文件，方式hive查询的时候报错
'''
cmd_remove = '/bin/bash %s/file2dw/remove_damaged_file.sh %s %s' % (project_path, index_dir_name, lzo_file_name)
logger.info('remove damaged files: {0}'.format(cmd_remove))
remove_result = cmd.run(cmd_remove)
if remove_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove))
else:
'''
将任务标识为未启动，重新执行：0
'''
datas = {'load_status': 0}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
'''
将任务标示为：(模拟) 已从任务队列中移除
'''
datas = {'in_queue': 0}
update_result = mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
# 如果更新失败，则再调用一次，如果还是失败，则等待自动修复机制，但这种概率很小了
if update_result != 1:
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
return True
except Exception as exc:
print (exc)
mysql.rollback()
datas = {'in_queue': 0}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
raise self.retry(exc=exc, countdown=60)
def run_task(self, task_param):
mysql = Custom_MySQL(using='etl_manage')
mysql.begin()
datas = {'etl_status': 0}
where = {'id': int(task_param['id'])}
try:
log_param = {
'game': task_param['game'],
'platform': task_param['platform'],
'log_date': task_param['log_date'],
'log_name': task_param['log_name'],
'log_dir': task_param['log_dir'],
'col_num': task_param['col_num']
}
log_name_param = {
'log_name': task_param['log_name'],
'source_ip': task_param['source_ip'],
'log_time': task_param['log_time']
}
do_rate = task_param['do_rate']
flag = task_param['flag']
log_dir = '%(log_dir)s/%(game)s/%(platform)s/%(log_date)s/%(log_name)s' % log_param
log_name = '%(log_name)s_%(source_ip)s_%(log_time)s' % log_name_param
log_name_notime = '%(log_name)s_%(source_ip)s' % log_name_param
col_num = task_param['col_num']
project_path = os.getcwd()
#判断如果有md5文件和数据文件同时存在，则开始执行
if (do_rate == "1day" and flag == "log") or (
os.path.exists('%s/%s.log.md5' % (log_dir, log_name)) is True
and os.path.exists('%s/%s.log' % (log_dir, log_name)) is True):
#排除同名文件存在的可能，同时为修复执行提供方便
if os.path.exists('%s/%s.txt' % (log_dir, log_name)):
cmd_remove = 'rm -f %s/%s.txt*' % (log_dir, log_name)
logger.info('remove history file: {0}'.format(cmd_remove))
remove_result = cmd.run(cmd_remove)
if remove_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' %
(remove_result['status'],
remove_result['output'], cmd_remove))
'''
将任务标识为开始执行：1
'''
datas['etl_status'] = 1
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
'''
校验数据
'''
#如果是“天”频次的日志数据，则特殊处理，直接将同一ip的合成一个文件，同时执行校验
if do_rate == "1day" and flag == "log":
cmd_merge = '/bin/bash %s/etl_data/merge_data.sh %s %s %s' % (
project_path, log_dir, log_name_notime, col_num)
logger.info('check data: {0}'.format(cmd_merge))
merge_result = cmd.run(cmd_merge)
else:
cmd_merge = '/bin/bash %s/etl_data/check_data.sh %s %s %s' % (
project_path, log_dir, log_name, col_num)
logger.info('check data: {0}'.format(cmd_merge))
merge_result = cmd.run(cmd_merge)
if merge_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' %
(merge_result['status'], merge_result['output'],
cmd_merge))
datas['etl_status'] = 0
else:
'''
读取校验格式后的文件总条数
'''
row = open('%s/%s.txt.row' % (log_dir, log_name)).read()
'''
将文件总条数写入数据库，并将任务标识为为校验已完成：2
'''
datas['etl_status'] = 2
datas['row_num'] = int(row)
#datas = {'etl_status': 2, 'row_num': int(row)}
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
'''
压缩数据
'''
cmd_compress = '/bin/bash %s/etl_data/compress_data.sh %s %s' % (
project_path, log_dir, log_name)
logger.info('compress data: {0}'.format(cmd_compress))
compress_result = cmd.run(cmd_compress)
if compress_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' %
(compress_result['status'],
compress_result['output'], cmd_compress))
datas['etl_status'] = 0
else:
'''
将任务标识为压缩完成：4
'''
datas['etl_status'] = 4
datas.pop('row_num')
mysql.update('etl_data_log', ' id = %(id)d' % where,
**datas)
mysql.commit()
'''
生成MD5文件
'''
cmd_md5 = '/bin/bash %s/etl_data/md5_data.sh %s %s' % (
project_path, log_dir, log_name)
logger.info('md5 data: {0}'.format(cmd_md5))
md5_result = cmd.run(cmd_md5)
if md5_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' %
(md5_result['status'],
md5_result['output'], cmd_md5))
datas['etl_status'] = 0
else:
'''
将任务标识为生成MD5完成(即为校验、合并、压缩均已完成)：6
'''
datas['etl_status'] = 6
'''
执行完毕，模拟从队列中清楚任务:0
'''
datas['in_etl_queue'] = 0
update_result = mysql.update('etl_data_log', ' id = %(id)d' % where,
**datas)
# 如果更新失败，则再调用一次，如果还是失败，则等待自动修复机制，但这种概率很小了
if update_result != 1:
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
return True
except Exception as exc:
logger.error('etl_data error: %s' % exc)
mysql.rollback()
datas = {'in_etl_queue': 0, 'etl_status': 0}
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
raise self.retry(exc=exc, countdown=30)
result = mysql.get(
'select f.*,s.db_name,s.table_name from dm2report_log as f left join structure as s '
'on f.target_id=s.id where f.id = %(id)s ' % where)
if result is not None:
exec_num = int(result['exec_num'])
in_queue = int(result['in_queue'])
if in_queue == 0:
if exec_num < 3:
exec_num += 1
in_queue = 1
datas = {'exec_num': exec_num, 'in_queue': in_queue}
update_result = mysql.update('dm2report_log',
' id = %(id)d' % where,
**datas)
# 为防止修改数据库时出现异常，调整为：确认修改字段状态成功后，再加入队列
if update_result == 1:
run_task.apply_async((result, ), queue='dm2report')
else:
print "update data{'exec_num', 'in_queue'} error"
else:
datas = {'status': -1}
mysql.update('dm2report_log', ' id = %(id)d' % where,
**datas)
mysql.commit()
except Exception as exc:
def run_task(self, task_param):
mysql = Custom_MySQL(using='etl_manage')
hive = Custom_Hive(using='ares_dw')
redis = Custom_Redis(using='etl_manage')
mysql.begin()
where = {'id': int(task_param['id'])}
try:
log_date = datetime.datetime.strptime(task_param['log_date'],
'%Y%m%d').strftime("%Y-%m-%d")
log_date_1 = (
datetime.datetime.strptime(task_param['log_date'], '%Y%m%d') -
datetime.timedelta(days=1)).strftime("%Y-%m-%d")
dir_param = {
'game': task_param['game'],
'platform': task_param['platform'],
'log_date': task_param['log_date'],
'log_name': task_param['log_name']
}
filename_dict = {
'log_name': task_param['log_name'],
'log_time': task_param['log_time']
}
index_dict = {
'db_name': task_param['db_name'],
'table_name': task_param['table_name'],
'platform': task_param['platform'],
'log_date': log_date
}
partition = {'platform': task_param['platform'], 'log_date': log_date}
index_dict_1 = {
'db_name': task_param['db_name'],
'table_name': task_param['table_name'],
'platform': task_param['platform'],
'log_date': log_date_1
}
partition_1 = {
'platform': task_param['platform'],
'log_date': log_date_1
}
log_dir = "/%(game)s/%(platform)s/%(log_date)s/%(log_name)s" % dir_param
flag = task_param['flag']
file_name = "%(log_name)s" % filename_dict
file_name_txt = "%(log_name)s_%(log_time)s_result.txt" % filename_dict
file_name_lzo = "%(log_name)s_%(log_time)s_result.txt.lzo" % filename_dict
file_name_row = "%(log_name)s_%(log_time)s_result.txt.row" % filename_dict
index_dir_name = "%(db_name)s.db/%(table_name)s/plat_form=%(platform)s/log_date=%(log_date)s/" % index_dict
partition_name = "plat_form='%(platform)s',log_date='%(log_date)s'" % partition
project_path = os.getcwd()
log_time = task_param['log_time']
do_rate = task_param['do_rate']
#if flag == "snap" or (do_rate == "1day" and flag == "log")
if flag == "snap":
index_dir_name = "%(db_name)s.db/%(table_name)s/plat_form=%(platform)s/log_date=%(log_date)s/" % index_dict_1
partition_name = "plat_form='%(platform)s',log_date='%(log_date)s'" % partition_1
#从redis中，获取当前数据对应存储到哪块磁盘
if redis.get("disk_xml") is None:
disk_tmp = open('/data/etl_manage/conf/disk_game.xml', 'r')
redis.set("disk_xml", str(disk_tmp.read()))
disk_list = str(redis.get("disk_xml"))
root = ET.fromstring(disk_list)
disk = ""
for gameinfo in root.findall('game'):
if gameinfo.get('name') == task_param['game']:
disk = gameinfo.get('disk')
continue
#local_log_dir = '/disk1/tmp_data'+log_dir
local_log_dir = '/' + disk + '/data' + log_dir
logger.info('local_log_dir: {0}'.format(local_log_dir))
#判断目录是否存在
if os.path.exists('%s' % local_log_dir):
#排除同名文件存在的可能，同时为修复执行提供方便
if os.path.exists('%s/%s' % (local_log_dir, file_name_txt)):
cmd_remove = 'rm -f %s/%s*' % (local_log_dir, file_name_txt)
logger.info('remove history file: {0}'.format(cmd_remove))
remove_result = cmd.run(cmd_remove)
if remove_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' %
(remove_result['status'],
remove_result['output'], cmd_remove))
datas = {'load_status': 1}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
mysql.commit()
cmd_merge = '/bin/bash %s/mergefile2dw/merge_data.sh %s %s %s %s %s' \
'' % (project_path, local_log_dir, local_log_dir, file_name, do_rate, log_time)
logger.info('merge data: {0}'.format(cmd_merge))
merge_result = cmd.run(cmd_merge)
logger.info('merge data result {0}'.format(merge_result['output']))
if merge_result['status'] == 0:
#读取总条数
row = open('%s/%s' % (local_log_dir, file_name_row)).read()
'''
合并数据完成：2
'''
datas = {'load_status': 2, 'row_num': int(row)}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
mysql.commit()
'''
压缩数据
'''
cmd_compress = '/bin/bash %s/mergefile2dw/compress_data.sh %s %s' % (
project_path, local_log_dir, file_name_txt)
logger.info('compress data: {0}'.format(cmd_compress))
compress_result = cmd.run(cmd_compress)
if compress_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' %
(compress_result['status'],
compress_result['output'], cmd_compress))
else:
'''
压缩数据完成：3
'''
datas = {'load_status': 3}
mysql.update('file2dw_log', ' id = %(id)d' % where,
**datas)
mysql.commit()
#执行load之前，删除同名文件，防止同一个文件出现两次的可能
cmd_remove = '/bin/bash %s/mergefile2dw/remove_damaged_file.sh %s %s' % (
project_path, index_dir_name, file_name_lzo)
logger.info('remove damaged files: {0}'.format(cmd_remove))
remove_result = cmd.run(cmd_remove)
if remove_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' %
(remove_result['status'],
remove_result['output'], cmd_remove))
'''
文件加载到hive中
'''
load_sql = task_param['load_sql']
load_sql = load_sql.replace(
'{dir_path}', local_log_dir + "/" + file_name_lzo)
load_sql = load_sql.replace('{table_name}',
task_param['table_name'])
load_sql = load_sql.replace('{partition_name}',
'%s' % partition_name)
load_sql = load_sql.replace('{db_name}',
task_param['db_name'])
logger.info('hive load SQL: {0}'.format(load_sql))
result = hive.load(load_sql)
logger.info('hive load result {0}'.format(
result['output']))
if result['status'] == 0:
'''
将任务标识为加载文件完成：4
'''
datas = {'load_status': 4}
mysql.update('file2dw_log', ' id = %(id)d' % where,
**datas)
mysql.commit()
'''
建立索引，否则lzo将不支持split
'''
#print index_dir_name
cmd_index = '/bin/bash %s/mergefile2dw/create_lzo_indexer.sh %s %s' % (
project_path, index_dir_name, file_name_lzo)
logger.info('create lzo index: {0}'.format(cmd_index))
index_result = cmd.run(cmd_index)
if index_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' %
(index_result['status'],
index_result['output'], cmd_index))
else:
if "create index success" in index_result[
'output']:
'''
将任务标识为建立lzo索引完成：5
'''
datas = {'load_status': 5}
mysql.update('file2dw_log',
' id = %(id)d' % where, **datas)
else:
'''
如果load数据失败，则删除半途出现错误的文件，防止hive查询的时候报错
'''
cmd_remove = '/bin/bash %s/mergefile2dw/remove_damaged_file.sh %s %s' % (
project_path, index_dir_name,
file_name_lzo)
logger.info('remove damaged files: {0}'.format(
cmd_remove))
remove_result = cmd.run(cmd_remove)
if remove_result['status'] != 0:
logger.error(
'Error Code %s : %s Cmd: %s' %
(remove_result['status'],
remove_result['output'], cmd_remove))
else:
logger.error(
'Error Code %s : %s Cmd: %s' %
(result['status'], result['output'], load_sql))
else:
'''
合并数据失败
'''
datas = {'load_status': 0}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
logger.error('Error Code %s : %s Cmd: %s' %
(merge_result['status'], merge_result['output'],
merge_result))
'''
执行完毕，模拟从队列中清楚任务:0
'''
datas = {'in_queue': 0}
update_result = mysql.update('file2dw_log', ' id = %(id)d' % where,
**datas)
# 如果更新失败，则再调用一次，如果还是失败，则等待自动修复机制，但这种概率很小了
if update_result != 1:
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
return True
except Exception as exc:
logger.error('mergefile2dw error: %s' % exc)
mysql.rollback()
datas = {'in_queue': 0, 'load_status': 0}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
raise self.retry(exc=exc, countdown=30)
def run_task(self, task_param):
redis = Custom_Redis(using='etl_task')
mysql = Custom_MySQL(using='etl_manage')
mysql.begin()
datas = {'status': 0}
where = {'id': int(task_param['id'])}
hive = Custom_Hive(using='hadoops2')
task_key_exc = ""
try:
game = task_param['game']
platform = task_param['platform']
task_dict = {'log_name': task_param['log_name'], 'do_rate': task_param['do_rate'],
'log_date': task_param['log_date'], 'task_date': task_param['task_date']}
task_key_tmp = "%(log_name)s%(do_rate)s%(log_date)s%(task_date)s" % task_dict
task_key_md5 = hashlib.md5()
task_key_md5.update(task_key_tmp)
task_key_md5_result = task_key_md5.hexdigest()
task_key = "dw2dm_"+str(game)+"_"+str(platform)+"_"+str(task_key_md5_result)
task_key_exc = task_key
'''
celery 本身的机制存在漏洞，会将一个已经完成任务再次分配给其他的worker，致使同一个任务执行多次
为防止此种现象，在任务开始执行时，将任务的“唯一标示”写入redis中，标注已执行
'''
#如果task_key is None, 则表示该条任务没有执行过，正常执行即可
#如果task_key = 0, 则表示该条任务上次执行失败，允许重复执行
if redis.get(task_key) == "0" or redis.get(task_key) is None:
exec_sql = task_param['exec_sql']
log_date = (datetime.datetime.strptime(task_param['log_date'], '%Y%m%d')).strftime("%Y-%m-%d")
#prefix_sql = task_param['prefix_sql']
#post_sql = task_param['post_sql']
index_dict = {'db_name': task_param['db_name'], 'table_name': task_param['table_name'],
'platform': task_param['platform'], 'log_date': log_date}
index_dir_name = "%(db_name)s.db/%(table_name)s/plat_form=%(platform)s/log_date=%(log_date)s/" % index_dict
#用于删除索引
del_index_dir_name = "%(db_name)s.db/%(table_name)s/plat_form=%(platform)s/log_date=%(log_date)s/" % index_dict
if platform == 'all':
index_dict = {'db_name': task_param['db_name'], 'table_name': task_param['table_name'],
'log_date': log_date}
#用于建立索引(建立索引时，不能使用*通配符，所以仅指定到表名)
index_dir_name = "%(db_name)s.db/%(table_name)s/" % index_dict
#用于删除索引，虽然建立的时候指定到表名，但删除的时候，仅删除当天的
del_index_dir_name = "%(db_name)s.db/%(table_name)s/*/log_date=%(log_date)s/*" % index_dict
hql_conf = "set hive.exec.dynamic.partition.mode=nonstrict;" \
"set hive.exec.compress.output=true;" \
"set mapred.output.compress=true;" \
"set mapred.output.compression.codec=com.hadoop.compression.lzo.LzopCodec;" \
"set io.compression.codecs=com.hadoop.compression.lzo.LzopCodec; "
#获取项目根路径：/data/etl_manage
project_path = os.getcwd()
'''
将任务标示为开始执行：1
'''
datas['status'] = 1
datas['start_time'] = str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
mysql.update('dw2dm_log', ' id = %(id)d' % where, **datas)
mysql.commit()
#在redis标注 任务正在执行
redis.set(task_key, 1)
#执行正式的 select xx insert xx 语句
result = hive.select_insert(hql_conf+exec_sql)
logger.info('exec exec_sql: select xxx insert xxx {0}'.format(result['output']))
if result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], exec_sql))
'''
执行失败，将其状态标为未执行：0
'''
datas['status'] = 0
#在redis标注 任务未执行
redis.set(task_key, 0)
else:
'''
将任务标示为执行hive ql, select xxx insert xxx完成：2
'''
datas['status'] = 2
datas.pop('start_time')
mysql.update('dw2dm_log', ' id = %(id)d' % where, **datas)
mysql.commit()
'''
建立索引，否则lzo将不支持split
'''
cmd_index = '/bin/bash %s/dw2dm/create_lzo_indexer.sh %s' % (project_path, index_dir_name)
logger.info('create lzo index: {0}'.format(cmd_index))
index_result = cmd.run(cmd_index)
if index_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (index_result['status'], index_result['output'], cmd_index))
#在redis标注 任务未执行
redis.set(task_key, 0)
else:
if "create index success" in index_result['output']:
'''
将任务标识为建立lzo索引完成：3
'''
datas['status'] = 3
datas['end_time'] = str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
#在redis标注 任务已完成且成功
redis.set(task_key, 2)
else:
'''
如果load数据失败，则删除半途出现错误的文件，防止hive查询的时候报错
'''
cmd_remove = '/bin/bash %s/dw2dm/remove_damaged_file.sh %s' % (project_path, del_index_dir_name)
logger.info('remove damaged files: {0}'.format(cmd_remove))
remove_result = cmd.run(cmd_remove)
if remove_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove))
#在redis标注 任务未执行
redis.set(task_key, 0)
#如果task_key=2, 则标示该条任务已经运行成功
elif redis.get(task_key) == "2":
datas['status'] = 3
#该条任务正在运行中
else:
return True
'''
执行完毕，模拟从队列中清除任务:0
'''
datas['in_queue'] = 0
update_result = mysql.update('dw2dm_log', ' id = %(id)d' % where, **datas)
# 如果更新失败，则再调用一次，如果还是失败，则等待自动修复机制，但这种概率很小了
if update_result != 1:
mysql.update('dw2dm_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
return True
except Exception as exc:
logger.error('dw2dm_log error: %s' % exc)
redis.set(task_key_exc, 0)
mysql.rollback()
datas = {'in_queue': 0, 'status': 0}
mysql.update('dw2dm_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
raise self.retry(exc=exc, countdown=60)
def run_task(self, task_param):
redis = Custom_Redis(using='etl_manage')
mysql = Custom_MySQL(using='etl_manage')
mysql.begin()
datas = {'download_status': 0}
where = {'id': int(task_param['id'])}
local_log_dir = ""
lzo_file_name = ""
try:
dir_param = {'game': task_param['game'], 'platform': task_param['platform'],
'log_date': task_param['log_date'], 'log_name': task_param['log_name']}
filename_dict = {'log_name': task_param['log_name'], 'log_time': task_param['log_time'], 'source_ip': task_param['source_ip']}
log_dir = "/%(game)s/%(platform)s/%(log_date)s/%(log_name)s/" % dir_param
txt_file_name = "%(log_name)s_%(source_ip)s_%(log_time)s.txt" % filename_dict
lzo_file_name = "%(log_name)s_%(source_ip)s_%(log_time)s.txt.lzo" % filename_dict
md5_file_name = "%(log_name)s_%(source_ip)s_%(log_time)s.txt.lzo.md5" % filename_dict
lzo_download_url = task_param['download_url'].rstrip('/') + log_dir + lzo_file_name
md5_download_url = task_param['download_url'].rstrip('/') + log_dir + md5_file_name
#从redis中，获取当前数据对应存储到哪块磁盘
if redis.get("disk_xml") is None:
disk_tmp = open('/data/etl_manage/conf/disk_game.xml', 'r')
redis.set("disk_xml", str(disk_tmp.read()))
disk_list = str(redis.get("disk_xml"))
root = ET.fromstring(disk_list)
disk = ""
for gameinfo in root.findall('game'):
if gameinfo.get('name') == task_param['game']:
disk = gameinfo.get('disk')
continue
#local_log_dir = '/disk1/tmp_data' + log_dir
local_log_dir = '/'+disk+'/data' + log_dir
#创建本地目录
if not os.path.exists(local_log_dir):
os.makedirs(local_log_dir)
#排除同名文件存在的可能，同时为修复执行提供方便
if os.path.exists('%s%s' % (local_log_dir, txt_file_name)):
cmd_remove = 'rm -f %s%s*' % (local_log_dir, txt_file_name)
logger.info('remove history file: {0}'.format(cmd_remove))
remove_result = cmd.run(cmd_remove)
if remove_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove))
'''
下载md5文件,如果md5文件不存在则退出，不再继续执行程序，同时不向数据库写入任何标示
'''
md5_line = ' wget -o /tmp/log/wget_log -O %s%s %s' % (local_log_dir, md5_file_name, md5_download_url)
logger.info('md5 info: {0}'.format(md5_line))
md5_result = cmd.run(md5_line)
if md5_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (md5_result['status'], md5_result['output'], md5_line))
else:
'''
将任务标识为开始执行：1
'''
datas['download_status'] = 1
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
'''
下载数据文件
'''
lzo_line = ' wget -o /tmp/log/wget_log -O %s%s %s' % (local_log_dir, lzo_file_name, lzo_download_url)
logger.info('file info: {0}'.format(lzo_line))
lzo_result = cmd.run(lzo_line)
if lzo_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (lzo_result['status'], lzo_result['output'], lzo_line))
datas['download_status'] = 0
else:
'''
将任务标识为下载完成：2
'''
datas['download_status'] = 2
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
'''
md5校验，如果未通过则不再继续执行程序
'''
check_line = "cat %s%s |grep `md5sum %s%s|cut -d ' ' -f 1`" % (local_log_dir, md5_file_name, local_log_dir, lzo_file_name)
logger.info('md5 or md5 info: {0}'.format(check_line))
check_result = cmd.run(check_line)
if check_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (check_result['status'], check_result['output'], check_line))
datas['download_status'] = 0
else:
'''
lzop解压缩
'''
cmd_line = ' lzop -dP %s%s' % (local_log_dir, lzo_file_name)
logger.info('file info: {0}'.format(cmd_line))
cmd_result = cmd.run(cmd_line)
if cmd_result['status'] != 0:
logger.error('Lzop Code %s : %s Cmd: %s' % (cmd_result['status'], cmd_result['output'], cmd_line))
datas['download_status'] = 0
else:
'''
将任务标识md5一致，完成下载任务：3
'''
datas['download_status'] = 3
'''
将任务标示为：(模拟) 已从任务队列中移除
'''
datas['in_download_queue'] = 0
update_result = mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
# 如果更新失败，则再调用一次，如果还是失败，则等待自动修复机制，但这种概率很小了
if update_result != 1:
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
return True
except Exception as exc:
print (exc)
logger.error('download error : %s' % exc)
mysql.rollback()
kill_proces = "kill -9 `ps -ef |grep wget |grep -v grep |grep '%s%s'|awk '{print $2}'`" % (local_log_dir, lzo_file_name)
cmd.run(kill_proces)
datas = {'in_download_queue': 0, 'download_status': 0}
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
raise self.retry(exc=exc, countdown=60)
def run_task(self, task_param):
mysql = Custom_MySQL(using='hadoops2')
mysql_etl = Custom_MySQL(using='etl_manage')
mysql.begin()
mysql_etl.begin()
where = {'id': int(task_param['id'])}
try:
hive = Custom_Hive(using='ares_dw')
game = task_param['game']
platform = task_param['platform']
table_name = task_param['table_name']
log_date = task_param['log_date']
prefix_sql = task_param['prefix_sql']
exec_sql = task_param['exec_sql']
post_sql = task_param['post_sql']
date_cycle = task_param['date_cycle']
random_str = str(random.randint(0, 999999999))
stimes = str(int(time.time()))
tmp_file_dir = "/tmp/tmp/%s/%s/%s" % (game, platform, log_date)
#创建本地目录
if not os.path.exists(tmp_file_dir):
os.makedirs(tmp_file_dir)
tmp_file = "%s/%s_%s_%s_%s.txt" % (tmp_file_dir, table_name,
date_cycle, random_str, stimes)
hql_conf = "SET hive.support.concurrency=false;SET hive.exec.compress.output=true;" \
"SET mapred.output.compression.codec=com.hadoop.compression.lzo.LzopCodec; "
'''
将任务标示为开始执行：1
'''
datas = {
'status':
1,
'start_time':
str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
}
mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
#执行前置sql，将数据临时写入本地，用于mysql load数据
if prefix_sql is not None:
result = mysql.delete_by_sql(prefix_sql)
logger.info('exec prefix_sql: delete old data {0}'.format(
result['output']))
if result['status'] == 0:
'''
将任务标示为删除临时文件完成：2
'''
datas = {'status': 2}
mysql_etl.update('dm2report_log', ' id = %(id)d' % where,
**datas)
mysql_etl.commit()
'''
开始执行hive ql,将数据dump到本地
'''
result = hive.dump(hql_conf + exec_sql, tmp_file)
logger.info('exec exec_sql: dump data {0}'.format(
result['output']))
if result['status'] == 0 and True == os.path.exists(
'%s' % tmp_file):
'''
将任务标示为dump hive数据完成：3
'''
datas = {'status': 3, 'tmp_file_name': tmp_file}
mysql_etl.update('dm2report_log', ' id = %(id)d' % where,
**datas)
mysql_etl.commit()
#执行后置sql
if post_sql is not None:
post_sql = post_sql.replace('{dir_path}', tmp_file)
post_sql = post_sql.replace('{table_name}',
task_param['table_name'])
post_sql = post_sql.replace('{db_name}',
task_param['db_name'])
result = mysql.load(post_sql)
logger.info(
'exec post_sql: load data to hdfs {0}'.format(
result['output']))
if result['status'] == 0:
'''
将任务标识为录入mysql完成：4
'''
datas = {
'status':
4,
'end_time':
str(datetime.datetime.now().strftime(
'%Y-%m-%d %H:%M:%S'))
}
mysql_etl.update('dm2report_log',
' id = %(id)d' % where, **datas)
else:
logger.error(
'Error Code %s : %s Cmd: %s' %
(result['status'], result['output'], post_sql))
else:
logger.error(
'Error Code %s : %s Cmd: %s' %
(result['status'], result['output'], exec_sql))
else:
logger.error('Error Code %s : %s Cmd: %s' %
(result['status'], result['output'], prefix_sql))
'''
执行失败，将其状态标为未执行：0
'''
datas = {'status': 0}
mysql_etl.update('dm2report_log', ' id = %(id)d' % where,
**datas)
'''
将任务标示为：(模拟) 已从任务队列中移除
'''
datas = {'in_queue': 0}
update_result = mysql_etl.update('dm2report_log',
' id = %(id)d' % where, **datas)
# 如果数据库更新失败，再调用一次。 如果还是失败，等待自动修复机制，但这样的概率应该很小了。
if update_result != 1:
mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
mysql.commit()
mysql_etl.close()
mysql.close()
return True
except Exception as exc:
logger.error('dm2report error: %s' % exc)
mysql_etl.rollback()
mysql.rollback()
datas = {'in_queue': 0, 'status': 0}
mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
mysql_etl.close()
mysql.close()
raise self.retry(exc=exc, countdown=60)
def run_task(self, task_param):
mysql = Custom_MySQL(using='hadoops2')
mysql_etl = Custom_MySQL(using='etl_manage')
mysql.begin()
mysql_etl.begin()
where = {'id': int(task_param['id'])}
try:
hive = Custom_Hive(using='ares_dw')
game = task_param['game']
platform = task_param['platform']
table_name = task_param['table_name']
log_date = task_param['log_date']
prefix_sql = task_param['prefix_sql']
exec_sql = task_param['exec_sql']
post_sql = task_param['post_sql']
date_cycle = task_param['date_cycle']
random_str = str(random.randint(0, 999999999))
stimes = str(int(time.time()))
tmp_file_dir = "/tmp/tmp/%s/%s/%s" % (game, platform, log_date)
#创建本地目录
if not os.path.exists(tmp_file_dir):
os.makedirs(tmp_file_dir)
tmp_file = "%s/%s_%s_%s_%s.txt" % (tmp_file_dir, table_name, date_cycle, random_str, stimes)
hql_conf = "SET hive.support.concurrency=false;SET hive.exec.compress.output=true;" \
"SET mapred.output.compression.codec=com.hadoop.compression.lzo.LzopCodec; "
'''
将任务标示为开始执行：1
'''
datas = {'status': 1, 'start_time': str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))}
mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
#执行前置sql，将数据临时写入本地，用于mysql load数据
if prefix_sql is not None:
result = mysql.delete_by_sql(prefix_sql)
logger.info('exec prefix_sql: delete old data {0}'.format(result['output']))
if result['status'] == 0:
'''
将任务标示为删除临时文件完成：2
'''
datas = {'status': 2}
mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
'''
开始执行hive ql,将数据dump到本地
'''
result = hive.dump(hql_conf+exec_sql, tmp_file)
logger.info('exec exec_sql: dump data {0}'.format(result['output']))
if result['status'] == 0 and True == os.path.exists('%s' % tmp_file):
'''
将任务标示为dump hive数据完成：3
'''
datas = {'status': 3, 'tmp_file_name': tmp_file}
mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
#执行后置sql
if post_sql is not None:
post_sql = post_sql.replace('{dir_path}', tmp_file)
post_sql = post_sql.replace('{table_name}', task_param['table_name'])
post_sql = post_sql.replace('{db_name}', task_param['db_name'])
result = mysql.load(post_sql)
logger.info('exec post_sql: load data to hdfs {0}'.format(result['output']))
if result['status'] == 0:
'''
将任务标识为录入mysql完成：4
'''
datas = {'status': 4, 'end_time': str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))}
mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas)
else:
logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], post_sql))
else:
logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], exec_sql))
else:
logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], prefix_sql))
'''
执行失败，将其状态标为未执行：0
'''
datas = {'status': 0}
mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas)
'''
将任务标示为：(模拟) 已从任务队列中移除
'''
datas = {'in_queue': 0}
update_result = mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas)
# 如果数据库更新失败，再调用一次。 如果还是失败，等待自动修复机制，但这样的概率应该很小了。
if update_result != 1:
mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
mysql.commit()
mysql_etl.close()
mysql.close()
return True
except Exception as exc:
logger.error('dm2report error: %s' % exc)
mysql_etl.rollback()
mysql.rollback()
datas = {'in_queue': 0, 'status': 0}
mysql_etl.update('dm2report_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
mysql_etl.close()
mysql.close()
raise self.retry(exc=exc, countdown=60)
def run_task(self, task_param):
redis = Custom_Redis(using='etl_manage')
mysql = Custom_MySQL(using='etl_manage')
mysql.begin()
datas = {'download_status': 0}
where = {'id': int(task_param['id'])}
local_log_dir = ""
lzo_file_name = ""
try:
dir_param = {
'game': task_param['game'],
'platform': task_param['platform'],
'log_date': task_param['log_date'],
'log_name': task_param['log_name']
}
filename_dict = {
'log_name': task_param['log_name'],
'log_time': task_param['log_time'],
'source_ip': task_param['source_ip']
}
log_dir = "/%(game)s/%(platform)s/%(log_date)s/%(log_name)s/" % dir_param
txt_file_name = "%(log_name)s_%(source_ip)s_%(log_time)s.txt" % filename_dict
lzo_file_name = "%(log_name)s_%(source_ip)s_%(log_time)s.txt.lzo" % filename_dict
md5_file_name = "%(log_name)s_%(source_ip)s_%(log_time)s.txt.lzo.md5" % filename_dict
lzo_download_url = task_param['download_url'].rstrip(
'/') + log_dir + lzo_file_name
md5_download_url = task_param['download_url'].rstrip(
'/') + log_dir + md5_file_name
#从redis中，获取当前数据对应存储到哪块磁盘
if redis.get("disk_xml") is None:
disk_tmp = open('/data/etl_manage/conf/disk_game.xml', 'r')
redis.set("disk_xml", str(disk_tmp.read()))
disk_list = str(redis.get("disk_xml"))
root = ET.fromstring(disk_list)
disk = ""
for gameinfo in root.findall('game'):
if gameinfo.get('name') == task_param['game']:
disk = gameinfo.get('disk')
continue
#local_log_dir = '/disk1/tmp_data' + log_dir
local_log_dir = '/' + disk + '/data' + log_dir
#创建本地目录
if not os.path.exists(local_log_dir):
os.makedirs(local_log_dir)
#排除同名文件存在的可能，同时为修复执行提供方便
if os.path.exists('%s%s' % (local_log_dir, txt_file_name)):
cmd_remove = 'rm -f %s%s*' % (local_log_dir, txt_file_name)
logger.info('remove history file: {0}'.format(cmd_remove))
remove_result = cmd.run(cmd_remove)
if remove_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' %
(remove_result['status'], remove_result['output'],
cmd_remove))
'''
下载md5文件,如果md5文件不存在则退出，不再继续执行程序，同时不向数据库写入任何标示
'''
md5_line = ' wget -o /tmp/log/wget_log -O %s%s %s' % (
local_log_dir, md5_file_name, md5_download_url)
logger.info('md5 info: {0}'.format(md5_line))
md5_result = cmd.run(md5_line)
if md5_result['status'] != 0:
logger.error(
'Error Code %s : %s Cmd: %s' %
(md5_result['status'], md5_result['output'], md5_line))
else:
'''
将任务标识为开始执行：1
'''
datas['download_status'] = 1
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
'''
下载数据文件
'''
lzo_line = ' wget -o /tmp/log/wget_log -O %s%s %s' % (
local_log_dir, lzo_file_name, lzo_download_url)
logger.info('file info: {0}'.format(lzo_line))
lzo_result = cmd.run(lzo_line)
if lzo_result['status'] != 0:
logger.error(
'Error Code %s : %s Cmd: %s' %
(lzo_result['status'], lzo_result['output'], lzo_line))
datas['download_status'] = 0
else:
'''
将任务标识为下载完成：2
'''
datas['download_status'] = 2
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
'''
md5校验，如果未通过则不再继续执行程序
'''
check_line = "cat %s%s |grep `md5sum %s%s|cut -d ' ' -f 1`" % (
local_log_dir, md5_file_name, local_log_dir, lzo_file_name)
logger.info('md5 or md5 info: {0}'.format(check_line))
check_result = cmd.run(check_line)
if check_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' %
(check_result['status'],
check_result['output'], check_line))
datas['download_status'] = 0
else:
'''
lzop解压缩
'''
cmd_line = ' lzop -dP %s%s' % (local_log_dir,
lzo_file_name)
logger.info('file info: {0}'.format(cmd_line))
cmd_result = cmd.run(cmd_line)
if cmd_result['status'] != 0:
logger.error('Lzop Code %s : %s Cmd: %s' %
(cmd_result['status'],
cmd_result['output'], cmd_line))
datas['download_status'] = 0
else:
'''
将任务标识md5一致，完成下载任务：3
'''
datas['download_status'] = 3
'''
将任务标示为：(模拟) 已从任务队列中移除
'''
datas['in_download_queue'] = 0
update_result = mysql.update('etl_data_log', ' id = %(id)d' % where,
**datas)
# 如果更新失败，则再调用一次，如果还是失败，则等待自动修复机制，但这种概率很小了
if update_result != 1:
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
return True
except Exception as exc:
print(exc)
logger.error('download error : %s' % exc)
mysql.rollback()
kill_proces = "kill -9 `ps -ef |grep wget |grep -v grep |grep '%s%s'|awk '{print $2}'`" % (
local_log_dir, lzo_file_name)
cmd.run(kill_proces)
datas = {'in_download_queue': 0, 'download_status': 0}
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
raise self.retry(exc=exc, countdown=60)
def insert_db(self,main_category_id,sub_category_id,platform,app):
temp_app ={}
temp_app['name']=app['type']+'['+app['memo']+']'
temp_app['platform'] = platform
temp_app['type']=app['type']
temp_app['port']=app['port']
temp_app['main_category_id'] = main_category_id
temp_app['sub_category_id'] = sub_category_id
if app.get('db_type',False):
temp_app['db_type'] = app['db_type']
sql ='select count(*) as count from app_info where '
#同一游戏同一区组
where ='type="%s" and port="%s" and main_category_id="%s" \
and sub_category_id="%s"'% (app['type'],app['port'],main_category_id,sub_category_id)
db = Custom_MySQL(using='center_app')
#处理内网
if app['ip'].split('.')[0] in ['10','172']:
inner_ip ='and inner_ip="%s"'%(app['ip'])
count = db.count(sql+where+inner_ip)
if count==None:
print 'SQL Error:%s'% sql+where+inner_ip
return False
count = count['count']
try:
temp_app['public_ip'] = db.get('select public_ip from assets where inner_ip="%s"'% app['ip'])['public_ip']
except:
pass
if count==0:
temp_app['inner_ip'] = app['ip']
db.insert('app_info',**temp_app)
else:
db.update('app_info',where+inner_ip,**temp_app)
else:
import re
if app['type']=='web':
app['ip'] = app['ip'].replace('http://','').split('/')[0]
p=r'(?<![\.\d])(?:\d{1,3}\.){3}\d{1,3}(?![\.\d])'
mo = re.search(p ,app['ip'])
if not mo:
domain = app['ip'].replace('http://','').split('/')[0]
app['ip'] = db.get('select ip from domain where domain="%s"'% domain)['ip']
temp_app['domain'] = domain
public_ip ='and (public_ip="%s") '%(app['ip'])
count = db.count(sql+where+public_ip)
if count==None:
print 'SQL Error:%s'% sql+where+public_ip
return False
count = count['count']
if count==0:
temp_app['public_ip'] = app['ip']
db.insert('app_info',**temp_app)
else:
db.update('app_info',where+public_ip,**temp_app)
def run_task(self, task_param):
mysql = Custom_MySQL(using='etl_manage')
hive = Custom_Hive(using='ares_dw')
redis = Custom_Redis(using='etl_manage')
mysql.begin()
where = {'id': int(task_param['id'])}
try:
log_date = datetime.datetime.strptime(task_param['log_date'], '%Y%m%d').strftime("%Y-%m-%d")
log_date_1 = (datetime.datetime.strptime(task_param['log_date'], '%Y%m%d')-datetime.timedelta(days=1)).strftime("%Y-%m-%d")
dir_param = {'game': task_param['game'], 'platform': task_param['platform'],
'log_date': task_param['log_date'], 'log_name': task_param['log_name']}
filename_dict = {'log_name': task_param['log_name'], 'log_time': task_param['log_time']}
index_dict = {'db_name': task_param['db_name'], 'table_name': task_param['table_name'],
'platform': task_param['platform'], 'log_date': log_date}
partition = {'platform': task_param['platform'], 'log_date': log_date}
index_dict_1 = {'db_name': task_param['db_name'], 'table_name': task_param['table_name'],
'platform': task_param['platform'], 'log_date': log_date_1}
partition_1 = {'platform': task_param['platform'], 'log_date': log_date_1}
log_dir = "/%(game)s/%(platform)s/%(log_date)s/%(log_name)s" % dir_param
flag = task_param['flag']
file_name = "%(log_name)s" % filename_dict
file_name_txt = "%(log_name)s_%(log_time)s_result.txt" % filename_dict
file_name_lzo = "%(log_name)s_%(log_time)s_result.txt.lzo" % filename_dict
file_name_row = "%(log_name)s_%(log_time)s_result.txt.row" % filename_dict
index_dir_name = "%(db_name)s.db/%(table_name)s/plat_form=%(platform)s/log_date=%(log_date)s/" % index_dict
partition_name = "plat_form='%(platform)s',log_date='%(log_date)s'" % partition
project_path = os.getcwd()
log_time = task_param['log_time']
do_rate = task_param['do_rate']
#if flag == "snap" or (do_rate == "1day" and flag == "log")
if flag == "snap":
index_dir_name = "%(db_name)s.db/%(table_name)s/plat_form=%(platform)s/log_date=%(log_date)s/" % index_dict_1
partition_name = "plat_form='%(platform)s',log_date='%(log_date)s'" % partition_1
#从redis中，获取当前数据对应存储到哪块磁盘
if redis.get("disk_xml") is None:
disk_tmp = open('/data/etl_manage/conf/disk_game.xml', 'r')
redis.set("disk_xml", str(disk_tmp.read()))
disk_list = str(redis.get("disk_xml"))
root = ET.fromstring(disk_list)
disk = ""
for gameinfo in root.findall('game'):
if gameinfo.get('name') == task_param['game']:
disk = gameinfo.get('disk')
continue
#local_log_dir = '/disk1/tmp_data'+log_dir
local_log_dir = '/'+disk+'/data'+log_dir
logger.info('local_log_dir: {0}'.format(local_log_dir))
#判断目录是否存在
if os.path.exists('%s' % local_log_dir):
#排除同名文件存在的可能，同时为修复执行提供方便
if os.path.exists('%s/%s' % (local_log_dir, file_name_txt)):
cmd_remove = 'rm -f %s/%s*' % (local_log_dir, file_name_txt)
logger.info('remove history file: {0}'.format(cmd_remove))
remove_result = cmd.run(cmd_remove)
if remove_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove))
datas = {'load_status': 1}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
mysql.commit()
cmd_merge = '/bin/bash %s/mergefile2dw/merge_data.sh %s %s %s %s %s' \
'' % (project_path, local_log_dir, local_log_dir, file_name, do_rate, log_time)
logger.info('merge data: {0}'.format(cmd_merge))
merge_result = cmd.run(cmd_merge)
logger.info('merge data result {0}'.format(merge_result['output']))
if merge_result['status'] == 0:
#读取总条数
row = open('%s/%s' % (local_log_dir, file_name_row)).read()
'''
合并数据完成：2
'''
datas = {'load_status': 2, 'row_num': int(row)}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
mysql.commit()
'''
压缩数据
'''
cmd_compress = '/bin/bash %s/mergefile2dw/compress_data.sh %s %s' % (project_path, local_log_dir, file_name_txt)
logger.info('compress data: {0}'.format(cmd_compress))
compress_result = cmd.run(cmd_compress)
if compress_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (compress_result['status'], compress_result['output'], cmd_compress))
else:
'''
压缩数据完成：3
'''
datas = {'load_status': 3}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
mysql.commit()
#执行load之前，删除同名文件，防止同一个文件出现两次的可能
cmd_remove = '/bin/bash %s/mergefile2dw/remove_damaged_file.sh %s %s' % (project_path, index_dir_name, file_name_lzo)
logger.info('remove damaged files: {0}'.format(cmd_remove))
remove_result = cmd.run(cmd_remove)
if remove_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove))
'''
文件加载到hive中
'''
load_sql = task_param['load_sql']
load_sql = load_sql.replace('{dir_path}', local_log_dir+"/"+file_name_lzo)
load_sql = load_sql.replace('{table_name}', task_param['table_name'])
load_sql = load_sql.replace('{partition_name}', '%s' % partition_name)
load_sql = load_sql.replace('{db_name}', task_param['db_name'])
logger.info('hive load SQL: {0}'.format(load_sql))
result = hive.load(load_sql)
logger.info('hive load result {0}'.format(result['output']))
if result['status'] == 0:
'''
将任务标识为加载文件完成：4
'''
datas = {'load_status': 4}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
mysql.commit()
'''
建立索引，否则lzo将不支持split
'''
#print index_dir_name
cmd_index = '/bin/bash %s/mergefile2dw/create_lzo_indexer.sh %s %s' % (project_path, index_dir_name, file_name_lzo)
logger.info('create lzo index: {0}'.format(cmd_index))
index_result = cmd.run(cmd_index)
if index_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (index_result['status'], index_result['output'], cmd_index))
else:
if "create index success" in index_result['output']:
'''
将任务标识为建立lzo索引完成：5
'''
datas = {'load_status': 5}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
else:
'''
如果load数据失败，则删除半途出现错误的文件，防止hive查询的时候报错
'''
cmd_remove = '/bin/bash %s/mergefile2dw/remove_damaged_file.sh %s %s' % (project_path, index_dir_name, file_name_lzo)
logger.info('remove damaged files: {0}'.format(cmd_remove))
remove_result = cmd.run(cmd_remove)
if remove_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove))
else:
logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], load_sql))
else:
'''
合并数据失败
'''
datas = {'load_status': 0}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
logger.error('Error Code %s : %s Cmd: %s' % (merge_result['status'], merge_result['output'], merge_result))
'''
执行完毕，模拟从队列中清楚任务:0
'''
datas = {'in_queue': 0}
update_result = mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
# 如果更新失败，则再调用一次，如果还是失败，则等待自动修复机制，但这种概率很小了
if update_result != 1:
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
return True
except Exception as exc:
logger.error('mergefile2dw error: %s' % exc)
mysql.rollback()
datas = {'in_queue': 0, 'load_status': 0}
mysql.update('file2dw_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
raise self.retry(exc=exc, countdown=30)
def run_task(self, task_param):
redis = Custom_Redis(using='etl_manage')
mysql = Custom_MySQL(using='hadoops2')
mysql_etl = Custom_MySQL(using='etl_manage')
mysql.begin()
mysql_etl.begin()
where = {'id': int(task_param['id'])}
datas = {'load_status': 0}
try:
'''
业务代码块放下方
'''
dir_param = {'game': task_param['game'], 'platform': task_param['platform'],
'log_date': task_param['log_date'], 'log_name': task_param['log_name']}
filename_dict = {'log_name': task_param['log_name'], 'log_time': task_param['log_time'], 'source_ip': task_param['source_ip']}
prefix_sql = task_param['prefix_sql']
post_sql = task_param['post_sql']
log_dir = "/%(game)s/%(platform)s/%(log_date)s/%(log_name)s/" % dir_param
file_name = "%(log_name)s_%(source_ip)s_%(log_time)s.txt" % filename_dict
#从redis中，获取当前数据对应存储到哪块磁盘
if redis.get("disk_xml") is None:
disk_tmp = open('/data/etl_manage/conf/disk_game.xml', 'r')
redis.set("disk_xml", str(disk_tmp.read()))
disk_list = str(redis.get("disk_xml"))
root = ET.fromstring(disk_list)
disk = ""
for gameinfo in root.findall('game'):
if gameinfo.get('name') == task_param['game']:
disk = gameinfo.get('disk')
continue
#local_log_dir = '/disk1/tmp_data'+log_dir
local_log_dir = '/'+disk+'/data'+log_dir
#判断要录入的文件是否存在，如果存在则执行，否则不执行
if os.path.exists('%s%s' % (local_log_dir, file_name)):
'''
将任务标识为开始执行：1
'''
datas['load_status'] = 1
mysql_etl.update('file2mysql_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
logger.info('start load data')
#执行前置sql
if prefix_sql is not None and prefix_sql != '':
mysql.query(prefix_sql)
'''
执行load数据进mysql
'''
load_sql = task_param['load_sql']
load_sql = load_sql.replace('{dir_path}', local_log_dir+file_name)
load_sql = load_sql.replace('{table_name}', task_param['table_name'])
load_sql = load_sql.replace('{db_name}', task_param['db_name'])
result = mysql.load(load_sql)
logger.info('load data to mysql: {0}'.format(result['output']))
#判断录入mysql是否成功
if result['status'] == 0:
#执行后置sql
if post_sql is not None and post_sql != '':
post_sql = post_sql.replace('{table_name}', task_param['table_name'])
post_sql = post_sql.replace('{db_name}', task_param['db_name'])
mysql.query(post_sql)
'''
将任务标识为录入mysql完成：3
'''
datas['load_status'] = 3
else:
logger.error('Error Code %s : %s Cmd: %s' % (result['status'], result['output'], load_sql))
'''
录入mysql失败，将任务标示为未执行：0
'''
datas['load_status'] = 0
'''
将任务标示为：(模拟) 已从任务队列中移除
'''
datas['in_queue'] = 0
update_result = mysql_etl.update('file2mysql_log', ' id = %(id)d' % where, **datas)
# 如果更新失败，则再调用一次，如果还是失败，则等待自动修复机制，但这种概率很小了
if update_result != 1:
mysql_etl.update('file2mysql_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
mysql.commit()
mysql_etl.close()
mysql.close()
return True
except Exception as exc:
logger.error('file2mysql error: %s' % exc)
mysql_etl.rollback()
mysql.rollback()
datas = {'in_queue': 0, 'load_status': 0}
mysql_etl.update('file2mysql_log', ' id = %(id)d' % where, **datas)
mysql_etl.commit()
mysql_etl.close()
mysql.close()
raise self.retry(exc=exc, countdown=60)
"" % (pid, log_date, game, platform)
gpresult = mysql.query(gpsql)
if not gpresult:
formal_result = mysql.get(formal_sql)
# 如果为空，则全部完成
if formal_result is not None and formal_result != "":
exec_num = int(formal_result['exec_num'])
in_queue = int(formal_result['in_queue'])
if in_queue == 0:
if exec_num < 3:
exec_num += 1
in_queue = 1
datas = {'exec_num': exec_num, 'in_queue': in_queue}
update_result = mysql.update('dm2report_new_log',
' id = %s' % pid, **datas)
# 为防止修改数据库时出现异常，调整为：确认修改字段状态成功后，再加入队列
if update_result == 1:
run_task.apply_async((formal_result, ),
queue='dm2report_new')
else:
print "update data{'exec_num', 'in_queue'} error"
else:
datas = {'status': -1}
mysql.update('dm2report_new_log', ' id = %s' % pid,
**datas)
mysql.commit()
except Exception as exc:
print(exc)
def run(self):
db = Custom_MySQL(using='log')
status = {'flag': 1}
db.update(
'batch_detail', 'batch_id="%s" and ip ="%s"' %
(self.host['batch_id'], self.host['ip']), **status)
db.commit()
db.close()
#建立连接
self.ssh = paramiko.SSHClient()
try:
#如果没有密码就走public key
if self.host.get('pwd', True) == True:
privatekeyfile = os.path.expanduser('/root/.ssh/id_rsa')
paramiko.RSAKey.from_private_key_file(privatekeyfile)
#缺失host_knows时的处理方法
known_host = "/root/.ssh/known_hosts"
self.ssh.load_system_host_keys(known_host)
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
#os.system('/opt/local/junos/junos')
#连接远程客户机器
self.ssh.connect(hostname=self.host['ip'],
port=int(self.host['port']),
username=self.host['user'],
password=self.host['pwd'],
compress=True,
timeout=60)
#获取远程命令执行结果
stdin, stdout, stderr = self.ssh.exec_command(self.host['cmd'],
bufsize=65535,
timeout=60)
temp = stdout.readlines()
db = Custom_MySQL(using='log')
status = {'flag': 2, 'result': json.dumps(temp)}
db.update(
'batch_detail', 'batch_id="%s" and ip ="%s"' %
(self.host['batch_id'], self.host['ip']), **status)
db.commit()
db.close()
#输出执行结果
self.ssh.close()
except:
self.ssh.close()
#print trace_back()
#以防paramiko本身出问题，这里再用shell运行一次，如果还出问题再确认为问题
cmd = "ssh -p %s -o StrictHostKeyChecking=no %s@%s '%s'" % (
self.host['port'], self.host['user'], self.host['ip'],
self.host['cmd'])
(status, output) = commands.getstatusoutput(cmd)
if status == 0:
db = Custom_MySQL(using='log')
status = {'flag': 2, 'result': json.dumps(output)}
db.update(
'batch_detail', 'batch_id="%s" and ip ="%s"' %
(self.host['batch_id'], self.host['ip']), **status)
db.commit()
db.close()
else:
db = Custom_MySQL(using='log')
status = {'flag': -1, 'result': 'faild'}
db.update(
'batch_detail', 'batch_id="%s" and ip ="%s"' %
(self.host['batch_id'], self.host['ip']), **status)
db.commit()
db.close()
if etl_retry_num < 5:
if in_etl_queue == 0:
if etl_exec_num < 3:
if etl_exec_num == 0:
etl_retry_num += 1
etl_exec_num += 1
in_etl_queue = 1
datas = {
'etl_exec_num': etl_exec_num,
'in_etl_queue': in_etl_queue,
'etl_retry_num': etl_retry_num
}
update_result = mysql.update('etl_data_log',
' id = %(id)d' % where,
**datas)
if update_result == 1:
run_task.apply_async((result, ), queue='etl_data')
else:
print "update data{'etl_exec_num', 'in_etl_queue', 'etl_retry_num'} error"
else:
#任务在该次自动重试中执行失败
datas = {'etl_status': -1}
mysql.update('etl_data_log', ' id = %(id)d' % where,
**datas)
else:
#任务执行彻底失败，不再自动重试
datas = {'etl_status': -2}
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
if result is not None:
etl_exec_num = int(result['etl_exec_num'])
in_etl_queue = int(result['in_etl_queue'])
etl_retry_num = int(result['etl_retry_num'])
if etl_retry_num < 5:
if in_etl_queue == 0:
if etl_exec_num < 3:
if etl_exec_num == 0:
etl_retry_num += 1
etl_exec_num += 1
in_etl_queue = 1
datas = {'etl_exec_num': etl_exec_num, 'in_etl_queue': in_etl_queue, 'etl_retry_num': etl_retry_num}
update_result = mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
if update_result == 1:
run_task.apply_async((result,), queue='etl_data')
else:
print "update data{'etl_exec_num', 'in_etl_queue', 'etl_retry_num'} error"
else:
#任务在该次自动重试中执行失败
datas = {'etl_status': -1}
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
else:
#任务执行彻底失败，不再自动重试
datas = {'etl_status': -2}
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
def run(self):
db = Custom_MySQL(using='log')
status = {'flag':1}
db.update('batch_detail',
'batch_id="%s" and ip ="%s"' % (self.host['batch_id'],self.host['ip']),
**status)
db.commit()
db.close()
try:
#建立连接
self.ssh=paramiko.SSHClient()
#如果没有密码就走public key
if self.host.get('pwd',True) == True:
privatekeyfile = os.path.expanduser('/root/.ssh/id_rsa')
paramiko.RSAKey.from_private_key_file(privatekeyfile)
#缺失host_knows时的处理方法
known_host = "/root/.ssh/known_hosts"
self.ssh.load_system_host_keys(known_host)
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
#os.system('/opt/local/junos/junos')
#连接远程客户机器
self.ssh.connect(
hostname =self.host['ip'],
port =int(self.host['port']),
username =self.host['user'],
password =self.host['pwd'],
compress =True,
timeout =20
)
#获取远程命令执行结果
stdin, stdout, stderr = self.ssh.exec_command(self.host['cmd'],bufsize=65535, timeout=10)
temp = stdout.readlines()
db = Custom_MySQL(using='log')
status = {'flag':2,'result':''.join(temp)}
db.update('batch_detail',
'batch_id="%s" and ip ="%s"' % (self.host['batch_id'],self.host['ip']),**status)
db.commit()
db.close()
if temp ==[]:
self.grandchild.put({'flag':'1','ip':self.host['ip'],'data':temp})
else:
self.grandchild.put({'flag':'0','ip':self.host['ip'],'data':temp})
#输出执行结果
self.ssh.close()
except :
#print trace_back()
#以防paramiko本身出问题，这里再用shell运行一次，如果还出问题再确认为问题
cmd ="ssh -p %s -o StrictHostKeyChecking=no %s@%s %s"%(self.host['port'],self.host['user'],self.host['ip'],self.host['cmd'])
(status,output) = commands.getstatusoutput(cmd)
if status == 0:
db = Custom_MySQL(using='log')
status = {'flag':2,'result':output}
db.update('batch_detail',
'batch_id="%s" and ip ="%s"' % (self.host['batch_id'],self.host['ip']),
**status)
db.commit()
db.close()
self.grandchild.put({'flag':'1','ip':self.host['ip'],'data':output})
else:
db = Custom_MySQL(using='log')
status = {'flag':-1,'result':'faild'}
db.update('batch_detail',
'batch_id="%s" and ip ="%s"' % (self.host['batch_id'],self.host['ip']),
**status)
db.commit()
db.close()
self.grandchild.put({'flag':'0','ip':self.host['ip'],'data':trace_back()})
"" % (pid, log_date, game, platform)
gpresult = mysql.query(gpsql)
if not gpresult:
formal_result = mysql.get(formal_sql)
# 如果为空，则全部完成
if formal_result is not None and formal_result != "":
exec_num = int(formal_result['exec_num'])
in_queue = int(formal_result['in_queue'])
if in_queue == 0:
if exec_num < 3:
exec_num += 1
in_queue = 1
datas = {'exec_num': exec_num, 'in_queue': in_queue}
update_result = mysql.update('dw2dm_log', ' id = %s' % pid, **datas)
if update_result == 1:
run_task.apply_async((formal_result,), queue='dw2dm')
else:
print "update data{'exec_num', 'in_queue'} error"
else:
datas = {'status': -1}
mysql.update('dw2dm_log', ' id = %s' % pid, **datas)
mysql.commit()
except Exception as exc:
print(exc)
mysql.rollback()
print "db rollback success"
mysql.close()
if retry_num < 5:
if in_queue == 0:
if exec_num < 3:
if exec_num == 0:
retry_num += 1
exec_num += 1
in_queue = 1
datas = {
'exec_num': exec_num,
'in_queue': in_queue,
'retry_num': retry_num
}
update_result = mysql.update('file2mysql_log',
' id = %(id)d' % where,
**datas)
# 为防止修改数据库时出现异常，调整为：确认修改字段状态成功后，再加入队列
if update_result == 1:
run_task.apply_async((result, ),
queue='file2mysql')
else:
print "update data{'exec_num', 'in_queue', 'retry_num'} error"
else:
datas = {'load_status': -1}
mysql.update('file2mysql_log', ' id = %(id)d' % where,
**datas)
else:
datas = {'load_status': -2}
mysql.update('file2mysql_log', ' id = %(id)d' % where, **datas)
def run_task(self, task_param):
mysql = Custom_MySQL(using='etl_manage')
mysql.begin()
datas = {'etl_status': 0}
where = {'id': int(task_param['id'])}
try:
log_param = {'game': task_param['game'], 'platform': task_param['platform'],
'log_date': task_param['log_date'], 'log_name': task_param['log_name'],
'log_dir': task_param['log_dir'], 'col_num': task_param['col_num']}
log_name_param = {'log_name': task_param['log_name'], 'source_ip': task_param['source_ip'], 'log_time': task_param['log_time']}
do_rate = task_param['do_rate']
flag = task_param['flag']
log_dir = '%(log_dir)s/%(game)s/%(platform)s/%(log_date)s/%(log_name)s' % log_param
log_name = '%(log_name)s_%(source_ip)s_%(log_time)s' % log_name_param
log_name_notime = '%(log_name)s_%(source_ip)s' % log_name_param
col_num = task_param['col_num']
project_path = os.getcwd()
#判断如果有md5文件和数据文件同时存在，则开始执行
if (do_rate == "1day" and flag == "log") or (os.path.exists('%s/%s.log.md5' % (log_dir, log_name)) is True and os.path.exists('%s/%s.log' % (log_dir, log_name)) is True):
#排除同名文件存在的可能，同时为修复执行提供方便
if os.path.exists('%s/%s.txt' % (log_dir, log_name)):
cmd_remove = 'rm -f %s/%s.txt*' % (log_dir, log_name)
logger.info('remove history file: {0}'.format(cmd_remove))
remove_result = cmd.run(cmd_remove)
if remove_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove))
'''
将任务标识为开始执行：1
'''
datas['etl_status'] = 1
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
'''
校验数据
'''
#如果是“天”频次的日志数据，则特殊处理，直接将同一ip的合成一个文件，同时执行校验
if do_rate == "1day" and flag == "log":
cmd_merge = '/bin/bash %s/etl_data/merge_data.sh %s %s %s' % (project_path, log_dir, log_name_notime, col_num)
logger.info('check data: {0}'.format(cmd_merge))
merge_result = cmd.run(cmd_merge)
else:
cmd_merge = '/bin/bash %s/etl_data/check_data.sh %s %s %s' % (project_path, log_dir, log_name, col_num)
logger.info('check data: {0}'.format(cmd_merge))
merge_result = cmd.run(cmd_merge)
if merge_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (merge_result['status'], merge_result['output'], cmd_merge))
datas['etl_status'] = 0
else:
'''
读取校验格式后的文件总条数
'''
row = open('%s/%s.txt.row' % (log_dir, log_name)).read()
'''
将文件总条数写入数据库，并将任务标识为为校验已完成：2
'''
datas['etl_status'] = 2
datas['row_num'] = int(row)
#datas = {'etl_status': 2, 'row_num': int(row)}
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
'''
压缩数据
'''
cmd_compress = '/bin/bash %s/etl_data/compress_data.sh %s %s' % (project_path, log_dir, log_name)
logger.info('compress data: {0}'.format(cmd_compress))
compress_result = cmd.run(cmd_compress)
if compress_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (compress_result['status'], compress_result['output'], cmd_compress))
datas['etl_status'] = 0
else:
'''
将任务标识为压缩完成：4
'''
datas['etl_status'] = 4
datas.pop('row_num')
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
'''
生成MD5文件
'''
cmd_md5 = '/bin/bash %s/etl_data/md5_data.sh %s %s' % (project_path, log_dir, log_name)
logger.info('md5 data: {0}'.format(cmd_md5))
md5_result = cmd.run(cmd_md5)
if md5_result['status'] != 0:
logger.error('Error Code %s : %s Cmd: %s' % (md5_result['status'], md5_result['output'], cmd_md5))
datas['etl_status'] = 0
else:
'''
将任务标识为生成MD5完成(即为校验、合并、压缩均已完成)：6
'''
datas['etl_status'] = 6
'''
执行完毕，模拟从队列中清楚任务:0
'''
datas['in_etl_queue'] = 0
update_result = mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
# 如果更新失败，则再调用一次，如果还是失败，则等待自动修复机制，但这种概率很小了
if update_result != 1:
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
return True
except Exception as exc:
logger.error('etl_data error: %s' % exc)
mysql.rollback()
datas = {'in_etl_queue': 0, 'etl_status': 0}
mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
mysql.commit()
mysql.close()
raise self.retry(exc=exc, countdown=30)