def admin_action(conn, user, job_id):
global USER_TABLE
select_sql = "select forbid from %s where username='******'" % (USER_TABLE, user)
forbid = int(sql.fetchone(conn, select_sql)[0])
print forbid
if forbid == 1:
kill_job(job_id)
return
select_sql = "select job_id, job_checksum, status from job_summary where user = '******'" % (user)
results = sql.fetchall(conn, select_sql)
if not results:
LOGGER.info("the user: %s have not any task" % user)
return
select_sql = "select job_checksum from job_summary where user = '******' and job_id = '%s'" % (user, job_id)
checksum = sql.fetchone(conn, select_sql)[0]
select_sql = "select * from job_conf where job_id = '%s'" % job_id
conf = sql.fetchone(conn, select_sql)
count = 0
for jobid, job_checksum, job_status in results:
# print jobid, job_checksum, job_status, checksum
if checksum == job_checksum and job_id != jobid:
select_sql = "select * from job_conf where job_id = '%s'" % jobid
job_conf = sql.fetchone(conn, select_sql)
if 'FAILED' in job_status:
count += 1
if count == HEALTH_POINT:
save_user(conn, user, 1)
kill_job(job_id)
def save_job_conf_file(hdfs_client, conn, log_fullpath, f, jobid):
global CREATE_JOB_CONF
conf_file = os.path.join(log_fullpath, f)
propertys = get_property(hdfs_client.open(conf_file))
propertys["job_id"] = jobid
sql_table = "job_conf"
if not CREATE_JOB_CONF:
create_sql = "CREATE TABLE IF NOT EXISTS '%s' (" % sql_table
for key in propertys.keys():
create_sql += "'%s' varchar(255)," % key
create_sql += "CONSTRAINT job_confId PRIMARY KEY ('job_id'))"
LOGGER.info(create_sql)
sql.create_table(conn, create_sql)
CREATE_JOB_CONF = True
select_sql = "SELECT * FROM %s WHERE job_id = '%s'" % (sql_table, jobid)
LOGGER.info(select_sql)
result = sql.fetchone(conn, select_sql)
if result:
update = ", ".join([ "'%s'='%s'" % (key, propertys[key]) for key in propertys.keys() if key != 'jobId'])
update_sql = "UPDATE %s SET %s WHERE job_id = '%s' " % (sql_table, update, jobid)
LOGGER.info(update_sql)
sql.execute(conn, update_sql)
else:
keys, values = "", ""
items = propertys.items()
keys = ", ".join(map(lambda item: "'%s'" % item[0], items))
values = ", ".join(map(lambda item: "'%s'" % item[1], items))
sql_insert = ("INSERT INTO %s(%s) VALUES (%s)") % (sql_table, keys, values)
LOGGER.info(sql_insert)
sql.execute(conn, sql_insert)
def save_job_summary_file(hdfs_client, conn, log_fullpath, f):
global CREATE_JOB_SUMMARY, USER_TABLE, STATUS_RUNNING
jobid = f[:len(f)-8]
summary_file = os.path.join(log_fullpath, f)
lines = ""
for line in hdfs_client.open(summary_file):
lines += line
job_info = dict((pair.split("=")) for pair in lines.split(","))
job_info['job_id'] = jobid
sql_table = "job_summary"
if not CREATE_JOB_SUMMARY:
job_info['job_checksum'] = 0
job_info['failedMaps'] = 0
job_info['failedReduces'] = 0
create_sql = '''CREATE TABLE IF NOT EXISTS '%s' (''' % sql_table
for key in job_info.keys():
if 'jobId' not in key:
create_sql += "'%s' varchar(255)," % str(key)
create_sql += '''CONSTRAINT 'job_summaryId_pri' PRIMARY KEY('user', 'job_id'),
CONSTRAINT 'job_summaryId_ref' FOREIGN KEY ('user') REFERENCES %s('username')
)''' % USER_TABLE
LOGGER.info("SQL:" + create_sql)
sql.create_table(conn, create_sql)
CREATE_JOB_SUMMARY = True
select_sql = "SELECT * FROM %s WHERE job_id = '%s'" % (sql_table, jobid)
LOGGER.info(select_sql)
result = sql.fetchone(conn, select_sql)
if result:
update = ", ".join([ "%s='%s'" % (key, job_info[key]) for key in job_info.keys() if 'jobId' not in key])
update_sql = "UPDATE %s SET %s WHERE job_id = '%s' " % (sql_table, update, jobid)
LOGGER.info(update_sql)
sql.execute(conn, update_sql)
else:
keys, values = "", ""
items = job_info.items()
# keys = ", ".join(map(lambda item: "'%s'" % item[0], items))
# values = ", ".join(map(lambda item: "'%s'" % item[1], items))
keys = ", ".join(["%s" % key for key in job_info.keys() if 'jobId' not in key])
values = ", ".join(["'%s'" % job_info[key] for key in job_info.keys() if 'jobId' not in key])
sql_insert = ("INSERT INTO %s(%s) VALUES (%s)") % (sql_table, keys, values)
LOGGER.info(sql_insert)
sql.execute(conn, sql_insert)
def save_job_conf(conn, propertys):
sql_table = "job_conf"
jobid = propertys['job_id']
select_sql = "SELECT * FROM %s WHERE job_id = '%s'" % (sql_table, jobid)
LOGGER.info(select_sql)
result = sql.fetchone(conn, select_sql)
if result:
update = ", ".join([ "'%s'='%s'" % (key, propertys[key]) for key in propertys.keys()])
update_sql = "UPDATE %s SET %s WHERE job_id = '%s' " % (sql_table, update, jobid)
LOGGER.info(update_sql)
sql.execute(conn, update_sql)
else:
keys, values = "", ""
items = propertys.items()
keys = ", ".join(map(lambda item: "'%s'" % item[0], items))
values = ", ".join(map(lambda item: "'%s'" % item[1], items))
sql_insert = ("INSERT INTO %s(%s) VALUES (%s)") % (sql_table, keys, values)
LOGGER.info(sql_insert)
sql.execute(conn, sql_insert)
def save_user(conn, username, forbid):
global CREATE_USER, USER_TABLE
create_sql = ''' CREATE TABLE IF NOT EXISTS '%s'(
'uid' INTEGER PRIMARY KEY AUTOINCREMENT,
'username' VARCHAR(255) NOT NULL UNIQUE,
'forbid' TINYINT(8) NOT NULL DEFAULT 0,
'health_point' INTEGER NOT NULL DEFAULT %d)''' % (USER_TABLE, HEALTH_POINT)
if not CREATE_USER:
sql.create_table(conn, create_sql)
CREATE_USER = True
select_sql = "SELECT * FROM %s WHERE %s = '%s' " % (USER_TABLE, "username", username)
LOGGER.info(select_sql)
result = sql.fetchone(conn, select_sql)
if result and forbid == 1:
update_sql = "UPDATE %s SET forbid = '%s' WHERE username = '******'" % (USER_TABLE, forbid, username)
sql.execute(conn, update_sql)
if not result:
insert_sql = "INSERT INTO '%s'(%s, %s) values('%s', '%s')" % (USER_TABLE, 'username', 'forbid', username, forbid)
LOGGER.info(insert_sql)
sql.execute(conn, insert_sql)
def save_job_summary(conn, job_info):
sql_table = "job_summary"
jobid = job_info['job_id']
select_sql = "SELECT * FROM %s WHERE job_id = '%s'" % (sql_table, jobid)
LOGGER.info(select_sql)
result = sql.fetchone(conn, select_sql)
if result:
update = ", ".join([ "%s='%s'" % (key, job_info[key]) for key in job_info.keys() if 'jobId' not in key])
update_sql = "UPDATE %s SET %s WHERE job_id = '%s' " % (sql_table, update, jobid)
LOGGER.info(update_sql)
sql.execute(conn, update_sql)
else:
keys, values = "", ""
items = job_info.items()
# keys = ", ".join(map(lambda item: "'%s'" % item[0], items))
# values = ", ".join(map(lambda item: "'%s'" % item[1], items))
keys = ", ".join(["%s" % key for key in job_info.keys() if 'jobId' not in key])
values = ", ".join(["'%s'" % job_info[key] for key in job_info.keys() if 'jobId' not in key])
sql_insert = ("INSERT INTO %s(%s) VALUES (%s)") % (sql_table, keys, values)
LOGGER.info(sql_insert)
sql.execute(conn, sql_insert)