def admin_action(conn, user, job_id): global USER_TABLE select_sql = "select forbid from %s where username='******'" % (USER_TABLE, user) forbid = int(sql.fetchone(conn, select_sql)[0]) print forbid if forbid == 1: kill_job(job_id) return select_sql = "select job_id, job_checksum, status from job_summary where user = '******'" % (user) results = sql.fetchall(conn, select_sql) if not results: LOGGER.info("the user: %s have not any task" % user) return select_sql = "select job_checksum from job_summary where user = '******' and job_id = '%s'" % (user, job_id) checksum = sql.fetchone(conn, select_sql)[0] select_sql = "select * from job_conf where job_id = '%s'" % job_id conf = sql.fetchone(conn, select_sql) count = 0 for jobid, job_checksum, job_status in results: # print jobid, job_checksum, job_status, checksum if checksum == job_checksum and job_id != jobid: select_sql = "select * from job_conf where job_id = '%s'" % jobid job_conf = sql.fetchone(conn, select_sql) if 'FAILED' in job_status: count += 1 if count == HEALTH_POINT: save_user(conn, user, 1) kill_job(job_id)
def save_job_conf_file(hdfs_client, conn, log_fullpath, f, jobid): global CREATE_JOB_CONF conf_file = os.path.join(log_fullpath, f) propertys = get_property(hdfs_client.open(conf_file)) propertys["job_id"] = jobid sql_table = "job_conf" if not CREATE_JOB_CONF: create_sql = "CREATE TABLE IF NOT EXISTS '%s' (" % sql_table for key in propertys.keys(): create_sql += "'%s' varchar(255)," % key create_sql += "CONSTRAINT job_confId PRIMARY KEY ('job_id'))" LOGGER.info(create_sql) sql.create_table(conn, create_sql) CREATE_JOB_CONF = True select_sql = "SELECT * FROM %s WHERE job_id = '%s'" % (sql_table, jobid) LOGGER.info(select_sql) result = sql.fetchone(conn, select_sql) if result: update = ", ".join([ "'%s'='%s'" % (key, propertys[key]) for key in propertys.keys() if key != 'jobId']) update_sql = "UPDATE %s SET %s WHERE job_id = '%s' " % (sql_table, update, jobid) LOGGER.info(update_sql) sql.execute(conn, update_sql) else: keys, values = "", "" items = propertys.items() keys = ", ".join(map(lambda item: "'%s'" % item[0], items)) values = ", ".join(map(lambda item: "'%s'" % item[1], items)) sql_insert = ("INSERT INTO %s(%s) VALUES (%s)") % (sql_table, keys, values) LOGGER.info(sql_insert) sql.execute(conn, sql_insert)
def save_job_summary_file(hdfs_client, conn, log_fullpath, f): global CREATE_JOB_SUMMARY, USER_TABLE, STATUS_RUNNING jobid = f[:len(f)-8] summary_file = os.path.join(log_fullpath, f) lines = "" for line in hdfs_client.open(summary_file): lines += line job_info = dict((pair.split("=")) for pair in lines.split(",")) job_info['job_id'] = jobid sql_table = "job_summary" if not CREATE_JOB_SUMMARY: job_info['job_checksum'] = 0 job_info['failedMaps'] = 0 job_info['failedReduces'] = 0 create_sql = '''CREATE TABLE IF NOT EXISTS '%s' (''' % sql_table for key in job_info.keys(): if 'jobId' not in key: create_sql += "'%s' varchar(255)," % str(key) create_sql += '''CONSTRAINT 'job_summaryId_pri' PRIMARY KEY('user', 'job_id'), CONSTRAINT 'job_summaryId_ref' FOREIGN KEY ('user') REFERENCES %s('username') )''' % USER_TABLE LOGGER.info("SQL:" + create_sql) sql.create_table(conn, create_sql) CREATE_JOB_SUMMARY = True select_sql = "SELECT * FROM %s WHERE job_id = '%s'" % (sql_table, jobid) LOGGER.info(select_sql) result = sql.fetchone(conn, select_sql) if result: update = ", ".join([ "%s='%s'" % (key, job_info[key]) for key in job_info.keys() if 'jobId' not in key]) update_sql = "UPDATE %s SET %s WHERE job_id = '%s' " % (sql_table, update, jobid) LOGGER.info(update_sql) sql.execute(conn, update_sql) else: keys, values = "", "" items = job_info.items() # keys = ", ".join(map(lambda item: "'%s'" % item[0], items)) # values = ", ".join(map(lambda item: "'%s'" % item[1], items)) keys = ", ".join(["%s" % key for key in job_info.keys() if 'jobId' not in key]) values = ", ".join(["'%s'" % job_info[key] for key in job_info.keys() if 'jobId' not in key]) sql_insert = ("INSERT INTO %s(%s) VALUES (%s)") % (sql_table, keys, values) LOGGER.info(sql_insert) sql.execute(conn, sql_insert)
def save_job_conf(conn, propertys): sql_table = "job_conf" jobid = propertys['job_id'] select_sql = "SELECT * FROM %s WHERE job_id = '%s'" % (sql_table, jobid) LOGGER.info(select_sql) result = sql.fetchone(conn, select_sql) if result: update = ", ".join([ "'%s'='%s'" % (key, propertys[key]) for key in propertys.keys()]) update_sql = "UPDATE %s SET %s WHERE job_id = '%s' " % (sql_table, update, jobid) LOGGER.info(update_sql) sql.execute(conn, update_sql) else: keys, values = "", "" items = propertys.items() keys = ", ".join(map(lambda item: "'%s'" % item[0], items)) values = ", ".join(map(lambda item: "'%s'" % item[1], items)) sql_insert = ("INSERT INTO %s(%s) VALUES (%s)") % (sql_table, keys, values) LOGGER.info(sql_insert) sql.execute(conn, sql_insert)
def save_user(conn, username, forbid): global CREATE_USER, USER_TABLE create_sql = ''' CREATE TABLE IF NOT EXISTS '%s'( 'uid' INTEGER PRIMARY KEY AUTOINCREMENT, 'username' VARCHAR(255) NOT NULL UNIQUE, 'forbid' TINYINT(8) NOT NULL DEFAULT 0, 'health_point' INTEGER NOT NULL DEFAULT %d)''' % (USER_TABLE, HEALTH_POINT) if not CREATE_USER: sql.create_table(conn, create_sql) CREATE_USER = True select_sql = "SELECT * FROM %s WHERE %s = '%s' " % (USER_TABLE, "username", username) LOGGER.info(select_sql) result = sql.fetchone(conn, select_sql) if result and forbid == 1: update_sql = "UPDATE %s SET forbid = '%s' WHERE username = '******'" % (USER_TABLE, forbid, username) sql.execute(conn, update_sql) if not result: insert_sql = "INSERT INTO '%s'(%s, %s) values('%s', '%s')" % (USER_TABLE, 'username', 'forbid', username, forbid) LOGGER.info(insert_sql) sql.execute(conn, insert_sql)
def save_job_summary(conn, job_info): sql_table = "job_summary" jobid = job_info['job_id'] select_sql = "SELECT * FROM %s WHERE job_id = '%s'" % (sql_table, jobid) LOGGER.info(select_sql) result = sql.fetchone(conn, select_sql) if result: update = ", ".join([ "%s='%s'" % (key, job_info[key]) for key in job_info.keys() if 'jobId' not in key]) update_sql = "UPDATE %s SET %s WHERE job_id = '%s' " % (sql_table, update, jobid) LOGGER.info(update_sql) sql.execute(conn, update_sql) else: keys, values = "", "" items = job_info.items() # keys = ", ".join(map(lambda item: "'%s'" % item[0], items)) # values = ", ".join(map(lambda item: "'%s'" % item[1], items)) keys = ", ".join(["%s" % key for key in job_info.keys() if 'jobId' not in key]) values = ", ".join(["'%s'" % job_info[key] for key in job_info.keys() if 'jobId' not in key]) sql_insert = ("INSERT INTO %s(%s) VALUES (%s)") % (sql_table, keys, values) LOGGER.info(sql_insert) sql.execute(conn, sql_insert)