def __init__(self, task_table="tasks", db_conf={}):
self._task_table = task_table
self._db_conf = db_conf
self._mysql = MysqlHelper(**db_conf)
class TaskBal(object):
def __init__(self, task_table="tasks", db_conf={}):
self._task_table = task_table
self._db_conf = db_conf
self._mysql = MysqlHelper(**db_conf)
def create(self, task):
"创建任务, create_table_state 创建表状态, insert_task_state 插入任务列表状态"
table_sql = task["table_sql"]
create_table_state = self._create_table(table_sql)
insert_task_state = 0
if create_table_state:
insert_task_state = self._insert_task(task)
return create_table_state, insert_task_state
def update_info(self, task):
"更新任务状态信息"
sql = "UPDATE " + self._task_table + " SET `done_number`=%(done_number)s, `error_number`=%(error_number), `retry_number`=%(retry_number)s, `all_number`=%(all_number)s, `expect_end_time`=%(all_number)s WHERE `id`=%(id);"
return self._mysql.update(sql, task)
def _create_table(self, sql):
sql = re.sub(re.findall("create +table", sql, re.I)[0], "CREATE TABLE IF NOT EXISTS ", sql)
create_result = self._mysql.insert(sql)
if create_result >= 0:
return 1
else:
return 0
def _insert_task(self, task):
sql = "INSERT INTO " + self._task_table + " (name, caption, start_cmd, config_path, files, create_user, client_create_id, state, parent_task_id, children_task_count, client_get_id, client_get_time,save_type, save_path, table_name, table_sql) VALUES (%(name)s, %(caption)s, %(start_cmd)s, %(config_path)s, %(files)s, %(create_user)s, %(client_create_id)s, %(state)s, %(parent_task_id)s, %(children_task_count)s, %(client_get_id)s, %(client_get_time)s, %(save_type)s, %(save_path)s, %(table_name)s, %(table_sql)s);"
return self._mysql.insert(sql, task)
def cut(self):
"切分任务"
pass
def delete(self):
"删除任务"
pass
def saver(self):
"保存任务"
pass
def get_task(self, client_id, client_name, base_dir):
"""返回未完成任务列表、和新配发的任务。tuple([running_tasks], {new_task})
如果不存在未完成的任务，返回空列表、并且配发空任务"""
running_tasks = self._get_running_task(client_id, client_name)
new_task = {}
if not running_tasks:
new_task = self._get_new_task()
if new_task:
task_id = new_task.get("id")
# -------------------使用静态文件方式访问
# files = [i.strip() for i in new_task.get("files").split("\n")]
# new_task["files"] = files
# -------------------使用静态文件方式访问
# TODO: 此处提取file\config
config_path = new_task.get("config_path")
file_paths = new_task.get("files").split("\n")
task_config = self._load_task_config(config_path, base_dir)
task_files = self._load_task_files(file_paths, base_dir)
new_task["config"] = task_config
new_task["files"] = task_files
# TODO: 此处提取file\config
self._set_task_running(client_id, task_id)
return running_tasks, new_task
def _get_running_task(self, client_id, client_name):
"获取客户端尚未完成的任务"
sql = "select id, name, caption from %s where client_get_id=%s \
and state=1 and create_user='******';" % (self._task_table, client_id, client_name)
logging.info(sql)
tasks = self._mysql.select(sql)
return tasks
def _get_new_task(self):
sql = "select id, name, start_cmd, caption, config_path, files, parent_task_id, children_task_count, save_type, save_path, table_name, table_sql from %s where state=0 and (children_task_count=0 or children_task_count=1) limit 1;" % self._task_table
logging.info(sql)
tasks = self._mysql.select(sql)
return tasks[0] if tasks else {}
def _set_task_running(self, client_id, task_id):
sql = "UPDATE %s SET `state`=1, `client_get_id`=%s, `client_get_time`=NOW() WHERE `id`=%s;" % (
self._task_table, client_id, task_id)
logging.info(sql)
return self._mysql.update(sql)
def close(self, task_id, client_id):
"完成某个任务，并且完成当前任务所对应的子任务、递归完成父任务."
if self._close_children_task(task_id, client_id):
return self._close_parent_task(task_id, client_id)
return False
def _load_task_config(self, config_path, base_dir):
content = self.read_file(os.path.join(base_dir, config_path))
config = {"path": config_path, "content": content}
return config
def _load_task_files(self, file_paths, base_dir):
files = []
for i in file_paths:
content = self.read_file(os.path.join(base_dir, i))
files.append({"path": i, "content": content})
return files
def read_file(self, file_path):
content = ""
with file(file_path)as f:
content = f.read()
return content
def _close_parent_task(self, task_id, client_id):
parent_task_id = self._find_parent_task_id(task_id)
if parent_task_id == 0:
return True
sql = "select id from %s where parent_task_id = %d and c.state in (0, 1);" % (self._task_table, parent_task_id)
logging.info(sql)
# borther_tasks = [{"id":1}]
borther_tasks = self._mysql.select(sql)
if borther_tasks:
# 如果存在兄弟任务未完成，返回False
return False
else:
# 如果所有兄弟任务已完成，将父任务标记完成。递归父任务的兄弟任务。。
self._done(parent_task_id, client_id)
return self._close_parent_task(parent_task_id, client_id)
def _close_children_task(self, task_id, client_id):
sql = "select id from %s where parent_task_id = %d and state in (0, 1);" % (self._task_table, task_id)
logging.info(sql)
# child_tasks = [{"id": 1}, {"id":2}]
child_tasks = self._mysql.select(sql)
for child in child_tasks:
if child.get("id"):
return False
self._done(task_id, client_id)
return True
def _find_parent_task_id(self, task_id):
sql = "select parent_task_id as pid from %s where id=%d;" % (self._task_table, task_id)
logging.info(sql)
reset = self._mysql.select(sql)[0]
return reset.get("pid")
def _done(self, task_id, client_id):
sql = "update %s set state = 10,end_client_id = %d where id = %d;" % (self._task_table, client_id, task_id)
logging.info(sql)
return self._mysql.update(sql)
def get_md5(self, content):
return md5_checker.md5(content)
def save_upload_data(self, save_type, file_name, file_content, base_dir, compress, delete_source=False, table_name=None):
if save_type == ENUM_TASK_SAVE_TYPE.file:
self.save_file(file_name, file_content, base_dir)
return True, 0
elif save_type == ENUM_TASK_SAVE_TYPE.mysql:
self.un_compress(file_name, file_content, compress, base_dir, delete_source=True)
source_file_name = file_name.rstrip(compress)[:-1]
source_file_path = os.path.join(base_dir, source_file_name)
if not os.path.isfile(source_file_path):
return False, 0
with file(source_file_path) as f:
de_content = f.read()
row = self._insert_content_to_db(table_name, de_content)
os.remove(source_file_path)
return True, row
def save_file(self, file_name, content, base_dir=""):
path = os.path.join(base_dir, file_name)
dir_path = os.path.dirname(path)
if not os.path.exists(dir_path):
os.makedirs(dir_path)
with file(path, 'ab')as f:
f.write(content)
return True
def un_compress(self, file_name, file_content, compress_type, base_dir, delete_source=False):
file_path = os.path.join(base_dir, file_name)
with file(file_path, "wb")as f:
f.write(file_content)
os.chmod(file_path, stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH)
compressor.un_compress(file_path, compress_type=compress_type, source_type=ENUM_SOURCE_TYPE.file,
save_dir=base_dir, delete_source=delete_source)
def _insert_content_to_db(self, table_name, content):
add_rows = 0
for line_json in content.splitlines():
line_dict = json.loads(line_json)
sql = self._create_sql(table_name, line_dict)
add_rows += self._insert_data_row(sql)
return add_rows
def _create_sql(self, table_name, data_dict):
sql = "insert `%s`(%s) values('%s')" % (table_name, \
", ".join(data_dict.keys()), \
"', '".join([i for i in data_dict.values()]))
def _insert_data_row(self, sql):
logging.info(sql)
return self._mysql.insert(sql)
class ClientBal(object):
def __init__(self, table_name="clients", db_conf={}):
self._table_name = table_name
self._db_conf = db_conf
self._mysql = MysqlHelper(**db_conf)
def create(self, client_name):
"创建客户端"
client_id = self.get_client_id_by_name(client_name)
if client_id:
return client_id
else:
sql = "INSERT INTO `%s` (`name`) VALUES ('%s');" % (self._table_name, client_name)
client_count = self._mysql.insert(sql)
if client_count > 0:
client_id = self.get_client_id_by_name(client_name)
if client_id:
return client_id
#"创建失败(查找失败)"
return CLIENT_CREATE_ERROR
else:
#"创建失败(插入失败)"
return CLIENT_CREATE_ERROR
def update_info(self, client):
sql = "UPDATE " + self._table_name +" SET cpu=%(cpu)s, mem_us=%(mem_us)s, mem_all=%(mem_all)s, speed_rx=%(speed_rx)s, speed_expect=%(speed_expect)s, driver_us=%(driver_us)s, driver_all=%(driver_us)s WHERE id=%(client_id)s;"
return self._mysql.update(sql, client)
def login(self, client_id, client_name):
"客户端登录"
client = self.get_client(client_id, client_name)
if not client:
#未注册
return 20010#CLIENT_NOT_FIND
client_state = client.get('state')
if client_state == CLIENT_UNPASS:
#未验证
return 20001#CLIENT_UNPASS
elif client_state == CLIENT_PASS:
login_state =client.get("login")
if login_state == CLIENT_LOGGING:
#TODO:已登录的情况，重新登录? 单点登录?
return 20013#CLIENT_LOGGING
elif login_state == CLIENT_UNLOGGING:
#未登录，登录
self._login(client_id)
return 20014#CLIENT_UNLOGGING
else:
logging.error("登录失败, client_id:%s, client_name:%s --> login_state:%s" % (client_id, client_name, login_state))
return 1
else:
logging.error("登录失败, client_id:%s, client_name:%s --> client_state:%s" % (client_id, client_name, client_state))
def get_client_id_by_name(self, client_name):
client = self.get_client(client_name=client_name)
client_id = client.get("id")
if client_id:
return client_id
def get_client(self, client_id=-1, client_name=""):
"检查客户端"
if client_id > 0 and client_name:
sql = "select * from %s where id=%s and name='%s'" % (self._table_name, client_id, client_name)
elif client_id > 0:
sql = "select * from %s where id=%s;" % (self._table_name, client_id)
elif client_name:
sql = "select * from %s where name='%s';" % (self._table_name, client_name)
logging.info(sql)
clients = self._mysql.select(sql)
#clients = [{"id":1, "name":"zhipeng", "state":"0", "login":"******"}]
return clients[0] if clients else {}
def _login(self, client_id):
sql = "UPDATE %s SET `login`=%d, `last_time`=now() WHERE `id`=%s;" % (self._table_name, CLIENT_LOGGING, client_id)
return self._mysql.update(sql)
def __init__(self, table_name="clients", db_conf={}):
self._table_name = table_name
self._db_conf = db_conf
self._mysql = MysqlHelper(**db_conf)
class Manager(object):
def __init__(self, task_table="tasks", db_conf={}):
self._task_table = task_table
self._db_conf = db_conf
self._mysql = MysqlHelper(**db_conf)
def create(self):
"创建任务"
pass
def cut(self):
"切分任务"
pass
def update(self):
"更新任务"
pass
def delete(self):
"删除任务"
pass
def saver(self):
"保存任务"
pass
#
# def get_task(self, task_id, client_id, task_name):
# sql = "select 1 from tasks where task_id=%d and client_get_id=%d and task_name='%s';" % (task_id, client_id, task_name)
def close(self, task_id, client_id):
"完成某个任务，并且完成当前任务所对应的子任务、递归完成父任务."
if self._close_children_task(task_id, client_id):
return self._close_parent_task(task_id, client_id)
return False
def _close_parent_task(self, task_id, client_id):
parent_task_id = self._find_parent_task_id(task_id)
if parent_task_id == 0:
return True
sql = "select id from %s where parent_task_id = %d and c.state in (0, 1);" % (self._task_table, parent_task_id)
logging.info(sql)
# borther_tasks = [{"id":1}]
borther_tasks = self._mysql.select(sql)
if borther_tasks:
# 如果存在兄弟任务未完成，返回False
return False
else:
# 如果所有兄弟任务已完成，将父任务标记完成。递归父任务的兄弟任务。。
self._done(parent_task_id, client_id)
return self._close_parent_task(parent_task_id, client_id)
def _close_children_task(self, task_id, client_id):
sql = "select id from %s where parent_task_id = %d and state in (0, 1);" % (self._task_table, task_id)
logging.info(sql)
# child_tasks = [{"id": 1}, {"id":2}]
child_tasks = self._mysql.select(sql)
for child in child_tasks:
if child.get("id"):
return False
self._done(task_id, client_id)
return True
def _find_parent_task_id(self, task_id):
sql = "select parent_task_id as pid from %s where id=%d;" % (self._task_table, task_id)
logging.info(sql)
reset = self._mysql.select(sql)[0]
return reset.get("pid")
def _done(self, task_id, client_id):
sql = "update %s set state = 10,end_client_id = %d where id = %d;" % (self._task_table, client_id, task_id)
logging.info(sql)
return self._mysql.update(sql)
def get_md5(self, content, is_file=False):
return md5_checker.md5(content)
def un_compress(self, content, compress_type):
return compressor.un_compress(content, compress_type=compress_type)
def insert_data_row(self, sql):
logging.info(sql)
return self._mysql.insert(sql)
