class DBTaskManager(object):
def __init__(self, username, password):
self.username = username
self.password = password
self._last_check_login = 0
self._last_update_task = 0
self._last_update_downloading_task = 0
self._last_get_task_list = 0
#fix for _last_get_task_list
self.time = time
self._xunlei = LiXianAPI()
self.last_task_id = 0
self.islogin = self._xunlei.login(self.username, self.password)
self._last_check_login = time()
@property
def xunlei(self):
if self._last_check_login + options.check_interval < time():
if not self._xunlei.check_login():
self._xunlei.logout()
self.islogin = self._xunlei.login(self.username, self.password)
self._last_check_login = time()
return self._xunlei
@property
def gdriveid(self):
return self._xunlei.gdriveid
@property
def uid(self):
return self._xunlei.uid
@sqlalchemy_rollback
def _update_tasks(self, tasks):
session = Session()
nm_list = []
bt_list = []
for task in tasks:
if task.task_type in ("bt", "magnet"):
bt_list.append(task.id)
else:
nm_list.append(task.id)
for res in self.xunlei.get_task_process(nm_list, bt_list):
task = self.get_task(res['task_id'])
task.status = res['status']
if task.status == "failed":
task.invalid = True
task.process = res['process']
if res['cid'] and res['lixian_url']:
task.cid = res['cid']
task.lixian_url = res['lixian_url']
task = session.merge(task)
if task.status in ("downloading", "finished"):
if not self._update_file_list(task):
task.status = "failed"
task.invalid = True
session.add(task)
session.commit()
@sqlalchemy_rollback
def _update_task_list(self, limit=10, st=0, ignore=False):
session = Session()
tasks = self.xunlei.get_task_list(limit, st)
for task in tasks[::-1]:
if task['status'] == "finished":
self.last_task_id = task['task_id']
db_task_status = session.query(
db.Task.status).filter(db.Task.id == task['task_id']).first()
if db_task_status and db_task_status[0] == "finished":
continue
db_task = db.Task()
db_task.id = task['task_id']
db_task.create_uid = self.uid
db_task.cid = task['cid']
db_task.url = task['url']
db_task.lixian_url = task['lixian_url']
db_task.taskname = task['taskname']
db_task.task_type = task['task_type']
db_task.status = task['status']
if db_task.status == "failed":
db_task.invalid = True
db_task.process = task['process']
db_task.size = task['size']
db_task.format = task['format']
db_task = session.merge(db_task)
if not self._update_file_list(db_task):
db_task.status = "failed"
db_task.invalid = True
session.add(db_task)
session.commit()
@sqlalchemy_rollback
def _update_file_list(self, task):
session = Session()
if task.task_type == "normal":
tmp_file = dict(task_id=task.id,
cid=task.cid,
url=task.url,
lixian_url=task.lixian_url,
title=task.taskname,
status=task.status,
dirtitle=task.taskname,
process=task.process,
size=task.size,
format=task.format)
files = [
tmp_file,
]
elif task.task_type in ("bt", "magnet"):
try:
files = self.xunlei.get_bt_list(task.id, task.cid)
except Exception, e:
logging.error(repr(e))
return False
for file in files:
if file['lixian_url']:
self.last_task_id = file['task_id']
db_file = db.File()
db_file.id = file['task_id']
db_file.task_id = task.id
db_file.cid = file['cid']
db_file.url = file['url']
db_file._lixian_url = fix_lixian_url(file['lixian_url'])
db_file.title = file['title']
db_file.dirtitle = file['dirtitle']
db_file.status = file['status']
db_file.process = file['process']
db_file.size = file['size']
db_file.format = file['format']
session.merge(db_file)
return True
class DBTaskManager(object):
def __init__(self, username, password):
self.username = username
self.password = password
self._last_check_login = 0
self._last_update_task = 0
self._last_update_downloading_task = 0
self._last_get_task_list = 0
#fix for _last_get_task_list
self.time = time
self._xunlei = LiXianAPI()
self.task_id_sample = set()
self.islogin = self._xunlei.login(self.username, self.password)
self._last_check_login = time()
self._uid = None
self._gdriveid = None
self.last_task_id = 0
@property
def xunlei(self):
if self._last_check_login + options.check_interval < time():
if not self._xunlei.check_login():
self._xunlei.logout()
self.islogin = self._xunlei.login(self.username, self.password)
self._last_check_login = time()
return self._xunlei
@property
def gdriveid(self):
if self._gdriveid:
return self._gdriveid
return self._xunlei.gdriveid
@property
def uid(self):
if self._uid:
return self._uid
return self._xunlei.uid
@sqlalchemy_rollback
def _update_tasks(self, tasks):
session = Session()
nm_list = []
bt_list = []
for task in tasks:
if task.task_type in ("bt", "magnet"):
bt_list.append(task.id)
else:
nm_list.append(task.id)
for res in self.xunlei.get_task_process(nm_list, bt_list):
task = self.get_task(res['task_id'])
task.status = res['status']
if task.status == "failed":
task.invalid = True
task.process = res['process']
if res['cid'] and res['lixian_url']:
task.cid = res['cid']
task.lixian_url = res['lixian_url']
task = session.merge(task)
if task.status in ("downloading", "finished"):
if not self._update_file_list(task):
task.status = "downloading"
session.add(task)
session.commit()
@sqlalchemy_rollback
def _update_task_list(self, limit=10, st=0, ignore=False):
session = Session()
tasks = self.xunlei.get_task_list(limit, st)
for task in tasks[::-1]:
if len(self.task_id_sample) < TASK_ID_SAMPLE_SIZE and task['lixian_url']:
self.task_id_sample.add(task['task_id'])
if not self.last_task_id and task['lixian_url']:
self.last_task_id = task['task_id']
db_task_status = session.query(db.Task.status).filter(
db.Task.id == task['task_id']).first()
if db_task_status and db_task_status[0] == "finished" and self.last_task_id:
continue
db_task = db.Task()
db_task.id = task['task_id']
db_task.create_uid = self.uid
db_task.cid = task['cid']
db_task.url = task['url']
db_task.lixian_url = task['lixian_url']
db_task.taskname = task['taskname']
db_task.task_type = task['task_type']
db_task.status = task['status']
if db_task.status == "failed":
db_task.invalid = True
db_task.process = task['process']
db_task.size = task['size']
db_task.format = task['format']
db_task = session.merge(db_task)
if not self._update_file_list(db_task):
db_task.status = "failed"
db_task.invalid = True
session.add(db_task)
session.commit()
@sqlalchemy_rollback
def _update_file_list(self, task):
session = Session()
if task.task_type == "normal":
tmp_file = dict(
task_id = task.id,
cid = task.cid,
url = task.url,
lixian_url = task.lixian_url,
title = task.taskname,
status = task.status,
dirtitle = task.taskname,
process = task.process,
size = task.size,
format = task.format
)
files = [tmp_file, ]
elif task.task_type in ("bt", "magnet"):
try:
files = self.xunlei.get_bt_list(task.id, task.cid)
except Exception, e:
logging.error(repr(e))
return False
take_task_id_sample = True
for file in files:
if take_task_id_sample and len(self.task_id_sample) < TASK_ID_SAMPLE_SIZE and file['lixian_url']:
self.task_id_sample.add(file['task_id'])
take_task_id_sample = False
if not self.last_task_id and file['lixian_url']:
self.last_task_id = file['task_id']
db_file = db.File()
db_file.id = file['task_id']
db_file.task_id = task.id
db_file.cid = file['cid']
db_file.url = file['url']
db_file._lixian_url = fix_lixian_url(file['lixian_url'])
db_file.title = file['title']
db_file.dirtitle = file['dirtitle']
db_file.status = file['status']
db_file.process = file['process']
db_file.size = file['size']
db_file.format = file['format']
session.merge(db_file)
return True
class DBTaskManager(object):
def __init__(self, username, password):
self.username = username
self.password = password
self._last_check_login = 0
self._last_update_all_task = 0
self._last_update_downloading_task = 0
self._last_get_task_list = 0
#fix for _last_get_task_list
self.time = time
self._last_update_task = 0
self._last_update_task_size = 0
self._last_update_task_lock = Lock()
self._xunlei = LiXianAPI()
verifycode = self._xunlei._get_verifycode(self.username)
if not verifycode:
with open('verifycode.jpg', 'w') as verifycode_fp:
verifycode_fp.write(self._xunlei.verifycode())
verifycode = raw_input('Please open ./verifycode.jpg and enter the verifycode: ')
self.islogin = self._xunlei.login(self.username, self.password, verifycode)
self._last_check_login = time()
@property
def xunlei(self):
if self._last_check_login + options.check_interval < time():
if not self._xunlei.check_login():
self._xunlei.logout()
self.islogin = self._xunlei.login(self.username, self.password)
self._last_check_login = time()
return self._xunlei
@property
def gdriveid(self):
return self._xunlei.gdriveid
@property
def uid(self):
return self._xunlei.uid
def get_vip(self):
return {"uid": self.uid,
"gdriveid": self.gdriveid,
"tid": 1
}
@sqlalchemy_rollback
def _update_tasks(self, tasks):
session = Session()
while tasks:
nm_list = []
bt_list = []
for task in tasks[:100]:
if task.task_type in ("bt", "magnet"):
bt_list.append(task.id)
else:
nm_list.append(task.id)
for res in self.xunlei.get_task_process(nm_list, bt_list):
task = self.get_task(res['task_id'])
if not task: continue
task.status = res['status']
task.process = res['process']
if task.status == "failed":
task.invalid = True
if res['cid'] and res['lixian_url']:
task.cid = res['cid']
task.lixian_url = res['lixian_url']
if task.status in ("downloading", "finished"):
if not self._update_file_list(task):
task.status = "downloading"
session.add(task)
tasks = tasks[100:]
session.commit()
session.close()
@sqlalchemy_rollback
def _update_task_list(self, limit=10, st=0, ignore=False):
now = self.time()
with self._last_update_task_lock:
if now <= self._last_update_task and limit <= self._last_update_task_size:
return
self._last_update_task = self.time()
self._last_update_task_size = limit
session = Session()
tasks = self.xunlei.get_task_list(limit, st)
for task in tasks[::-1]:
db_task_status = session.query(db.Task.status).filter(
db.Task.id == task['task_id']).first()
if db_task_status and db_task_status[0] == "finished":
continue
db_task = self.get_task(int(task['task_id']))
changed = False
if not db_task:
changed = True
db_task = db.Task()
db_task.id = task['task_id']
db_task.create_uid = self.uid
db_task.cid = task['cid']
db_task.url = task['url']
db_task.lixian_url = task['lixian_url']
db_task.taskname = task['taskname'] or "NULL"
db_task.task_type = task['task_type']
db_task.status = task['status']
db_task.invalid = True
db_task.process = task['process']
db_task.size = task['size']
db_task.format = task['format']
else:
db_task.lixian_url = task['lixian_url']
if db_task.status != task['status']:
changed = True
db_task.status = task['status']
if db_task.status == "failed":
db_task.invalid = True
if db_task.process != task['process']:
changed = True
db_task.process = task['process']
session.add(db_task)
if changed and not self._update_file_list(db_task, session):
db_task.status = "failed"
db_task.invalid = True
session.add(db_task)
session.commit()
session.close()
@sqlalchemy_rollback
def _update_file_list(self, task, session=None):
if session is None:
session = Session()
if task.task_type == "normal":
tmp_file = dict(
task_id = task.id,
cid = task.cid,
url = task.url,
lixian_url = task.lixian_url,
title = task.taskname,
status = task.status,
dirtitle = task.taskname,
process = task.process,
size = task.size,
format = task.format
)
files = [tmp_file, ]
elif task.task_type in ("bt", "magnet"):
try:
files = self.xunlei.get_bt_list(task.id, task.cid)
except Exception, e:
logging.error(repr(e))
return False
for file in files:
db_file = session.query(db.File).get(int(file['task_id']))
if not db_file:
db_file = db.File()
db_file.id = file['task_id']
db_file.task_id = task.id
db_file.cid = file['cid']
db_file.url = file['url']
db_file._lixian_url = file['lixian_url'] #fix_lixian_url(file['lixian_url'])
db_file.title = file['title']
db_file.dirtitle = file['dirtitle']
db_file.status = file['status']
db_file.process = file['process']
db_file.size = file['size']
db_file.format = file['format']
else:
db_file._lixian_url = file['lixian_url'] #fix_lixian_url(file['lixian_url'])
db_file.status = file['status']
db_file.process = file['process']
session.add(db_file)
return True
class TaskManager(object):
def __init__(self, username, password):
self.username = username
self.password = password
self._last_check_login = 0
self._tasks = dict()
self._task_list = deque()
self._task_urls = set()
self._last_update_task_list = 0
self._file_list = dict()
self._xunlei = LiXianAPI()
self.islogin = self._xunlei.login(self.username, self.password)
self._last_check_login = time()
@property
def xunlei(self):
if self._last_check_login + options.check_interval < time():
if not self._xunlei.check_login():
self._xunlei.logout()
self.islogin = self._xunlei.login(self.username, self.password)
self._last_check_login = time()
return self._xunlei
@property
def gdriveid(self):
return self.xunlei.gdriveid
def _update_task_list(self, limit=10, st=0, ignore=False):
tasks = self.xunlei.get_task_list(limit, st)
for task in tasks[::-1]:
task['last_update_time'] = datetime.now()
if task['task_id'] not in self._tasks:
if ignore: continue
task['first_seen'] = datetime.now()
self._task_list.appendleft(task)
self._tasks[task['task_id']] = task
self._task_urls.add(task['url'])
else:
self._tasks[task['task_id']].update(task)
def get_task(self, task_id):
if task_id in self._tasks:
return self._tasks[task_id]
else:
return None
def get_task_list(self, start_task_id=0, limit=30):
if self._last_update_task_list + options.finished_task_check_interval < time():
self._update_task_list(options.task_list_limit)
self._last_update_task_list = time()
# skip
pos = iter(self._task_list)
if start_task_id:
for task in pos:
if task['task_id'] == start_task_id:
break
result = []
count = 0
need_update = set()
for task in pos:
if count >= limit: break
result.append(task)
count += 1
if task['status'] != "finished" and task['last_update_time'] \
+ options.downloading_task_check_interval < datetime.now():
need_update.add(task['task_id'])
if need_update:
self._update_task_list(options.task_list_limit, "downloading")
# if updated downloading list and hadn't find task which is
# needed to update, maybe it's finished.
# FIXME: try to get info of a specified task
for task_id in need_update:
task = self.get_task(task_id)
if task['last_update_time'] + options.downloading_task_check_interval < datetime.now():
task['status'] = "finished"
task['process'] = 100
if task['task_id'] in self._file_list:
del self._file_list[task['task_id']]
return result
def get_file_list(self, task_id):
task = self.get_task(task_id)
if not task: return {}
if task_id in self._file_list:
file_list = self._file_list[task_id]
if file_list["last_update_time"] \
+ self._get_check_interval(task['status']) > datetime.now():
return file_list["files"]
if task['task_type'] == "normal":
files = []
tmp_file = dict(
task_id = task['task_id'],
url = task['url'],
lixian_url = task['lixian_url'],
title = task['taskname'],
status = task['status'],
dirtitle = task['taskname'],
process = task['process'],
size = task['size'],
format = task['format']
)
files.append(tmp_file)
elif task['task_type'] in ("bt", "magnet"):
files = self.xunlei.get_bt_list(task['task_id'], task['cid'])
self._file_list[task_id] = {"last_update_time": datetime.now(), "files": files}
return files
def add_task(self, url):
if url in self._task_urls:
return False
url_type = determin_url_type(url)
if url_type in ("bt", "magnet"):
result = self.xunlei.add_bt_task(url)
elif url_type in ("normal", "ed2k", "thunder"):
result = self.xunlei.add_task(url)
else:
result = self.xunlei.add_batch_task([url, ])
if result:
self._update_task_list(5)
return result
def _get_check_interval(self, status):
if status == "finished":
return options.finished_task_check_interval
else:
return options.downloading_task_check_interval
class DBTaskManager(object):
def __init__(self, username, password):
self.username = username
self.password = password
self._last_update_all_task = 0
self._last_update_downloading_task = 0
self._last_get_task_list = 0
#fix for _last_get_task_list
self.time = time
self._last_update_task = 0
self._last_update_task_size = 0
self._last_update_task_lock = Lock()
self._xunlei = LiXianAPI()
_verifycode = self._xunlei._get_verifycode(self.username)
if not _verifycode:
with open('verifycode.jpg', 'w') as verifycode_fp:
verifycode_fp.write(self._xunlei.verifycode())
_verifycode = raw_input(
'Please open ./verifycode.jpg and enter the verifycode: ')
self.islogin = self._xunlei.login(self.username, self.password,
_verifycode)
self._last_login = time()
self._last_check_login = time()
@property
def xunlei(self):
if self._last_login + options.re_login_interval > time() and\
self._last_check_login + options.check_interval > time() and\
self._xunlei.check_login():
self._last_check_login = time()
return self._xunlei
self.re_login()
return self._xunlei
def re_login(self):
self._xunlei.logout()
self.islogin = self._xunlei.login(self.username, self.password)
self._last_login = time()
self._last_check_login = time()
return self.islogin
@property
def gdriveid(self):
return self._xunlei.gdriveid
@property
def uid(self):
return self._xunlei.uid
def get_vip(self):
return {"uid": self.uid, "gdriveid": self.gdriveid, "tid": 1}
@sqlalchemy_rollback
def _update_tasks(self, tasks):
session = Session()
while tasks:
nm_list = []
bt_list = []
for task in tasks[:100]:
if task.task_type in ("bt", "magnet"):
bt_list.append(task.id)
else:
nm_list.append(task.id)
for res in self.xunlei.get_task_process(nm_list, bt_list):
task = self.get_task(res['task_id'])
if not task: continue
task.status = res['status']
task.process = res['process']
if task.status == "failed":
task.invalid = True
if res['cid'] and res['lixian_url']:
task.cid = res['cid']
task.lixian_url = res['lixian_url']
if task.status in ("downloading", "finished"):
if not self._update_file_list(task):
task.status = "downloading"
session.add(task)
tasks = tasks[100:]
session.commit()
session.close()
@sqlalchemy_rollback
def _update_task_list(self, limit=10, st=0, ignore=False):
now = self.time()
with self._last_update_task_lock:
if now <= self._last_update_task and limit <= self._last_update_task_size:
return
self._last_update_task = self.time()
self._last_update_task_size = limit
session = Session()
tasks = self.xunlei.get_task_list(limit, st)
for task in tasks[::-1]:
db_task_status = session.query(db.Task.status).filter(
db.Task.id == task['task_id']).first()
if db_task_status and db_task_status[0] == "finished":
continue
db_task = self.get_task(int(task['task_id']))
changed = False
if not db_task:
changed = True
db_task = db.Task()
db_task.id = task['task_id']
db_task.create_uid = self.uid
db_task.cid = task['cid']
db_task.url = task['url']
db_task.lixian_url = task['lixian_url']
db_task.taskname = unescape(task['taskname']) or "NULL"
db_task.task_type = task['task_type']
db_task.status = task['status']
db_task.invalid = True
db_task.process = task['process']
db_task.size = task['size']
db_task.format = task['format']
else:
db_task.lixian_url = task['lixian_url']
if db_task.status != task['status']:
changed = True
db_task.status = task['status']
if db_task.status == "failed":
db_task.invalid = True
if db_task.process != task['process']:
changed = True
db_task.process = task['process']
session.add(db_task)
if changed and not self._update_file_list(db_task, session):
db_task.status = "failed"
db_task.invalid = True
session.add(db_task)
session.commit()
session.close()
@sqlalchemy_rollback
def _update_file_list(self, task, session=None):
if session is None:
session = Session()
if task.task_type == "normal":
tmp_file = dict(task_id=task.id,
cid=task.cid,
url=task.url,
lixian_url=task.lixian_url,
title=task.taskname,
status=task.status,
dirtitle=task.taskname,
process=task.process,
size=task.size,
format=task.format)
files = [
tmp_file,
]
elif task.task_type in ("bt", "magnet"):
try:
files = self.xunlei.get_bt_list(task.id, task.cid)
except Exception, e:
logging.error(repr(e))
return False
for file in files:
db_file = session.query(db.File).get(int(file['task_id']))
if not db_file:
db_file = db.File()
db_file.id = file['task_id']
db_file.task_id = task.id
db_file.cid = file['cid']
db_file.url = file['url']
db_file._lixian_url = file[
'lixian_url'] #fix_lixian_url(file['lixian_url'])
db_file.title = unescape(file['title'])
db_file.dirtitle = unescape(file['dirtitle'])
db_file.status = file['status']
db_file.process = file['process']
db_file.size = file['size']
db_file.format = file['format']
else:
db_file._lixian_url = file[
'lixian_url'] #fix_lixian_url(file['lixian_url'])
db_file.status = file['status']
db_file.process = file['process']
session.add(db_file)
return True