def update_train_rank(user_id):
logger.info('[Account] update_train_rank <User #{}>'.format(user_id))
user_info = session.query(UserInfo).filter_by(user_id=user_id).first()
if user_info is None:
logger.warn(
"[Account] update_train_rank => UserInfo of User #{} does't exists"
.format(user_id))
return
accounts = session.query(Account).filter_by(user_id=user_id)
ranks = []
for account in accounts:
solved, submitted = account.solved, account.submitted
oj_name = account.oj_name
top_account = session.query(Account).filter_by(oj_name=oj_name)\
.order_by(Account.solved.desc(), Account.submitted.desc())\
.first()
max_solved = max(top_account.solved, solved)
if max_solved == 0:
this_rank = 1000
else:
this_rank = (solved / max_solved) * 1000
ranks.append(this_rank)
logger.debug("[Account] update_train_rank <User #{}> {} => {}".format(
user_id, oj_name, this_rank))
end_rank = sum(ranks) / len(ranks)
user_info.train_rank = end_rank
user_info.save()
logger.info("[Account] update_train_rank success <User #{}> => {}".format(
user_id, end_rank))
async def get_submits(self):
page_size, max_id = 500, 2 ** 31 - 1
while True:
url = self.status_url.format(self.account.nickname, page_size, max_id)
response = await self.fetch(url, method=HttpMethod.GET,
headers=dict(Cookie=self.cookie),
validate_cert=False)
res = json.loads(response.body.decode('utf-8'))
if 'error' in res:
await gen.sleep(60)
continue
status_data = res['data']
if len(status_data) == 0:
break
submits_list = []
for row in status_data:
submit_at = datetime.fromtimestamp(int(row[9]) / 1000)
status = {
'type': DataType.Submit, 'account': self.account,
'status': submit.SubmitStatus.BROKEN,
'run_id': row[0], 'pro_id': row[3], 'lang': row[7], 'run_time': row[5],
'memory': row[6], 'submit_time': submit_at, 'result': row[4],
'code': None, 'origin_oj': row[2]
}
submits_list.append(status)
logger.debug('{} {} Success to get {} new status'.format(self.TAG, self.account, len(submits_list)))
await self.put_queue(submits_list)
max_id = status_data[-1][0] - 1
async def get_submits(self):
start, size = 0, 50
while True:
url = httputil.url_concat(self.status_url,
gen_status_params(self.account.nickname, start, size))
response = await self.fetch(url)
res = json.loads(response.body.decode('utf-8'))
status_data = res['aaData']
if len(status_data) == 0:
break
status_list = []
for row in status_data:
run_time = row[5][:-3] if row[5] != '' else '-1'
memory = row[6][:-3] if row[6] != '' else '-1'
status = {
'type': DataType.Submit, 'account': self.account,
'status': submit.SubmitStatus.BROKEN,
'run_id': row[1], 'pro_id': row[2], 'result': row[3],
'lang': row[4], 'run_time': run_time, 'memory': memory,
'submit_time': row[8], 'code': None
}
status_list.append(status)
logger.debug('{} {} Success to get {} new status'.format(
self.TAG, self.account, len(status_list)))
self.put_queue(status_list)
start += size
async def get_submits(self):
first = ''
while True:
status_list = await self.fetch_status(first)
if not status_list or len(status_list) == 0:
break
logger.debug('{} {} Success to get {} new status'.format(self.TAG, self.account, len(status_list)))
await self.put_queue(status_list)
first = int(status_list[-1]['run_id']) - 1
async def get_submits(self):
start, size = 1, 50
while True:
status_list = await self.get_status(self.account.nickname, start, size)
if not status_list or len(status_list) == 0:
break
logger.debug('{} {} Success to get {} new status'.format(
self.TAG, self.account, len(status_list)))
await self.put_queue(status_list)
start += size
async def get_submits(self):
first = ''
while True:
status_list = await self.fetch_status(first)
if not status_list or len(status_list) == 0:
break
logger.debug('{} {} Success to get {} new status'.format(
self.TAG, self.account, len(status_list)))
await self.put_queue(status_list)
first = int(status_list[-1]['run_id']) - 1
async def get_code(self, run_id, **kwargs):
url = self.code_prefix.format(run_id)
try:
response = await self.load_page(url, {'cookie': self.cookie})
if not response:
logger.error('{} {} Fail to load code {} page'.format(self.TAG, self.account, run_id))
logger.error('{}: response => {}'.format(self.TAG, response))
return False
res = json.loads(response.body.decode('utf-8'))
code = res['source']
logger.debug('{} {} Success to load code {} page'.format(self.TAG, self.account, run_id))
return unescape(code)
except Exception as ex:
logger.error('{} fetch {}\'s {} code error {}'.format(self.TAG, self.account, run_id, ex))
async def get_code(self, run_id, **kwargs):
url = self.source_code_prefix.format(run_id)
try:
response = await self.load_page(url, {'cookie': self.cookie})
if not response:
return False
soup = self.get_lxml_bs4(response.body)
pre_node = soup.find('pre')
if not pre_node:
return False
logger.debug("{} fetch {}\'s code {} success".format(self.TAG, self.account, run_id))
return pre_node.text
except Exception as ex:
logger.error(ex)
logger.error('{} fetch {}\'s {} code error'.format(self.TAG, self.account, run_id))
async def get_code(self, run_id, **kwargs):
url = self.source_code_prefix.format(run_id)
try:
response = await self.load_page(url, {'cookie': self.cookie})
if not response:
return False
soup = self.get_lxml_bs4(response.body)
pre_node = soup.find('pre')
if not pre_node:
return False
logger.debug("{} fetch {}\'s code {} success".format(self.TAG, self.account, run_id))
return pre_node.text
except Exception as ex:
logger.error(ex)
logger.error('{} fetch {}\'s {} code error'.format(self.TAG, self.account, run_id))
async def get_code(self, run_id, **kwargs):
url = self.source_code_prefix.format(run_id)
try:
response = await self.load_page(url, {'Cookie': self.cookie})
if not response:
return False
soup = self.get_lxml_bs4(response.body)
code_area = soup.find('textarea', id='usercode')
if not code_area:
logger.error('{} {} Fail to load code {} page'.format(self.TAG, self.account, run_id))
logger.error('{}: {}'.format(self.TAG, code_area))
return False
code = code_area.text
logger.debug('{} {} Success to load code {} page'.format(self.TAG, self.account, run_id))
return code
except Exception as ex:
logger.error(ex)
logger.error('{} fetch {}\'s {} code error'.format(self.TAG, self.account, run_id))
async def get_code(self, run_id, **kwargs):
url = self.source_code_prefix.format(run_id)
try:
response = await self.load_page(url, {'Cookie': self.cookie})
if not response:
return False
soup = self.get_lxml_bs4(response.body)
code_area = soup.find('textarea', id='usercode')
if not code_area:
logger.error('{} {} Fail to load code {} page'.format(
self.TAG, self.account, run_id))
logger.error('{}: {}'.format(self.TAG, code_area))
return False
code = code_area.text
logger.debug('{} {} Success to load code {} page'.format(
self.TAG, self.account, run_id))
return code
except Exception as ex:
logger.error(ex)
logger.error('{} fetch {}\'s {} code error'.format(
self.TAG, self.account, run_id))
async def login(self):
if self.has_login:
return True
post_body = parse.urlencode({
'username': self.account.nickname,
'password': self.account.password,
'cksave': 1,
'login': '******'
})
response = await self.fetch(self.login_url, method=HttpMethod.POST,
body=post_body)
code = response.code
res = json.loads(response.body.decode('utf-8'))
if code != 200 and code != 302 or res['code'] != 0:
return False
set_cookie = response.headers.get_list('Set-Cookie')
self.cookie = ';'.join(list(
filter(lambda cookie: cookie.find('deleted') == -1, set_cookie)
))
self.has_login = True
logger.debug('{} login success {}'.format(self.TAG, self.account))
return True
async def get_submits(self):
page_size, max_id = 500, 2**31 - 1
while True:
url = self.status_url.format(self.account.nickname, page_size,
max_id)
response = await self.fetch(url,
method=HttpMethod.GET,
headers=dict(Cookie=self.cookie),
validate_cert=False)
res = json.loads(response.body.decode('utf-8'))
if 'error' in res:
await gen.sleep(60)
continue
status_data = res['data']
if len(status_data) == 0:
break
submits_list = []
for row in status_data:
submit_at = datetime.fromtimestamp(int(row[9]) / 1000)
status = {
'type': DataType.Submit,
'account': self.account,
'status': submit.SubmitStatus.BROKEN,
'run_id': row[0],
'pro_id': row[3],
'lang': row[7],
'run_time': row[5],
'memory': row[6],
'submit_time': submit_at,
'result': row[4],
'code': None,
'origin_oj': row[2]
}
submits_list.append(status)
logger.debug('{} {} Success to get {} new status'.format(
self.TAG, self.account, len(submits_list)))
await self.put_queue(submits_list)
max_id = status_data[-1][0] - 1
