def get_url(url, username, request_id, action, type, isdir, is_multilayer):
"""
从RCMS获取用户的频道信息，匹配出channel_code
:param url:
:param username:
:param request_id:
:param action:
:param isdir:
:return:
"""
if isdir:
#检查dir情况下，url是否合法，不合法则变为url
if not url.endswith('/'):
logger.info('get url url is not dir: url %s isdir %s' %(url, isdir))
isdir = False
# isValid, is_multilayer, channel_code, ignore_case = rcmsapi.isValidUrl(username, url)
# ignore_case is mean is not ignore case
isValid, channel_code, ignore_case = True, None, False
#检查任务优先级
high_priority = False
if isValid:
high_priority = is_refresh_high_priority(channel_code)
return {"r_id": request_id, "url": url, "ignore_case": ignore_case, "status": 'PROGRESS' if isValid else 'INVALID',
"isdir": isdir, "username": username, "parent": username, "created_time": datetime.now(), "action": action,
"is_multilayer": is_multilayer, "channel_code": channel_code, 'type': type, 'high_priority':high_priority,
"channel_name":get_channelname(url)}
def submit(refresh_task):
'''
提交任务到消息队列
Parameters
----------
refresh_task : 任务
ignore_result 设置任务存储状态，如果为True,不存状态，也查询不了返回值
default_retry_delay 设置重试提交到消息队列间隔时间，默认10 分钟，单位为秒
max_retries 设置重试次数，默认为3
Returns
-------
-------
修饰符 @task 将submit函数变成了异步任务。在webapp中调用submit并不会立即执行该函数，
而是将函数名、 参数等打包成消息发送到消息队列中，再由worker执行实际的代码
'''
try:
urls = getUrlsInLimit(getUrls(refresh_task))
logger.debug('submit: %s' % urls)
if not urls:
return
setOveload(refresh_task, urls)
# https add port 443
# for url in urls:
# logger.debug("before url not have 443:%s" % url)
# url['url'] = add_https_443(url.get('url', ''))
# logger.debug('end url have 443:%s' % url)
# logger.debug('submit: %s' % urls)
db.url.insert(urls)
username = refresh_task.get('username')
try:
user_list = eval(config.get('refresh_redis_store_usernames', 'usernames'))
except Exception:
logger.debug('splitter_new submit error:%s' % traceback.format_exc())
user_list = []
try:
if username in user_list:
add_rid_url_info_into_redis(refresh_task.get('r_id'), urls)
except Exception:
logger.debug('insert result into redis error:%s' % traceback.formate_exc(e))
# logger.debug("rubin_test can delete splitter_refreshDevice submit urls:%s" % urls)
# re put the equipment into the URL
for url_t in urls:
url_t['devices'] = refresh_task.get('devices')
# the interface does not have channel_code, instead of using the channel name
url_t['channel_code'] = get_channelname(url_t.get('url'))
#筛选优先级任务
messages = []
messages_high = []
for url in urls:
if url.get("status") == 'PROGRESS':
url_info = get_refreshurl(refresh_task.get('username'), url)
if url.get('high_priority', False):
messages_high.append(url_info)
else:
messages.append(url_info)
#messages = [get_refreshurl(refresh_task.get('username'), url) for url in urls if url.get("status") == 'PROGRESS']
db.request.insert({"_id": refresh_task.get('r_id'), "username": refresh_task.get("username"), "parent": refresh_task.get("username"),
"callback": refresh_task.get("callback"), "status": "PROGRESS", "unprocess": len(messages),
"created_time": datetime.strptime(refresh_task.get('request_time'), '%Y-%m-%d %X') if refresh_task.get('request_time') else datetime.now(),
"remote_addr": refresh_task.get('remote_addr', ''), "serial_num": refresh_task.get('serial_num', '')})
queue.put_json2('url_queue', messages)
if messages_high:
queue.put_json2('url_high_priority_queue', messages_high)
if refresh_task.get('callback'):
noticeEmail(refresh_task)
except Exception:
logger.warning('submit error! do retry. error:%s' % traceback.format_exc())
raise submit.retry(exc=e)
try:
user_list = eval(config.get('refresh_redis_store_usernames', 'usernames'))
except Exception, e:
logger.debug('splitter_new submit error:%s' % traceback.format_exc(e))
user_list = []
try:
if username in user_list:
add_rid_url_info_into_redis(refresh_task.get('r_id'), urls)
except Exception, e:
logger.debug('insert result into redis error:%s' % traceback.formate_exc(e))
# logger.debug("rubin_test can delete splitter_refreshDevice submit urls:%s" % urls)
# re put the equipment into the URL
for url_t in urls:
url_t['devices'] = refresh_task.get('devices')
# the interface does not have channel_code, instead of using the channel name
url_t['channel_code'] = get_channelname(url_t.get('url'))
#筛选优先级任务
messages = []
messages_high = []
for url in urls:
if url.get("status") == 'PROGRESS':
url_info = get_refreshurl(refresh_task.get('username'), url)
if url.get('high_priority', False):
messages_high.append(url_info)
else:
messages.append(url_info)
#messages = [get_refreshurl(refresh_task.get('username'), url) for url in urls if url.get("status") == 'PROGRESS']
db.request.insert({"_id": refresh_task.get('r_id'), "username": refresh_task.get("username"), "parent": refresh_task.get("username"),
"callback": refresh_task.get("callback"), "status": "PROGRESS", "unprocess": len(messages),
"created_time": datetime.strptime(refresh_task.get('request_time'), '%Y-%m-%d %X') if refresh_task.get('request_time') else datetime.now(),
"remote_addr": refresh_task.get('remote_addr', ''), "serial_num": refresh_task.get('serial_num', '')})