def get_script(project):
projectdb = app.config['projectdb']
if not projectdb.verify_project_name(project):
return 'project name is not allowed!', 400
info = projectdb.get(project, fields=['name', 'script'])
return json.dumps(utils.unicode_obj(info)), \
200, {'Content-Type': 'application/json'}
def run(project):
start_time = time.time()
try:
task = utils.decode_unicode_obj(json.loads(request.form['task']))
except Exception:
result = {
'fetch_result': "",
'logs': u'task json error',
'follows': [],
'messages': [],
'result': None,
'time': time.time() - start_time,
}
return json.dumps(utils.unicode_obj(result)), \
200, {'Content-Type': 'application/json'}
project_info = {
'name': project,
'status': 'DEBUG',
'script': request.form['script'],
}
if request.form.get('webdav_mode') == 'true':
projectdb = app.config['projectdb']
info = projectdb.get(project, fields=['name', 'script'])
if not info:
result = {
'fetch_result': "",
'logs': u' in wevdav mode, cannot load script',
'follows': [],
'messages': [],
'result': None,
'time': time.time() - start_time,
}
return json.dumps(utils.unicode_obj(result)), \
200, {'Content-Type': 'application/json'}
project_info['script'] = info['script']
fetch_result = {}
try:
fetch_result = app.config['fetch'](task)
response = rebuild_response(fetch_result)
module = ProjectManager.build_module(project_info, {
'debugger': True
})
ret = module['instance'].run_task(module['module'], task, response)
except Exception:
type, value, tb = sys.exc_info()
tb = utils.hide_me(tb, globals())
logs = ''.join(traceback.format_exception(type, value, tb))
result = {
'fetch_result': fetch_result,
'logs': logs,
'follows': [],
'messages': [],
'result': None,
'time': time.time() - start_time,
}
else:
result = {
'fetch_result': fetch_result,
'logs': ret.logstr(),
'follows': ret.follows,
'messages': ret.messages,
'result': ret.result,
'time': time.time() - start_time,
}
result['fetch_result']['content'] = response.text
if (response.headers.get('content-type', '').startswith('image')):
result['fetch_result']['dataurl'] = dataurl.encode(
response.content, response.headers['content-type'])
try:
# binary data can't encode to JSON, encode result as unicode obj
# before send it to frontend
return json.dumps(utils.unicode_obj(result)), 200, {'Content-Type': 'application/json'}
except Exception:
type, value, tb = sys.exc_info()
tb = utils.hide_me(tb, globals())
logs = ''.join(traceback.format_exception(type, value, tb))
result = {
'fetch_result': "",
'logs': logs,
'follows': [],
'messages': [],
'result': None,
'time': time.time() - start_time,
}
return json.dumps(utils.unicode_obj(result)), 200, {'Content-Type': 'application/json'}
def on_task(self, task, response):
"""Deal one task"""
start_time = time.time()
response = rebuild_response(response)
try:
assert "taskid" in task, "need taskid in task"
project = task["project"]
updatetime = task.get("project_updatetime", None)
md5sum = task.get("project_md5sum", None)
project_data = self.project_manager.get(project, updatetime, md5sum)
assert project_data, "no such project!"
if project_data.get("exception"):
ret = ProcessorResult(logs=(project_data.get("exception_log"),), exception=project_data["exception"])
else:
ret = project_data["instance"].run_task(project_data["module"], task, response)
except Exception as e:
logstr = traceback.format_exc()
ret = ProcessorResult(logs=(logstr,), exception=e)
process_time = time.time() - start_time
if not ret.extinfo.get("not_send_status", False):
if ret.exception:
track_headers = dict(response.headers)
else:
track_headers = {}
for name in ("etag", "last-modified"):
if name not in response.headers:
continue
track_headers[name] = response.headers[name]
status_pack = {
"taskid": task["taskid"],
"project": task["project"],
"url": task.get("url"),
"track": {
"fetch": {
"ok": response.isok(),
"redirect_url": response.url if response.url != response.orig_url else None,
"time": response.time,
"error": response.error,
"status_code": response.status_code,
"encoding": response.encoding,
"headers": track_headers,
"content": response.text[:500] if ret.exception else None,
},
"process": {
"ok": not ret.exception,
"time": process_time,
"follows": len(ret.follows),
"result": (None if ret.result is None else utils.text(ret.result)[: self.RESULT_RESULT_LIMIT]),
"logs": ret.logstr()[-self.RESULT_LOGS_LIMIT :],
"exception": ret.exception,
},
"save": ret.save,
},
}
if "schedule" in task:
status_pack["schedule"] = task["schedule"]
# FIXME: unicode_obj should used in scheduler before store to database
# it's used here for performance.
self.status_queue.put(utils.unicode_obj(status_pack))
# FIXME: unicode_obj should used in scheduler before store to database
# it's used here for performance.
if ret.follows:
self.newtask_queue.put([utils.unicode_obj(newtask) for newtask in ret.follows])
for project, msg, url in ret.messages:
try:
self.on_task(
{
"taskid": utils.md5string(url),
"project": project,
"url": url,
"process": {"callback": "_on_message"},
},
{"status_code": 200, "url": url, "save": (task["project"], msg)},
)
except Exception as e:
logger.exception("Sending message error.")
continue
if ret.exception:
logger_func = logger.error
else:
logger_func = logger.info
logger_func(
"process %s:%s %s -> [%d] len:%d -> result:%.10r fol:%d msg:%d err:%r"
% (
task["project"],
task["taskid"],
task.get("url"),
response.status_code,
len(response.content),
ret.result,
len(ret.follows),
len(ret.messages),
ret.exception,
)
)
return True