def test_parse_upload(app, client):
req(app,
client,
view='parse.upload',
kws=dict(node=1),
data={'file': (BytesIO(b'my file contents'), "fake.log")},
location='/parse/uploaded/')
def test_run_with_task(app, client):
# ScrapydWeb_demo.egg: custom_settings = {}, also output specific settings & arguments in the log
upload_file_deploy(app,
client,
filename='ScrapydWeb_demo_no_request.egg',
project=cst.PROJECT,
redirect_project=cst.PROJECT)
req(app,
client,
view='tasks.xhr',
kws=dict(node=NODE, action='enable'),
ins='STATE_RUNNING',
nos='STATE_PAUSED')
with app.test_request_context():
text, __ = req(app,
client,
view='schedule.run',
kws=dict(node=NODE),
data=run_data,
location=url_for('tasks', node=NODE))
m = re.search(cst.TASK_NEXT_RUN_TIME_PATTERN, unquote_plus(text))
task_id = int(m.group(1))
print("task_id: %s" % task_id)
metadata['task_id'] = task_id
__, js = req(app,
client,
view='tasks.xhr',
kws=dict(node=NODE, action='dump', task_id=task_id))
assert js['data']['selected_nodes'] == [1, 2]
def test_dashboard(app, client):
req(app,
client,
view='dashboard',
kws=dict(node=1, ui='mobile'),
ins='dashboard - mobileui - ScrapydWeb',
mobileui=True)
def test_node_reports_pass(app, client):
with app.test_request_context():
url_report = url_for('log', node=1, opt='report', project='PROJECT_PLACEHOLDER',
spider='SPIDER_PLACEHOLDER', job='JOB_PLACEHOLDER')
ins = ["url_report: '%s'," % url_report, "start: '", "finish: '"]
req(app, client, view='nodereports', kws=dict(node=1), ins=ins)
req_single_scrapyd(app, client, view='nodereports', kws=dict(node=1), ins=ins)
def test_enable_logparser(app, client):
def json_loads_from_file(path):
with io.open(path, 'r', encoding='utf-8') as f:
return json.loads(f.read())
# In conftest.py: ENABLE_LOGPARSER=False
assert not os.path.exists(app.config['STATS_JSON_PATH'])
assert not os.path.exists(app.config['DEMO_JSON_PATH'])
app.config['ENABLE_LOGPARSER'] = True
app.config['ENABLE_EMAIL'] = False
# ['username:[email protected]:6800', ]
app.config['SCRAPYD_SERVERS'] = app.config['_SCRAPYD_SERVERS']
check_app_config(app.config)
logparser_pid = app.config['LOGPARSER_PID']
assert isinstance(logparser_pid, int) and logparser_pid > 0
assert app.config['POLL_PID'] is None
req(app, client, view='settings', kws=dict(node=1), ins='logparser_pid: %s' % logparser_pid)
sleep()
stats_json = json_loads_from_file(app.config['STATS_JSON_PATH'])
assert stats_json['logparser_version'] == cst.LOGPARSER_VERSION
assert cst.DEMO_JOBID in stats_json['datas'][cst.PROJECT][cst.SPIDER]
demo_json = json_loads_from_file(app.config['DEMO_JSON_PATH'])
assert demo_json['runtime'] == '0:01:08'
assert demo_json['finish_reason'] == 'finished'
assert demo_json['logparser_version'] == cst.LOGPARSER_VERSION
def test_auto_packaging(app, client):
data = {
'1': 'on',
'checked_amount': '1',
'folder': cst.PROJECT,
'project': cst.PROJECT,
'version': cst.VERSION
}
req(app,
client,
view='deploy.upload',
kws=dict(node=2),
data=data,
ins=[
'deploy results - ScrapydWeb', 'onclick="multinodeRunSpider();"',
'id="checkbox_1"'
],
nos='id="checkbox_2"')
data.update({'2': 'on', 'checked_amount': '2'})
req(app,
client,
view='deploy.upload',
kws=dict(node=2),
data=data,
ins=[
'deploy results - ScrapydWeb', 'onclick="multinodeRunSpider();"',
'id="checkbox_1"', 'id="checkbox_2"'
])
def test_check_with_task(app, client):
req(app,
client,
view='schedule.check',
kws=dict(node=NODE),
data=check_data,
jskws=dict(cmd="-d _version=%s" % cst.VERSION, filename=FILENAME))
def test_check_app_config(app, client):
cleantest(app, client)
# In conftest.py: ENABLE_LOGPARSER=False
assert not os.path.exists(app.config['STATS_JSON_PATH'])
check_app_config(app.config)
strings = []
assert app.config['LOGPARSER_PID'] is None
strings.append('logparser_pid: None')
poll_pid = app.config['POLL_PID']
if app.config.get('ENABLE_EMAIL', False):
assert isinstance(poll_pid, int) and poll_pid > 0
else:
assert poll_pid is None
strings.append('poll_pid: %s' % poll_pid)
req(app, client, view='settings', kws=dict(node=1), ins=strings)
assert not os.path.exists(app.config['STATS_JSON_PATH'])
# Test ENABLE_EMAIL = False
if app.config.get('ENABLE_EMAIL', False):
app.config['ENABLE_EMAIL'] = False
check_app_config(app.config)
assert app.config['LOGPARSER_PID'] is None
assert app.config['POLL_PID'] is None
req(app,
client,
view='settings',
kws=dict(node=1),
ins='poll_pid: None')
def test_jobs(app, client):
req(app,
client,
view='jobs',
kws=dict(node=1, ui='mobile'),
ins='jobs - mobileui - ScrapydWeb',
mobileui=True)
def test_select_multinode_checkbox(app, client):
for view in ['deploy.deploy', 'schedule.schedule']:
req(app,
client,
view=view,
kws=dict(node=2),
ins='CheckAll / UncheckAll')
def test_daemonstatus(app, client):
req(app,
client,
view='api',
kws=dict(node=1, opt='daemonstatus'),
jskws=dict(status=OK),
jskeys=['pending', 'running', 'finished'])
def test_run(app, client):
upload_file_deploy(app,
client,
filename='demo.egg',
project=cst.PROJECT,
redirect_project=cst.PROJECT)
data = {
'1': 'on',
'2': 'on',
'checked_amount': '2',
'filename': '%s_%s_%s.pickle' % (cst.PROJECT, cst.VERSION, cst.SPIDER)
}
req(app,
client,
view='schedule.run',
kws=dict(node=2),
data=data,
ins=[
'run results - ScrapydWeb', 'id="checkbox_2"',
'onclick="passToOverview();"'
])
req(app,
client,
view='api',
kws=dict(node=1,
opt='forcestop',
project=cst.PROJECT,
version_spider_job=cst.JOBID))
def test_listversions(app, client):
req(app,
client,
view='api',
kws=dict(node=1, opt='listversions', project=PROJECT),
jskws=dict(status=OK),
jskeys='versions')
def test_scrapy_cfg_first_node_not_exist(app, client):
switch_scrapyd(app)
for folder, result in cst.SCRAPY_CFG_DICT.items():
data = {
'1': 'on',
'2': 'on',
'checked_amount': '2',
'folder': folder,
'project': cst.PROJECT,
'version': cst.VERSION,
}
nos = []
if folder == 'demo_only_scrapy_cfg' or not result:
ins = [
'fail - ScrapydWeb', 'the first selected node returned status'
]
else:
ins = ['fail - ScrapydWeb', result]
nos = 'the first selected node returned status'
req(app,
client,
view='deploy.upload',
kws=dict(node=2),
data=data,
ins=ins,
nos=nos)
def test_listprojects(app, client):
req(app,
client,
view='api',
kws=dict(node=1, opt='listprojects'),
jskws=dict(status=OK),
jskeys='projects')
def test_index(app, client):
with app.test_request_context():
req(app,
client,
view='index',
kws=dict(ui='mobile'),
location=url_for('dashboard', node=1, ui='mobile'))
def test_listjobs(app, client):
req(app,
client,
view='api',
kws=dict(node=1, opt='listjobs', project=PROJECT),
jskws=dict(status=OK, url='listjobs.json'),
jskeys=['pending', 'running', 'finished'])
def test_schedule_xhr(app, client):
req(app,
client,
view='schedule.schedule_xhr',
kws=dict(node=2,
filename='%s_%s_%s.pickle' %
(cst.PROJECT, cst.VERSION, cst.SPIDER)),
jskws=dict(status=cst.ERROR))
def forcestop_a_job(job):
req(app,
client,
view='api',
kws=dict(node=1,
opt='forcestop',
project=cst.PROJECT,
version_spider_job=job))
def post_for_poll(job, job_finished=''):
kws = dict(node=1,
opt='stats',
project=cst.PROJECT,
spider=cst.SPIDER,
job=job,
job_finished=job_finished)
req(app, client, view='log', kws=kws, data={}, ins='Log analysis')
def post_for_caching(job, job_finished=''):
kws = dict(node=1,
opt='stats',
project=PROJECT,
spider=SPIDER,
job=job,
job_finished=job_finished)
req(app, client, view='log', kws=kws, data={}, ins='Stats collection')
def test_run_fail(app, client):
switch_scrapyd(app)
req(app,
client,
view='schedule.run',
kws=dict(node=NODE),
data=run_data,
ins='Multinode schedule terminated')
def multinode_command(app, client, opt, title, project, version_job=None):
data = {'1': 'on', '2': 'on'}
req(app,
client,
view='multinode',
kws=dict(node=1, opt=opt, project=project, version_job=version_job),
data=data,
ins=[title, 'id="checkbox_1"', 'id="checkbox_2"'])
def test_stats_with_logparser_disabled(app, client):
kws = dict(node=1, opt='stats', project=cst.PROJECT, spider=cst.SPIDER, job=cst.DEMO_JOBID)
req(app, client, view='log', kws=kws,
ins=["Using local logfile:", 'id="finish_reason">finished<'], nos='refresh_button')
kws['job'] = cst.DEMO_UNFINISHED_LOG.split('.')[0]
ins = ['id="finish_reason">N/A<', '<a id="refresh_button"', "var by = 'ScrapydWeb';"]
req(app, client, view='log', kws=kws, ins=ins)
def test_forcestop(app, client):
req(app,
client,
view='api',
kws=dict(node=1,
opt='forcestop',
project=PROJECT,
version_spider_job=JOBID),
jskws=dict(status=OK, times=2, prevstate=None))
def test_api_stop(app, client):
sleep()
req(app,
client,
view='api',
kws=dict(node=1, opt='stop', project=PROJECT,
version_spider_job=jobid),
jskws=dict(status=OK, prevstate='running'),
nos='times')
def listspiders(app, client, version):
req(app,
client,
view='api',
kws=dict(node=1,
opt='listspiders',
project=PROJECT,
version_spider_job=version),
jskws=dict(status=OK, spiders=SPIDER))
def test_log_not_exist(app, client):
# the Stats page
kws = dict(node=1, opt='stats', project=cst.PROJECT, spider=cst.SPIDER, job=cst.FAKE_JOBID)
ins = ['fail - ScrapydWeb', 'status_code: 404']
req(app, client, view='log', kws=kws, ins=ins)
# the Log page
kws['opt'] = 'utf8'
ins = ['fail - ScrapydWeb', 'status_code: 404']
req(app, client, view='log', kws=kws, ins=ins)
def test_parse_source_demo_log(app, client):
req(app,
client,
view='parse.source',
kws=dict(filename=cst.DEMO_LOG),
ins=[
'2018-10-23 18:28:34 [scrapy.utils.log] INFO: Scrapy 1.5.0 started (bot: demo)',
'2018-10-23 18:29:42 [scrapy.core.engine] INFO: Spider closed (finished)'
])
def test_auto_packaging_select_option(app, client):
ins = [
'(14 projects)', u"var folders = ['demo - 副本', 'demo',",
"var projects = ['demo-copy', 'demo',",
'<div>%s<' % cst.PROJECT, u'<div>demo - 副本<', '<div>demo<',
'<div>demo_only_scrapy_cfg<'
]
nos = ['<div>demo_without_scrapy_cfg<', '<h3>NO projects found']
req(app, client, view='deploy', kws=dict(node=2), ins=ins, nos=nos)
