def test_auto_eggifying(app, client):
data = {
'1': 'on',
'checked_amount': '1',
'project': PROJECT,
'version': VERSION
}
with app.test_request_context():
url = url_for('deploy.upload', node=2)
response = client.post(url,
content_type='multipart/form-data',
data=data)
text = get_text(response)
assert 'deploy results' in text and 'onclick="multinodeRunSpider();"' in text
data = {
'1': 'on',
'2': 'on',
'checked_amount': '2',
'project': PROJECT,
'version': VERSION
}
with app.test_request_context():
url = url_for('deploy.upload', node=2)
response = client.post(url,
content_type='multipart/form-data',
data=data)
text = get_text(response)
assert ('deploy results - ScrapydWeb' in text
and 'id="checkbox_2"' in text
and 'onclick="multinodeRunSpider();"' in text)
def test_run(app, client):
set_single_scrapyd(app)
# ScrapydWeb-demo.egg: custom_settings = {}, also log settings & arguments
upload_file_deploy(app, client, filename='ScrapydWeb-demo.egg', project=PROJECT, redirect_project=PROJECT)
with app.test_request_context():
url = url_for('schedule.run', node=1)
data = {'filename': '%s_%s_%s.pickle' % (PROJECT, VERSION, SPIDER)}
response = client.post(url, data=data)
assert url_for('dashboard', node=1) in get_text(response)
sleep()
url = url_for('log', node=1, opt='utf8', project=PROJECT, spider=SPIDER, job=JOBID)
response = client.get(url)
text = get_text(response)
assert 'JOB: %s' % JOBID in text
assert 'USER_AGENT: Mozilla/5.0' in text
assert 'COOKIES_ENABLED: False' in text
assert 'ROBOTSTXT_OBEY: False' in text
assert 'CONCURRENT_REQUESTS: 1' in text
assert 'DOWNLOAD_DELAY: 2' in text
assert 'CLOSESPIDER_TIMEOUT: 60' in text
assert 'CLOSESPIDER_PAGECOUNT: 10' in text
assert 'self.arg1: val1' in text
client.get(url_for('api', node=1, opt='forcestop', project=PROJECT, version_spider_job=JOBID))
def test_switch_node_skip(app, client):
with app.test_request_context():
url = url_for('overview', node=1)
response = client.get(url)
assert ('1 / 2' in get_text(response)
and 'onclick="switchNode' in get_text(response)
and 'id="skip_nodes_checkbox"' in get_text(response))
def test_items(app, client):
title = 'Directory listing for /items/'
with app.test_request_context():
url = url_for('items', node=1)
response = client.get(url)
assert ((title in get_text(response) or 'No Such Resource' in get_text(response))
and not is_mobileui(response))
def test_switch_node_skip(app, client):
set_single_scrapyd(app)
with app.test_request_context():
url = url_for('dashboard', node=1)
response = client.get(url)
assert ('onclick="switchNode' not in get_text(response)
and 'id="skip_nodes_checkbox"' not in get_text(response))
def test_check_browser(app, client):
with app.test_request_context():
url = url_for('overview', node=1)
response = client.get(url, headers=HEADERS_DICT['IE'])
assert 'checkBrowser()' in get_text(response)
response = client.get(url, headers=HEADERS_DICT['EDGE'])
assert 'checkBrowser()' in get_text(response)
def test_check_browser(app, client):
set_single_scrapyd(app)
with app.test_request_context():
url = url_for('dashboard', node=1)
response = client.get(url, headers=HEADERS_DICT['IE'])
assert 'checkBrowser()' in get_text(response)
response = client.get(url, headers=HEADERS_DICT['EDGE'])
assert 'checkBrowser()' in get_text(response)
def test_dropdown_for_mobile_device(app, client):
with app.test_request_context():
url = url_for('overview', node=1)
response = client.get(url, headers=HEADERS_DICT['Chrome'])
text = get_text(response)
assert 'dropdown_mobileui.css' not in text and 'handleDropdown()' not in text
response = client.get(url, headers=HEADERS_DICT['iPhone'])
text = get_text(response)
assert 'dropdown_mobileui.css' in text and 'handleDropdown()' in text
response = client.get(url, headers=HEADERS_DICT['iPad'])
text = get_text(response)
assert 'dropdown_mobileui.css' in text and 'handleDropdown()' in text
def test_run_fail(app, client):
set_single_scrapyd(app, set_second=True)
with app.test_request_context():
url = url_for('schedule.run', node=1)
data = {'filename': '%s_%s_%s.pickle' % (PROJECT, VERSION, SPIDER)}
response = client.post(url, data=data)
assert 'Fail to schedule' in get_text(response)
def test_run(app, client):
upload_file_deploy(app,
client,
filename='demo.egg',
project=PROJECT,
redirect_project=PROJECT)
with app.test_request_context():
url = url_for('schedule.run', node=2)
data = {
'1': 'on',
'2': 'on',
'checked_amount': '2',
'filename': '%s_%s_%s.pickle' % (PROJECT, VERSION, SPIDER)
}
response = client.post(url, data=data)
text = get_text(response)
assert ('run results - ScrapydWeb' in text
and 'id="checkbox_2"' in text
and 'onclick="passToOverview();"' in text)
client.get(
url_for('api',
node=1,
opt='forcestop',
project=PROJECT,
version_spider_job=JOBID))
def test_select_multinode_checkbox(app, client):
with app.test_request_context():
keyword = 'CheckAll / UncheckAll'
for view in ['deploy.deploy', 'schedule.schedule']:
url = url_for(view, node=1)
response = client.get(url)
assert keyword in get_text(response)
def test_logs_inside(app, client):
with app.test_request_context():
for project, spider in [(PROJECT, None), (PROJECT, SPIDER)]:
title = 'Directory listing for /logs/%s/%s' % (project, spider or '')
url = url_for('logs', node=1, project=project, spider=spider)
response = client.get(url)
assert title in get_text(response) and not is_simple_ui(response)
def test_listversions(app, client):
with app.test_request_context():
url = url_for('manage', node=1, opt='listversions', project=PROJECT)
response = client.get(url)
text = get_text(response)
assert ('Delete Project' in text
and 'Delete Version' in text
and url_for('manage', node=1, opt='delproject', project=PROJECT) in text)
# {"status": "ok", "versions": []}
fake_url = url_for('manage', node=1, opt='listversions', project=FAKE_PROJECT)
response = client.get(fake_url)
text = get_text(response)
assert ('Delete Project' in text
and 'Delete the version' not in text
and url_for('manage', node=1, opt='delproject', project=FAKE_PROJECT) in text)
def test_deploy_from_post(app, client):
with app.test_request_context():
url = url_for('deploy.deploy', node=1)
data = {'1': 'on', '2': 'on'}
response = client.post(url, data=data)
text = get_text(response)
assert (re.search(r'id="checkbox_1".*?checked.*?/>', text, re.S)
and re.search(r'id="checkbox_2".*?checked.*?/>', text, re.S))
def test_page(app, client):
with app.test_request_context():
for view, title in VIEW_TITLE_MAP.items():
url = url_for(view, node=1)
response = client.get(url)
assert title in get_text(response) and not is_mobileui(response)
app.config['SCRAPYD_SERVERS'] = app.config['SCRAPYD_SERVERS'][::-1]
with app.test_request_context():
url = url_for('dashboard', node=1)
response = client.get(url)
assert 'status_code: -1' in get_text(response)
for view, title in VIEW_TITLE_MAP.items():
url = url_for(view, node=2)
response = client.get(url)
assert title in get_text(response) and not is_mobileui(response)
def test_page(app, client):
set_single_scrapyd(app)
with app.test_request_context():
for view, title in VIEW_TITLE_MAP.items():
url = url_for(view, node=1)
response = client.get(url)
assert title in get_text(response) and not is_mobileui(response)
def test_check_update(app, client):
with app.test_request_context():
@app.context_processor
def inject_variable():
return dict(CHECK_LATEST_VERSION_FREQ=1)
url = url_for('overview', node=1)
response = client.get(url)
text = get_text(response)
assert ('<script>setTimeout("checkLatestVersion(' in text
and '<!-- <script>setTimeout("checkLatestVersion(' not in text)
@app.context_processor
def inject_variable():
return dict(CHECK_LATEST_VERSION_FREQ=100)
response = client.get(url)
text = get_text(response)
assert '<script>setTimeout("checkLatestVersion(' not in text
def test_listprojects(app, client):
# upload_file_deploy(app, client, filename='demo.egg', project=PROJECT, redirect_project=PROJECT)
upload_file_deploy(app, client, filename='demo.zip', project=PROJECT, redirect_project=PROJECT)
title = 'Get the list of projects uploaded'
with app.test_request_context():
url = url_for('manage', node=1)
response = client.get(url)
assert title in get_text(response) and not is_simple_ui(response)
def test_parse_uploaded_demo_txt(app, client):
with app.test_request_context():
url = url_for('parse.uploaded',
node=1,
filename='demo.txt',
ui='simple')
response = client.get(url)
assert 'Stats collection' in get_text(response) and is_simple_ui(
response)
def test_schedule_with_url_project(app, client):
set_single_scrapyd(app)
with app.test_request_context():
url = url_for('schedule.schedule', node=1, project=PROJECT, version=DEFAULT_LATEST_VERSION, spider=SPIDER)
response = client.get(url)
text = get_text(response)
assert ("selectedProject = '%s'" % PROJECT in text
and "selectedVersion = 'default: the latest version'" in text
and "this.loadSpiders();" in text
and "selectedSpider = '%s'" % SPIDER in text)
def test_listspiders_del(app, client):
with app.test_request_context():
d = OrderedDict() # For python 2 compatibility
d['listspiders'] = dict(url=url_for('projects',
node=1,
opt='listspiders',
project=cst.PROJECT,
version_spider_job=cst.VERSION),
checks=['Run Spider (test)'])
d['listspiders_fail'] = dict(
url=url_for('projects',
node=1,
opt='listspiders',
project=cst.FAKE_PROJECT,
version_spider_job=cst.FAKE_VERSION),
checks=['listspiders.json', 'No such file or directory'])
d['delversion'] = dict(url=url_for('projects',
node=1,
opt='delversion',
project=cst.PROJECT,
version_spider_job=cst.VERSION),
checks=['version deleted'])
d['delversion_fail'] = dict(
url=url_for('projects',
node=1,
opt='delversion',
project=cst.FAKE_PROJECT,
version_spider_job=cst.FAKE_VERSION),
checks=['delversion.json', 'See details below'])
d['delproject'] = dict(url=url_for('projects',
node=1,
opt='delproject',
project=cst.PROJECT),
checks=['project deleted'])
d['delproject_fail'] = dict(
url=url_for('projects',
node=1,
opt='delproject',
project=cst.FAKE_PROJECT),
checks=['delproject.json', 'See details below'])
for k, v in d.items():
if k == 'delproject': # Should use OrderedDict For python 2 compatibility
upload_file_deploy(app,
client,
filename='demo.zip',
project=cst.PROJECT,
redirect_project=cst.PROJECT)
response = client.get(v['url'])
text = get_text(response)
for c in v['checks']:
assert c in text
def test_log_stats(app, client):
with app.test_request_context():
url = url_for('log',
node=1,
opt='stats',
project=PROJECT,
spider=SPIDER,
job=jobid,
ui='mobile')
response = client.get(url)
assert 'current_time' in get_text(response) and is_mobileui(response)
def multinode_command(app, client, opt, title, project, version_job=None):
with app.test_request_context():
url = url_for('multinode',
node=1,
opt=opt,
project=project,
version_job=version_job)
response = client.post(url,
content_type='multipart/form-data',
data={'1': 'on'})
assert title in get_text(response)
def test_auto_eggifying_select_option(app, client):
set_single_scrapyd(app)
with app.test_request_context():
url = url_for('deploy.deploy', node=1)
response = client.get(url)
text = get_text(response)
assert ('<option value ="%s"' % PROJECT in text
and u'<option value ="demo"' in text
and u'<option value ="demo - 副本"' in text
and u'<option value ="demo_only_scrapy_cfg"' in text
and u'<option value ="demo_without_scrapy_cfg"' not in text)
def test_log_utf8(app, client):
with app.test_request_context():
url = url_for('log',
node=1,
opt='utf8',
project=PROJECT,
spider=SPIDER,
job=jobid,
ui='mobile')
response = client.get(url)
assert 'PROJECT (%s)' % PROJECT in get_text(response) and is_mobileui(
response)
def test_auto_eggifying_only_scrapy_cfg(app, client):
set_single_scrapyd(app)
data = {
'project': 'demo_only_scrapy_cfg',
'version': VERSION,
}
with app.test_request_context():
url = url_for('deploy.upload', node=1)
response = client.post(url,
content_type='multipart/form-data',
data=data)
assert 'No module named' in get_text(response)
def test_addversion(app, client):
data = {
'project': 'fakeproject',
'version': 'fakeversion',
'file': (BytesIO(b'my file contents'), "fake.egg")
}
with app.test_request_context():
url = url_for('deploy.upload', node=1)
response = client.post(url,
content_type='multipart/form-data',
data=data)
assert 'activate_egg' in get_text(response)
def test_log_stats(app, client):
with app.test_request_context():
url = url_for('log',
node=1,
opt='stats',
project=PROJECT,
spider=SPIDER,
job=jobid,
ui='simple')
response = client.get(url)
assert 'Stats collection' in get_text(response) and is_simple_ui(
response)
def test_log_utf8(app, client):
with app.test_request_context():
url = url_for('log',
node=1,
opt='utf8',
project=PROJECT,
spider=SPIDER,
job=jobid,
ui='simple')
response = client.get(url)
assert 'utf8 - ScrapydWeb' in get_text(response) and is_simple_ui(
response)
def test_auto_eggifying(app, client):
data = {
'project': PROJECT,
'version': VERSION,
}
with app.test_request_context():
url = url_for('deploy.upload', node=1)
response = client.post(url,
content_type='multipart/form-data',
data=data)
assert url_for('schedule.schedule', node=1,
project=PROJECT) in get_text(response)