def test_run_fail(app, client):
set_single_scrapyd(app, set_second=True)
with app.test_request_context():
url = url_for('schedule.run', node=1)
data = {'filename': '%s_%s_%s.pickle' % (PROJECT, VERSION, SPIDER)}
response = client.post(url, data=data)
assert 'Fail to schedule' in get_text(response)
def test_check(app, client):
set_single_scrapyd(app)
data = {
'project': PROJECT,
'_version': VERSION,
'spider': SPIDER,
'jobid': JOBID,
'USER_AGENT': 'chrome',
'COOKIES_ENABLED': 'False',
'ROBOTSTXT_OBEY': 'False',
'CONCURRENT_REQUESTS': '1',
'DOWNLOAD_DELAY': '2',
'additional': '-d setting=CLOSESPIDER_TIMEOUT=60 \r\n-d setting=CLOSESPIDER_PAGECOUNT=10 \r\n-d arg1=val1'
}
data_ = {
'project': PROJECT,
'_version': DEFAULT_LATEST_VERSION,
'spider': SPIDER,
'additional': '-d setting=CLOSESPIDER_TIMEOUT=60 -d arg1'
}
with app.test_request_context():
url = url_for('schedule.check', node=1)
response = client.post(url, data=data)
# js = response.get_json()
js = load_json(response)
assert js['filename'] == '%s_%s_%s.pickle' % (PROJECT, VERSION, SPIDER)
response = client.post(url, data=data_)
js = load_json(response)
assert js['filename'] == '%s_%s_%s.pickle' % (PROJECT, 'default-the-latest-version', SPIDER)
def test_run(app, client):
set_single_scrapyd(app)
# ScrapydWeb-demo.egg: custom_settings = {}, also log settings & arguments
upload_file_deploy(app, client, filename='ScrapydWeb-demo.egg', project=PROJECT, redirect_project=PROJECT)
with app.test_request_context():
url = url_for('schedule.run', node=1)
data = {'filename': '%s_%s_%s.pickle' % (PROJECT, VERSION, SPIDER)}
response = client.post(url, data=data)
assert url_for('dashboard', node=1) in get_text(response)
sleep()
url = url_for('log', node=1, opt='utf8', project=PROJECT, spider=SPIDER, job=JOBID)
response = client.get(url)
text = get_text(response)
assert 'JOB: %s' % JOBID in text
assert 'USER_AGENT: Mozilla/5.0' in text
assert 'COOKIES_ENABLED: False' in text
assert 'ROBOTSTXT_OBEY: False' in text
assert 'CONCURRENT_REQUESTS: 1' in text
assert 'DOWNLOAD_DELAY: 2' in text
assert 'CLOSESPIDER_TIMEOUT: 60' in text
assert 'CLOSESPIDER_PAGECOUNT: 10' in text
assert 'self.arg1: val1' in text
client.get(url_for('api', node=1, opt='forcestop', project=PROJECT, version_spider_job=JOBID))
def test_page(app, client):
set_single_scrapyd(app)
with app.test_request_context():
for view, title in VIEW_TITLE_MAP.items():
url = url_for(view, node=1)
response = client.get(url)
assert title in get_text(response) and not is_mobileui(response)
def test_switch_node_skip(app, client):
set_single_scrapyd(app)
with app.test_request_context():
url = url_for('dashboard', node=1)
response = client.get(url)
assert ('onclick="switchNode' not in get_text(response)
and 'id="skip_nodes_checkbox"' not in get_text(response))
def test_upload_file_deploy(app, client):
set_single_scrapyd(app)
upload_file_deploy_singlenode = partial(upload_file_deploy,
app=app,
client=client,
multinode=False)
filenames = [
'demo.egg', 'demo_inner.zip', 'demo_outer.zip',
'demo - Win7CNsendzipped.zip', 'demo - Win10cp1252.zip'
]
if WINDOWS_NOT_CP936:
filenames.extend([
'demo - Ubuntu.zip', 'demo - Ubuntu.tar.gz', 'demo - macOS.zip',
'demo - macOS.tar.gz'
])
else:
filenames.extend([
u'副本.zip', u'副本.tar.gz', u'副本.egg', u'demo - 副本 - Win7CN.zip',
u'demo - 副本 - Win7CNsendzipped.zip', u'demo - 副本 - Win10cp936.zip',
u'demo - 副本 - Ubuntu.zip', u'demo - 副本 - Ubuntu.tar.gz',
u'demo - 副本 - macOS.zip', u'demo - 副本 - macOS.tar.gz'
])
for filename in filenames:
if filename == 'demo.egg':
project = PROJECT
redirect_project = PROJECT
else:
project = re.sub(r'\.egg|\.zip|\.tar\.gz', '', filename)
project = 'demo_unicode' if project == u'副本' else project
redirect_project = re.sub(r'[^0-9A-Za-z_-]', '', project)
upload_file_deploy_singlenode(filename=filename,
project=project,
redirect_project=redirect_project)
for filename, alert in SCRAPY_CFG_DICT.items():
if alert:
upload_file_deploy_singlenode(filename='%s.zip' % filename,
project=filename,
alert=alert,
fail=True)
else:
upload_file_deploy_singlenode(filename='%s.zip' % filename,
project=filename,
redirect_project=filename)
app.config['SCRAPYD_SERVERS'] = ['not-exist:6801']
for filename, alert in SCRAPY_CFG_DICT.items():
if filename == 'demo_only_scrapy_cfg' or not alert:
alert = 'Fail to deploy project, got status'
upload_file_deploy_singlenode(filename='%s.zip' % filename,
project=filename,
alert=alert,
fail=True)
def test_select_multinode_checkbox(app, client):
set_single_scrapyd(app)
with app.test_request_context():
keyword = 'CheckAll / UncheckAll'
for view in ['deploy.deploy', 'schedule.schedule']:
url = url_for(view, node=1)
response = client.get(url)
assert keyword not in get_text(response)
def test_schedule_xhr(app, client):
set_single_scrapyd(app)
with app.test_request_context():
url = url_for('schedule.schedule_xhr', node=1, filename='%s_%s_%s.pickle' % (PROJECT, VERSION, SPIDER))
response = client.post(url)
js = load_json(response)
assert js['status'] == OK and js['jobid'] == JOBID
client.get(url_for('api', node=1, opt='forcestop', project=PROJECT, version_spider_job=JOBID))
def test_check_browser(app, client):
set_single_scrapyd(app)
with app.test_request_context():
url = url_for('dashboard', node=1)
response = client.get(url, headers=HEADERS_DICT['IE'])
assert 'checkBrowser()' in get_text(response)
response = client.get(url, headers=HEADERS_DICT['EDGE'])
assert 'checkBrowser()' in get_text(response)
def test_items(app, client):
set_single_scrapyd(app)
title = 'Directory listing for /items/'
with app.test_request_context():
url = url_for('items', node=1)
response = client.get(url)
assert ((title in get_text(response)
or 'No Such Resource' in get_text(response))
and not is_mobileui(response))
def test_schedule_with_url_project(app, client):
set_single_scrapyd(app)
with app.test_request_context():
url = url_for('schedule.schedule', node=1, project=PROJECT, version=DEFAULT_LATEST_VERSION, spider=SPIDER)
response = client.get(url)
text = get_text(response)
assert ("selectedProject = '%s'" % PROJECT in text
and "selectedVersion = 'default: the latest version'" in text
and "this.loadSpiders();" in text
and "selectedSpider = '%s'" % SPIDER in text)
def test_auto_eggifying_select_option(app, client):
set_single_scrapyd(app)
with app.test_request_context():
url = url_for('deploy.deploy', node=1)
response = client.get(url)
text = get_text(response)
assert ('<option value ="%s"' % PROJECT in text
and u'<option value ="demo"' in text
and u'<option value ="demo - 副本"' in text
and u'<option value ="demo_only_scrapy_cfg"' in text
and u'<option value ="demo_without_scrapy_cfg"' not in text)
def test_auto_eggifying_only_scrapy_cfg(app, client):
set_single_scrapyd(app)
data = {
'project': 'demo_only_scrapy_cfg',
'version': VERSION,
}
with app.test_request_context():
url = url_for('deploy.upload', node=1)
response = client.post(url,
content_type='multipart/form-data',
data=data)
assert 'No module named' in get_text(response)
def test_deploy_xhr(app, client):
set_single_scrapyd(app)
with app.test_request_context():
eggname = '%s_%s_from_file_demo.egg' % (PROJECT, VERSION)
url = url_for('deploy.deploy_xhr',
node=1,
eggname=eggname,
project=PROJECT,
version=VERSION)
response = client.post(url)
js = load_json(response)
assert js['status'] == OK and js['project'] == PROJECT
def test_addversion(app, client):
set_single_scrapyd(app)
data = {
'project': 'fakeproject',
'version': 'fakeversion',
'file': (BytesIO(b'my file contents'), "fake.egg")
}
with app.test_request_context():
url = url_for('deploy.upload', node=1)
response = client.post(url,
content_type='multipart/form-data',
data=data)
assert 'activate_egg' in get_text(response)
def test_dropdown_for_mobile_device(app, client):
set_single_scrapyd(app)
with app.test_request_context():
url = url_for('dashboard', node=1)
response = client.get(url, headers=HEADERS_DICT['Chrome'])
text = get_text(response)
assert 'dropdown_mobileui.css' not in text and 'handleDropdown()' not in text
response = client.get(url, headers=HEADERS_DICT['iPhone'])
text = get_text(response)
assert 'dropdown_mobileui.css' in text and 'handleDropdown()' in text
response = client.get(url, headers=HEADERS_DICT['iPad'])
text = get_text(response)
assert 'dropdown_mobileui.css' in text and 'handleDropdown()' in text
def test_index(app, client):
set_single_scrapyd(app)
url = '/'
url_mobileui = '/?ui=mobile'
for __, headers in HEADERS_DICT.items():
response = client.get(url_mobileui, headers=headers)
assert response.headers['Location'].endswith('/1/dashboard/?ui=mobile')
for key in ['Chrome', 'iPad']:
response = client.get(url, headers=HEADERS_DICT[key])
assert response.headers['Location'].endswith('/1/dashboard/')
for key in ['iPhone', 'Android']:
response = client.get(url, headers=HEADERS_DICT[key])
assert response.headers['Location'].endswith('/1/dashboard/?ui=mobile')
def test_auto_eggifying_unicode(app, client):
if WINDOWS_NOT_CP936:
return
set_single_scrapyd(app)
data = {
'project': u'demo - 副本',
'version': VERSION,
}
with app.test_request_context():
url = url_for('deploy.upload', node=1)
response = client.post(url,
content_type='multipart/form-data',
data=data)
assert url_for('schedule.schedule',
node=1,
project='demo-',
version=VERSION) in get_text(response)
def test_auto_eggifying(app, client):
set_single_scrapyd(app)
data = {
'project': PROJECT,
'version': VERSION,
}
with app.test_request_context():
url = url_for('deploy.upload', node=1)
response = client.post(url,
content_type='multipart/form-data',
data=data)
# http://localhost/1/schedule/ScrapydWeb-demo/2018-01-01T01_01_01/
# app.logger.debug(response.headers['Location'])
redirect_url = url_for('schedule.schedule',
node=1,
project=PROJECT,
version=VERSION)
assert response.headers['Location'].endswith(redirect_url)
def test_check_update(app, client):
set_single_scrapyd(app)
with app.test_request_context():
@app.context_processor
def inject_variable():
return dict(CHECK_LATEST_VERSION_FREQ=1)
url = url_for('dashboard', node=1)
response = client.get(url)
text = get_text(response)
assert ('<script>setTimeout("checkLatestVersion(' in text
and '<!-- <script>setTimeout("checkLatestVersion(' not in text)
@app.context_processor
def inject_variable():
return dict(CHECK_LATEST_VERSION_FREQ=100)
response = client.get(url)
text = get_text(response)
assert '<script>setTimeout("checkLatestVersion(' not in text
def test_history_log(app, client):
set_single_scrapyd(app)
with app.test_request_context():
url = url_for('schedule.history', filename='history.log')
response = client.get(url)
assert 'history.log' in get_text(response)
def test_upload_file_deploy(app, client):
set_single_scrapyd(app)
upload_file_deploy_singlenode = partial(upload_file_deploy,
app=app,
client=client,
multinode=False)
upload_file_deploy_singlenode(filename='demo.egg',
project=PROJECT,
redirect_project=PROJECT)
upload_file_deploy_singlenode(filename='demo - Win7CNsendzipped.zip',
project='demo - Win7CNsendzipped',
redirect_project='demo-Win7CNsendzipped')
upload_file_deploy_singlenode(filename='demo - Win10cp1252.zip',
project='demo - Win10cp1252',
redirect_project='demo-Win10cp1252')
if WINDOWS_NOT_CP936:
upload_file_deploy_singlenode(filename='demo - Ubuntu.zip',
project='demo - Ubuntu',
redirect_project='demo-Ubuntu')
upload_file_deploy_singlenode(filename='demo - Ubuntu.tar.gz',
project='demo - Ubuntu',
redirect_project='demo-Ubuntu')
upload_file_deploy_singlenode(filename='demo - macOS.zip',
project='demo - macOS',
redirect_project='demo-macOS')
upload_file_deploy_singlenode(filename='demo - macOS.tar.gz',
project='demo - macOS',
redirect_project='demo-macOS')
else:
upload_file_deploy_singlenode(filename=u'副本.zip',
project='demo_zip',
redirect_project='demo_zip')
upload_file_deploy_singlenode(filename=u'副本.tar.gz',
project='demo_tar_gz',
redirect_project='demo_tar_gz')
upload_file_deploy_singlenode(filename=u'副本.egg',
project='demo_egg',
redirect_project='demo_egg')
upload_file_deploy_singlenode(filename=u'demo - 副本 - Win7CN.zip',
project=u'demo - 副本 - Win7CN',
redirect_project='demo--Win7CN')
upload_file_deploy_singlenode(
filename=u'demo - 副本 - Win7CNsendzipped.zip',
project=u'demo - 副本 - Win7CNsendzipped',
redirect_project='demo--Win7CNsendzipped')
upload_file_deploy_singlenode(filename=u'demo - 副本 - Win10cp936.zip',
project=u'demo - 副本 - Win10cp936',
redirect_project='demo--Win10cp936')
upload_file_deploy_singlenode(filename=u'demo - 副本 - Ubuntu.zip',
project=u'demo - 副本 - Ubuntu',
redirect_project='demo--Ubuntu')
upload_file_deploy_singlenode(filename=u'demo - 副本 - Ubuntu.tar.gz',
project=u'demo - 副本 - Ubuntu',
redirect_project='demo--Ubuntu')
upload_file_deploy_singlenode(filename=u'demo - 副本 - macOS.zip',
project=u'demo - 副本 - macOS',
redirect_project='demo--macOS')
upload_file_deploy_singlenode(filename=u'demo - 副本 - macOS.tar.gz',
project=u'demo - 副本 - macOS',
redirect_project='demo--macOS')
upload_file_deploy_singlenode(filename='demo_inner.zip',
project='demo_inner',
redirect_project='demo_inner')
upload_file_deploy_singlenode(filename='demo_outer.zip',
project='demo_outer',
redirect_project='demo_outer')
upload_file_deploy_singlenode(filename='demo_without_scrapy_cfg.zip',
project='demo_without_scrapy_cfg',
alert='scrapy.cfg NOT found',
fail=True)
upload_file_deploy_singlenode(filename='demo_only_scrapy_cfg.zip',
project='demo_only_scrapy_cfg',
alert='No module named',
fail=True)
