def test_run_fail(app, client): set_single_scrapyd(app, set_second=True) with app.test_request_context(): url = url_for('schedule.run', node=1) data = {'filename': '%s_%s_%s.pickle' % (PROJECT, VERSION, SPIDER)} response = client.post(url, data=data) assert 'Fail to schedule' in get_text(response)
def test_check(app, client): set_single_scrapyd(app) data = { 'project': PROJECT, '_version': VERSION, 'spider': SPIDER, 'jobid': JOBID, 'USER_AGENT': 'chrome', 'COOKIES_ENABLED': 'False', 'ROBOTSTXT_OBEY': 'False', 'CONCURRENT_REQUESTS': '1', 'DOWNLOAD_DELAY': '2', 'additional': '-d setting=CLOSESPIDER_TIMEOUT=60 \r\n-d setting=CLOSESPIDER_PAGECOUNT=10 \r\n-d arg1=val1' } data_ = { 'project': PROJECT, '_version': DEFAULT_LATEST_VERSION, 'spider': SPIDER, 'additional': '-d setting=CLOSESPIDER_TIMEOUT=60 -d arg1' } with app.test_request_context(): url = url_for('schedule.check', node=1) response = client.post(url, data=data) # js = response.get_json() js = load_json(response) assert js['filename'] == '%s_%s_%s.pickle' % (PROJECT, VERSION, SPIDER) response = client.post(url, data=data_) js = load_json(response) assert js['filename'] == '%s_%s_%s.pickle' % (PROJECT, 'default-the-latest-version', SPIDER)
def test_run(app, client): set_single_scrapyd(app) # ScrapydWeb-demo.egg: custom_settings = {}, also log settings & arguments upload_file_deploy(app, client, filename='ScrapydWeb-demo.egg', project=PROJECT, redirect_project=PROJECT) with app.test_request_context(): url = url_for('schedule.run', node=1) data = {'filename': '%s_%s_%s.pickle' % (PROJECT, VERSION, SPIDER)} response = client.post(url, data=data) assert url_for('dashboard', node=1) in get_text(response) sleep() url = url_for('log', node=1, opt='utf8', project=PROJECT, spider=SPIDER, job=JOBID) response = client.get(url) text = get_text(response) assert 'JOB: %s' % JOBID in text assert 'USER_AGENT: Mozilla/5.0' in text assert 'COOKIES_ENABLED: False' in text assert 'ROBOTSTXT_OBEY: False' in text assert 'CONCURRENT_REQUESTS: 1' in text assert 'DOWNLOAD_DELAY: 2' in text assert 'CLOSESPIDER_TIMEOUT: 60' in text assert 'CLOSESPIDER_PAGECOUNT: 10' in text assert 'self.arg1: val1' in text client.get(url_for('api', node=1, opt='forcestop', project=PROJECT, version_spider_job=JOBID))
def test_page(app, client): set_single_scrapyd(app) with app.test_request_context(): for view, title in VIEW_TITLE_MAP.items(): url = url_for(view, node=1) response = client.get(url) assert title in get_text(response) and not is_mobileui(response)
def test_switch_node_skip(app, client): set_single_scrapyd(app) with app.test_request_context(): url = url_for('dashboard', node=1) response = client.get(url) assert ('onclick="switchNode' not in get_text(response) and 'id="skip_nodes_checkbox"' not in get_text(response))
def test_upload_file_deploy(app, client): set_single_scrapyd(app) upload_file_deploy_singlenode = partial(upload_file_deploy, app=app, client=client, multinode=False) filenames = [ 'demo.egg', 'demo_inner.zip', 'demo_outer.zip', 'demo - Win7CNsendzipped.zip', 'demo - Win10cp1252.zip' ] if WINDOWS_NOT_CP936: filenames.extend([ 'demo - Ubuntu.zip', 'demo - Ubuntu.tar.gz', 'demo - macOS.zip', 'demo - macOS.tar.gz' ]) else: filenames.extend([ u'副本.zip', u'副本.tar.gz', u'副本.egg', u'demo - 副本 - Win7CN.zip', u'demo - 副本 - Win7CNsendzipped.zip', u'demo - 副本 - Win10cp936.zip', u'demo - 副本 - Ubuntu.zip', u'demo - 副本 - Ubuntu.tar.gz', u'demo - 副本 - macOS.zip', u'demo - 副本 - macOS.tar.gz' ]) for filename in filenames: if filename == 'demo.egg': project = PROJECT redirect_project = PROJECT else: project = re.sub(r'\.egg|\.zip|\.tar\.gz', '', filename) project = 'demo_unicode' if project == u'副本' else project redirect_project = re.sub(r'[^0-9A-Za-z_-]', '', project) upload_file_deploy_singlenode(filename=filename, project=project, redirect_project=redirect_project) for filename, alert in SCRAPY_CFG_DICT.items(): if alert: upload_file_deploy_singlenode(filename='%s.zip' % filename, project=filename, alert=alert, fail=True) else: upload_file_deploy_singlenode(filename='%s.zip' % filename, project=filename, redirect_project=filename) app.config['SCRAPYD_SERVERS'] = ['not-exist:6801'] for filename, alert in SCRAPY_CFG_DICT.items(): if filename == 'demo_only_scrapy_cfg' or not alert: alert = 'Fail to deploy project, got status' upload_file_deploy_singlenode(filename='%s.zip' % filename, project=filename, alert=alert, fail=True)
def test_select_multinode_checkbox(app, client): set_single_scrapyd(app) with app.test_request_context(): keyword = 'CheckAll / UncheckAll' for view in ['deploy.deploy', 'schedule.schedule']: url = url_for(view, node=1) response = client.get(url) assert keyword not in get_text(response)
def test_schedule_xhr(app, client): set_single_scrapyd(app) with app.test_request_context(): url = url_for('schedule.schedule_xhr', node=1, filename='%s_%s_%s.pickle' % (PROJECT, VERSION, SPIDER)) response = client.post(url) js = load_json(response) assert js['status'] == OK and js['jobid'] == JOBID client.get(url_for('api', node=1, opt='forcestop', project=PROJECT, version_spider_job=JOBID))
def test_check_browser(app, client): set_single_scrapyd(app) with app.test_request_context(): url = url_for('dashboard', node=1) response = client.get(url, headers=HEADERS_DICT['IE']) assert 'checkBrowser()' in get_text(response) response = client.get(url, headers=HEADERS_DICT['EDGE']) assert 'checkBrowser()' in get_text(response)
def test_items(app, client): set_single_scrapyd(app) title = 'Directory listing for /items/' with app.test_request_context(): url = url_for('items', node=1) response = client.get(url) assert ((title in get_text(response) or 'No Such Resource' in get_text(response)) and not is_mobileui(response))
def test_schedule_with_url_project(app, client): set_single_scrapyd(app) with app.test_request_context(): url = url_for('schedule.schedule', node=1, project=PROJECT, version=DEFAULT_LATEST_VERSION, spider=SPIDER) response = client.get(url) text = get_text(response) assert ("selectedProject = '%s'" % PROJECT in text and "selectedVersion = 'default: the latest version'" in text and "this.loadSpiders();" in text and "selectedSpider = '%s'" % SPIDER in text)
def test_auto_eggifying_select_option(app, client): set_single_scrapyd(app) with app.test_request_context(): url = url_for('deploy.deploy', node=1) response = client.get(url) text = get_text(response) assert ('<option value ="%s"' % PROJECT in text and u'<option value ="demo"' in text and u'<option value ="demo - 副本"' in text and u'<option value ="demo_only_scrapy_cfg"' in text and u'<option value ="demo_without_scrapy_cfg"' not in text)
def test_auto_eggifying_only_scrapy_cfg(app, client): set_single_scrapyd(app) data = { 'project': 'demo_only_scrapy_cfg', 'version': VERSION, } with app.test_request_context(): url = url_for('deploy.upload', node=1) response = client.post(url, content_type='multipart/form-data', data=data) assert 'No module named' in get_text(response)
def test_deploy_xhr(app, client): set_single_scrapyd(app) with app.test_request_context(): eggname = '%s_%s_from_file_demo.egg' % (PROJECT, VERSION) url = url_for('deploy.deploy_xhr', node=1, eggname=eggname, project=PROJECT, version=VERSION) response = client.post(url) js = load_json(response) assert js['status'] == OK and js['project'] == PROJECT
def test_addversion(app, client): set_single_scrapyd(app) data = { 'project': 'fakeproject', 'version': 'fakeversion', 'file': (BytesIO(b'my file contents'), "fake.egg") } with app.test_request_context(): url = url_for('deploy.upload', node=1) response = client.post(url, content_type='multipart/form-data', data=data) assert 'activate_egg' in get_text(response)
def test_dropdown_for_mobile_device(app, client): set_single_scrapyd(app) with app.test_request_context(): url = url_for('dashboard', node=1) response = client.get(url, headers=HEADERS_DICT['Chrome']) text = get_text(response) assert 'dropdown_mobileui.css' not in text and 'handleDropdown()' not in text response = client.get(url, headers=HEADERS_DICT['iPhone']) text = get_text(response) assert 'dropdown_mobileui.css' in text and 'handleDropdown()' in text response = client.get(url, headers=HEADERS_DICT['iPad']) text = get_text(response) assert 'dropdown_mobileui.css' in text and 'handleDropdown()' in text
def test_index(app, client): set_single_scrapyd(app) url = '/' url_mobileui = '/?ui=mobile' for __, headers in HEADERS_DICT.items(): response = client.get(url_mobileui, headers=headers) assert response.headers['Location'].endswith('/1/dashboard/?ui=mobile') for key in ['Chrome', 'iPad']: response = client.get(url, headers=HEADERS_DICT[key]) assert response.headers['Location'].endswith('/1/dashboard/') for key in ['iPhone', 'Android']: response = client.get(url, headers=HEADERS_DICT[key]) assert response.headers['Location'].endswith('/1/dashboard/?ui=mobile')
def test_auto_eggifying_unicode(app, client): if WINDOWS_NOT_CP936: return set_single_scrapyd(app) data = { 'project': u'demo - 副本', 'version': VERSION, } with app.test_request_context(): url = url_for('deploy.upload', node=1) response = client.post(url, content_type='multipart/form-data', data=data) assert url_for('schedule.schedule', node=1, project='demo-', version=VERSION) in get_text(response)
def test_auto_eggifying(app, client): set_single_scrapyd(app) data = { 'project': PROJECT, 'version': VERSION, } with app.test_request_context(): url = url_for('deploy.upload', node=1) response = client.post(url, content_type='multipart/form-data', data=data) # http://localhost/1/schedule/ScrapydWeb-demo/2018-01-01T01_01_01/ # app.logger.debug(response.headers['Location']) redirect_url = url_for('schedule.schedule', node=1, project=PROJECT, version=VERSION) assert response.headers['Location'].endswith(redirect_url)
def test_check_update(app, client): set_single_scrapyd(app) with app.test_request_context(): @app.context_processor def inject_variable(): return dict(CHECK_LATEST_VERSION_FREQ=1) url = url_for('dashboard', node=1) response = client.get(url) text = get_text(response) assert ('<script>setTimeout("checkLatestVersion(' in text and '<!-- <script>setTimeout("checkLatestVersion(' not in text) @app.context_processor def inject_variable(): return dict(CHECK_LATEST_VERSION_FREQ=100) response = client.get(url) text = get_text(response) assert '<script>setTimeout("checkLatestVersion(' not in text
def test_history_log(app, client): set_single_scrapyd(app) with app.test_request_context(): url = url_for('schedule.history', filename='history.log') response = client.get(url) assert 'history.log' in get_text(response)
def test_upload_file_deploy(app, client): set_single_scrapyd(app) upload_file_deploy_singlenode = partial(upload_file_deploy, app=app, client=client, multinode=False) upload_file_deploy_singlenode(filename='demo.egg', project=PROJECT, redirect_project=PROJECT) upload_file_deploy_singlenode(filename='demo - Win7CNsendzipped.zip', project='demo - Win7CNsendzipped', redirect_project='demo-Win7CNsendzipped') upload_file_deploy_singlenode(filename='demo - Win10cp1252.zip', project='demo - Win10cp1252', redirect_project='demo-Win10cp1252') if WINDOWS_NOT_CP936: upload_file_deploy_singlenode(filename='demo - Ubuntu.zip', project='demo - Ubuntu', redirect_project='demo-Ubuntu') upload_file_deploy_singlenode(filename='demo - Ubuntu.tar.gz', project='demo - Ubuntu', redirect_project='demo-Ubuntu') upload_file_deploy_singlenode(filename='demo - macOS.zip', project='demo - macOS', redirect_project='demo-macOS') upload_file_deploy_singlenode(filename='demo - macOS.tar.gz', project='demo - macOS', redirect_project='demo-macOS') else: upload_file_deploy_singlenode(filename=u'副本.zip', project='demo_zip', redirect_project='demo_zip') upload_file_deploy_singlenode(filename=u'副本.tar.gz', project='demo_tar_gz', redirect_project='demo_tar_gz') upload_file_deploy_singlenode(filename=u'副本.egg', project='demo_egg', redirect_project='demo_egg') upload_file_deploy_singlenode(filename=u'demo - 副本 - Win7CN.zip', project=u'demo - 副本 - Win7CN', redirect_project='demo--Win7CN') upload_file_deploy_singlenode( filename=u'demo - 副本 - Win7CNsendzipped.zip', project=u'demo - 副本 - Win7CNsendzipped', redirect_project='demo--Win7CNsendzipped') upload_file_deploy_singlenode(filename=u'demo - 副本 - Win10cp936.zip', project=u'demo - 副本 - Win10cp936', redirect_project='demo--Win10cp936') upload_file_deploy_singlenode(filename=u'demo - 副本 - Ubuntu.zip', project=u'demo - 副本 - Ubuntu', redirect_project='demo--Ubuntu') upload_file_deploy_singlenode(filename=u'demo - 副本 - Ubuntu.tar.gz', project=u'demo - 副本 - Ubuntu', redirect_project='demo--Ubuntu') upload_file_deploy_singlenode(filename=u'demo - 副本 - macOS.zip', project=u'demo - 副本 - macOS', redirect_project='demo--macOS') upload_file_deploy_singlenode(filename=u'demo - 副本 - macOS.tar.gz', project=u'demo - 副本 - macOS', redirect_project='demo--macOS') upload_file_deploy_singlenode(filename='demo_inner.zip', project='demo_inner', redirect_project='demo_inner') upload_file_deploy_singlenode(filename='demo_outer.zip', project='demo_outer', redirect_project='demo_outer') upload_file_deploy_singlenode(filename='demo_without_scrapy_cfg.zip', project='demo_without_scrapy_cfg', alert='scrapy.cfg NOT found', fail=True) upload_file_deploy_singlenode(filename='demo_only_scrapy_cfg.zip', project='demo_only_scrapy_cfg', alert='No module named', fail=True)