def test_repository():
source = load_json('source/repository.json')
expectation = load_json('correct/repository.json')
actual_result = [Repository(**x).to_dict() for x in source]
assert actual_result == expectation
def test_check(app, client):
set_single_scrapyd(app)
data = {
'project': PROJECT,
'_version': VERSION,
'spider': SPIDER,
'jobid': JOBID,
'USER_AGENT': 'chrome',
'COOKIES_ENABLED': 'False',
'ROBOTSTXT_OBEY': 'False',
'CONCURRENT_REQUESTS': '1',
'DOWNLOAD_DELAY': '2',
'additional': '-d setting=CLOSESPIDER_TIMEOUT=60 \r\n-d setting=CLOSESPIDER_PAGECOUNT=10 \r\n-d arg1=val1'
}
data_ = {
'project': PROJECT,
'_version': DEFAULT_LATEST_VERSION,
'spider': SPIDER,
'additional': '-d setting=CLOSESPIDER_TIMEOUT=60 -d arg1'
}
with app.test_request_context():
url = url_for('schedule.check', node=1)
response = client.post(url, data=data)
# js = response.get_json()
js = load_json(response)
assert js['filename'] == '%s_%s_%s.pickle' % (PROJECT, VERSION, SPIDER)
response = client.post(url, data=data_)
js = load_json(response)
assert js['filename'] == '%s_%s_%s.pickle' % (PROJECT, 'default-the-latest-version', SPIDER)
def test_delversion(app, client):
with app.test_request_context():
url = url_for('api', node=1, opt='delversion', project=PROJECT, version_spider_job=VERSION)
response = client.get(url)
js = load_json(response)
assert ((js['status'] == OK and 'delversion.json' in js['url']) or
(js['status'] == ERROR and '%s.egg' % VERSION in js['message']))
def test_delproject(app, client):
with app.test_request_context():
url = url_for('api', node=1, opt='delproject', project=PROJECT)
response = client.get(url)
js = load_json(response)
assert ((js['status'] == OK and 'delproject.json' in js['url']) or
(js['status'] == ERROR and PROJECT in js['message']))
def test_get_token(url, data, expected, status_code, client, monkeypatch):
monkeypatch.setattr('app.mod_api.views.current_app.facilities',
{'wolfsburg': {}})
rv = client.post(url, data=json.dumps(data),
headers={'content-type': 'application/json'})
assert rv.status_code == status_code
if expected is not True:
assert load_json(rv.data) == expected
else:
token = load_json(rv.data)['token']
assert data == verify_token(token)
def test_basic(self, db_session: Session) -> None:
"""Check creating a basic job works as expected."""
job_service = get_configured_job_service(db_session, files=False)
user = get_random_user()
job_data = load_json('pg')
# here: Job is added to database but dag file is not created (mocked away, as it happens in a separate pkg)
result = job_service.create(user=user, **job_data)
assert result['status'] == 'success'
assert result['code'] == 201
assert result['headers']['Location'].startswith('jobs/jb-')
assert result['headers']['OpenEO-Identifier'].startswith('jb-')
results_job_id = result['headers']['OpenEO-Identifier']
assert results_job_id == result['headers']['Location'][5:]
assert db_session.query(Job).filter(Job.user_id == user["id"]).filter(
Job.id == results_job_id).count() == 1
dag_handler = DagHandler()
assert not isfile(
dag_handler.get_dag_path_from_id(
dag_handler.get_preparation_dag_id(
job_id=result['headers']['OpenEO-Identifier'])))
job_service.processes_service.put_user_defined.assert_called_once_with(
user=user, process_graph_id="pg_id", **job_data["process"])
def test_parser():
with open(f'{BASE_DIR}/fixtures/source/maillog') as fd:
logs = fd.read().splitlines()
expectation = load_json('correct/parsed_maillog.json')
actual_results = [parse(log) for log in logs]
assert actual_results == expectation
def test_schedule_xhr(app, client):
with app.test_request_context():
url = url_for('schedule.schedule_xhr',
node=2,
filename='%s_%s_%s.pickle' % (PROJECT, VERSION, SPIDER))
response = client.post(url)
js = load_json(response)
assert js['status'] == ERROR
def test_stop(app, client):
upload_file_deploy(app, client, filename='demo.egg', project=PROJECT, redirect_project=PROJECT)
with app.test_request_context():
url = url_for('api', node=1, opt='stop', project=PROJECT, version_spider_job=JOBID)
response = client.get(url)
js = load_json(response)
assert js['status'] == OK and 'prevstate' in js and 'times' not in js # js['prevstate'] == 'running'
def test_schedule_xhr(app, client):
set_single_scrapyd(app)
with app.test_request_context():
url = url_for('schedule.schedule_xhr', node=1, filename='%s_%s_%s.pickle' % (PROJECT, VERSION, SPIDER))
response = client.post(url)
js = load_json(response)
assert js['status'] == OK and js['jobid'] == JOBID
client.get(url_for('api', node=1, opt='forcestop', project=PROJECT, version_spider_job=JOBID))
def result(self):
if self._result is None:
path = self.base + ".result.json"
if not os.path.isfile(path):
pytest.xfail(
f"Result JSON does not exist for test {self.category}/{self.name}"
)
self._result = load_json(path)
return self._result
def test_valid_token(url, status_code, data, flask_app):
@flask_app.route('/<facility>/valid')
@valid_token
def valid(facility):
return jsonify({'access': True})
rv = flask_app.test_client().get(url)
assert rv.status_code == status_code
assert load_json(rv.data) == data
def test_deploy_xhr(app, client):
with app.test_request_context():
eggname = '%s_%s_from_file_demo.egg' % (PROJECT, VERSION)
url = url_for('deploy.deploy_xhr',
node=1,
eggname=eggname,
project=PROJECT,
version=VERSION)
response = client.post(url)
js = load_json(response)
assert js['status'] == OK and js['project'] == PROJECT
def bench(self):
if self._bench is None:
path = self.base + ".bench.json"
if not os.path.isfile(path):
pytest.xfail(
f"Benchmark JSON does not exist for test {self.category}/{self.name}"
)
self._result = load_json(path)
return self._bench
def test_check(app, client):
data = {
'project': PROJECT,
'_version': VERSION,
'spider': SPIDER,
'jobid': JOBID
}
with app.test_request_context():
url = url_for('schedule.check', node=2)
response = client.post(url, data=data)
js = load_json(response)
assert js['filename'] == '%s_%s_%s.pickle' % (PROJECT, VERSION, SPIDER)
def test_api_forcestop(app, client):
with app.test_request_context():
url = url_for('api',
node=1,
opt='forcestop',
project=PROJECT,
version_spider_job=jobid,
ui='simple')
response = client.get(url)
js = load_json(response)
assert js['status'] == OK and js['prevstate'] is None and js[
'times'] == 2
def test_writer(writer: Type[FileWriter], fmt: str, answer_filename: str):
output_filename_for_test = 'dummy'
records = load_json('source/records.json')
obj = writer(filepath=output_filename_for_test)
obj.handle(records)
expected_filepath = f'{output_filename_for_test}.{fmt}'
assert obj.filepath == expected_filepath
is_same = is_same_file_contents(
obj.filepath, f'{BASE_DIR}/fixtures/correct/{answer_filename}')
remove_file(expected_filepath)
assert is_same is True
def test_lent_list(lent_return, expected, client, monkeypatch):
monkeypatch.setattr('app.mod_api.views.current_app.facilities',
{
'wolfsburg': {
'lent_list': lambda x, y: lent_return
}
})
rv = client.get(
('/api/wolfsburg/lent?token='
'eyJwYXNzd29yZCI6ImJhciIsInVzZXJuYW1lIjoiZm9vIn0.'
'pIUBfh1BSvoROF8wgHsebtQyFK8'))
assert load_json(rv.data) == expected
def test_search(search_return, expected, client, monkeypatch):
monkeypatch.setattr(
'app.mod_api.views.current_app.facilities',
{
'wolfsburg': {
'search': lambda term, page: search_return
}
})
rv = client.post('/api/wolfsburg/search',
data=json.dumps({'term': 'batman'}),
headers={'content-type': 'application/json'})
assert load_json(rv.data) == expected
def test_api_stop(app, client):
sleep()
with app.test_request_context():
url = url_for('api',
node=1,
opt='stop',
project=PROJECT,
version_spider_job=jobid,
ui='mobile')
response = client.get(url)
js = load_json(response)
assert js['status'] == OK and js[
'prevstate'] == 'running' and 'times' not in js
def test_valid_facility(facility, status_code, data, flask_app, monkeypatch):
class MockApp(object):
facilities = {'wolfsburg': {}}
monkeypatch.setattr('app.mod_api.decorators.current_app', MockApp)
@flask_app.route('/<facility>/valid')
@valid_facility
def valid(facility):
return jsonify({'access': True})
rv = flask_app.test_client().get('/{}/valid'.format(facility))
assert rv.status_code == status_code
assert load_json(rv.data) == data
def test_facility_list(client, monkeypatch):
monkeypatch.setattr('app.mod_api.views.current_app.facilities',
{
'wolfsburg': {'metadata': {}},
'nuernberg': {'metadata': {}},
'paris': {'metadata': {}}
})
rv = client.get('/api/facilities')
assert rv.status_code == 200
data = load_json(rv.data)
assert len(data['facilities']) == 3
assert 'wolfsburg' in data['facilities']
assert 'nuernberg' in data['facilities']
assert 'paris' in data['facilities']
def test_check(app, client):
data = {
'project':
PROJECT,
'_version':
VERSION,
'spider':
SPIDER,
'jobid':
JOBID,
'additional':
"-d setting=CLOSESPIDER_TIMEOUT=10 \r\n-d setting=CLOSESPIDER_PAGECOUNT=1 \r\n-d arg1=val1",
}
with app.test_request_context():
url = url_for('schedule.check', node=1)
response = client.post(url, data=data)
# js = response.get_json()
js = load_json(response)
assert js['filename'] == '%s_%s_%s.pickle' % (PROJECT, VERSION, SPIDER)
def test_api_start(app, client):
global jobid
upload_file_deploy(app,
client,
filename='demo.egg',
project=PROJECT,
redirect_project=PROJECT)
with app.test_request_context():
url = url_for('api',
node=1,
opt='start',
project=PROJECT,
version_spider_job=SPIDER,
ui='mobile')
response = client.get(url)
js = load_json(response)
jobid = js['jobid']
assert js['status'] == OK and js['jobid']
def test_log_utf8_stats(app, client):
upload_file_deploy(app, client, filename='demo.egg', project=PROJECT, redirect_project=PROJECT)
with app.test_request_context():
url = url_for('api', node=1, opt='start', project=PROJECT, version_spider_job=SPIDER)
response = client.get(url)
js = load_json(response)
jobid = js['jobid']
sleep()
# UTF8 page
url = url_for('log', node=1, opt='utf8', project=PROJECT, spider=SPIDER, job=jobid)
response = client.get(url)
assert 'utf8 - ScrapydWeb' in get_text(response) and not is_simple_ui(response)
client.get(url_for('api', node=1, opt='forcestop', project=PROJECT, version_spider_job=jobid))
# Stats page
url = url_for('log', node=1, opt='stats', project=PROJECT, spider=SPIDER, job=jobid)
response = client.get(url)
assert 'Stats collection' in get_text(response) and not is_simple_ui(response)
def test_start_processing_sync_job(self, db_session: Session) -> None:
"""Check process_sync works as expected."""
job_service = get_configured_job_service(db_session)
user = get_random_user()
job_data = load_json('pg')
_ = job_data.pop("title")
_ = job_data.pop("description")
result = job_service.process_sync(user=user, **job_data)
assert result['status'] == 'success'
assert 'result/sample-output.tif' in result['file']
_ = result.pop('file')
assert result == {
'code': 200,
'status': 'success',
'headers': {
'Content-Type': 'image/tiff',
'OpenEO-Costs': 0
}
}
def test_email(app, client):
with app.test_request_context():
if not app.config.get('ENABLE_EMAIL', False):
return
# Simulate caching post 'Finished'
url = url_for('api',
node=1,
opt='start',
project=PROJECT,
version_spider_job=SPIDER)
response = client.get(url)
js = load_json(response)
jobid = js['jobid']
sleep()
url = url_for('log',
node=1,
opt='stats',
project=PROJECT,
spider=SPIDER,
job=jobid,
job_finished='True')
response = client.post(url, content_type='multipart/form-data')
assert 'Stats collection' in get_text(response)
client.get(
url_for('api',
node=1,
opt='forcestop',
project=PROJECT,
version_spider_job=jobid))
# Simulate caching post 'ForceStopped'
app.config['ON_JOB_FINISHED'] = False
url = url_for('api',
node=1,
opt='start',
project=PROJECT,
version_spider_job=SPIDER)
response = client.get(url)
js = load_json(response)
jobid = js['jobid']
sleep()
url = url_for('log',
node=1,
opt='stats',
project=PROJECT,
spider=SPIDER,
job=jobid,
job_finished='')
response = client.post(url, content_type='multipart/form-data')
assert 'Stats collection' in get_text(response)
client.get(
url_for('api',
node=1,
opt='forcestop',
project=PROJECT,
version_spider_job=jobid))
# Simulate caching post 'Stopped'
app.config['LOG_CRITICAL_THRESHOLD'] = 0
app.config['LOG_REDIRECT_THRESHOLD'] = 1
app.config['LOG_REDIRECT_TRIGGER_STOP'] = True
url = url_for('api',
node=1,
opt='start',
project=PROJECT,
version_spider_job=SPIDER)
response = client.get(url)
js = load_json(response)
jobid = js['jobid']
sleep()
url = url_for('log',
node=1,
opt='stats',
project=PROJECT,
spider=SPIDER,
job=jobid,
job_finished='')
response = client.post(url, content_type='multipart/form-data')
assert 'Stats collection' in get_text(response)
client.get(
url_for('api',
node=1,
opt='forcestop',
project=PROJECT,
version_spider_job=jobid))
# Simulate caching post 'Triggered'
app.config['LOG_REDIRECT_THRESHOLD'] = 0
app.config['LOG_IGNORE_THRESHOLD'] = 1
url = url_for('api',
node=1,
opt='start',
project=PROJECT,
version_spider_job=SPIDER)
response = client.get(url)
js = load_json(response)
jobid = js['jobid']
sleep()
url = url_for('log',
node=1,
opt='stats',
project=PROJECT,
spider=SPIDER,
job=jobid,
job_finished='')
response = client.post(url, content_type='multipart/form-data')
assert 'Stats collection' in get_text(response)
client.get(
url_for('api',
node=1,
opt='forcestop',
project=PROJECT,
version_spider_job=jobid))
# Simulate caching post 'Running'
app.config['LOG_IGNORE_THRESHOLD'] = 0
app.config['ON_JOB_RUNNING_INTERVAL'] = 5
url = url_for('api',
node=1,
opt='start',
project=PROJECT,
version_spider_job=SPIDER)
response = client.get(url)
js = load_json(response)
jobid = js['jobid']
# Would NOT trigger email
sleep()
url = url_for('log',
node=1,
opt='stats',
project=PROJECT,
spider=SPIDER,
job=jobid,
job_finished='')
response = client.post(url, content_type='multipart/form-data')
assert 'Stats collection' in get_text(response)
# Would trigger email
sleep()
url = url_for('log',
node=1,
opt='stats',
project=PROJECT,
spider=SPIDER,
job=jobid,
job_finished='')
response = client.post(url, content_type='multipart/form-data')
assert 'Stats collection' in get_text(response)
# Would NOT trigger email
app.config['ON_JOB_RUNNING_INTERVAL'] = 0
sleep()
url = url_for('log',
node=1,
opt='stats',
project=PROJECT,
spider=SPIDER,
job=jobid,
job_finished='')
response = client.post(url, content_type='multipart/form-data')
assert 'Stats collection' in get_text(response)
client.get(
url_for('api',
node=1,
opt='forcestop',
project=PROJECT,
version_spider_job=jobid))
def test_listprojects(app, client):
with app.test_request_context():
url = url_for('api', node=1, opt='listprojects')
response = client.get(url)
js = load_json(response)
assert js['status'] == OK and 'projects' in js
def test_daemonstatus(app, client):
with app.test_request_context():
url = url_for('api', node=1, opt='daemonstatus')
response = client.get(url)
js = load_json(response)
assert js['status'] == OK and 'running' in js
def compact_arbor(reference_dir):
return load_json(reference_dir, "compact-arbor")
def compact_skeleton(reference_dir):
return load_json(reference_dir, "compact-skeleton")
def listspiders(app, client, version):
with app.test_request_context():
url = url_for('api', node=1, opt='listspiders', project=PROJECT, version_spider_job=version)
response = client.get(url)
js = load_json(response)
assert js['status'] == OK and SPIDER in js['spiders']
def json(self):
return load_json(self._path)
def test_listjobs(app, client):
with app.test_request_context():
url = url_for('api', node=1, opt='listjobs', project=PROJECT)
response = client.get(url)
js = load_json(response)
assert js['status'] == OK and 'listjobs.json' in js['url']
def setUp(self) -> None:
"""Instantiate TemporalToSnapshot"""
self.resp = load_json("testData/nfl-week-three.json")
self.line_event = LinesEventFacade(self.resp.get("events")[0])
self.test_ttss = TemporalToSnapshot()
def test_log_utf8_stats(app, client):
upload_file_deploy(app,
client,
filename='demo.egg',
project=PROJECT,
redirect_project=PROJECT)
with app.test_request_context():
url = url_for('api',
node=1,
opt='start',
project=PROJECT,
version_spider_job=SPIDER)
response = client.get(url)
js = load_json(response)
jobid = js['jobid']
sleep()
# Log page
url = url_for('log',
node=1,
opt='utf8',
project=PROJECT,
spider=SPIDER,
job=jobid)
response = client.get(url)
assert 'log - ScrapydWeb' in get_text(
response) and not is_mobileui(response)
# Stats page
url = url_for('log',
node=1,
opt='stats',
project=PROJECT,
spider=SPIDER,
job=jobid)
response = client.get(url)
assert 'Stats collection' in get_text(
response) and not is_mobileui(response)
# Dashboard page
url = url_for('dashboard', node=1)
response = client.get(url)
url_stop = url_for('api',
node=1,
opt='stop',
project=PROJECT,
version_spider_job=jobid)
assert url_stop in get_text(response)
client.get(
url_for('api',
node=1,
opt='forcestop',
project=PROJECT,
version_spider_job=jobid))
# /1/schedule/ScrapydWeb-demo/default:%20the%20latest%20version/test/
response = client.get(url)
url_start = url_for('schedule.schedule',
node=1,
project=PROJECT,
version=DEFAULT_LATEST_VERSION,
spider=SPIDER)
assert url_start in get_text(response)
