def test_handle_uses_duplicate_of_previous_iteration(
self, monkeypatch, redis):
descriptor = {'data_type': 'default'}
task_ids = self.etlhandler.handle(descriptors=[descriptor, descriptor],
data_tasks=[],
use_existing=True)
assert len(task_ids) == 1
assert len(redis.keys('data:*')) == 1
def test_valid_state_for_finished_etl_on_delete_all(
self, test_client, payload):
data_dir = os.path.join(app.config['FRACTALIS_TMP_DIR'], 'data')
data = payload()
test_client.post('/data?wait=1', data=data['serialized'])
test_client.delete('/data?wait=1')
assert not redis.keys('data:*')
assert len(os.listdir(data_dir)) == 0
with test_client.session_transaction() as sess:
assert not sess['data_tasks']
def test_valid_response_before_loaded_on_meta(self, test_client, payload):
data = payload()
test_client.post('/data', data=data['serialized'])
for key in redis.keys('data:*'):
value = redis.get(key)
data_state = json.loads(value)
rv = test_client.get('/data/meta/{}'.format(data_state['task_id']))
body = flask.json.loads(rv.get_data())
assert rv.status_code == 200
assert 'features' not in body['meta']
def test_valid_state_for_failed_etl_on_delete(self, test_client, faiload):
test_client.post('/data?wait=1', data=faiload['serialized'])
for key in redis.keys('data:*'):
value = redis.get(key)
data_state = json.loads(value)
assert not os.path.exists(data_state['file_path'])
test_client.delete('/data/{}?wait=1'.format(data_state['task_id']))
assert not redis.exists(key)
assert not os.path.exists(data_state['file_path'])
with test_client.session_transaction() as sess:
assert data_state['task_id'] not in sess['data_tasks']
def test_many_post_and_valid_state(self, test_client, payload):
requests = 5
data_dir = os.path.join(app.config['FRACTALIS_TMP_DIR'], 'data')
size = 0
for i in range(requests):
data = payload()
size += data['size']
rv = test_client.post('/data?wait=1', data=data['serialized'])
assert rv.status_code == 201
assert len(os.listdir(data_dir)) == size
assert len(redis.keys('data:*')) == size
def test_403_if_not_authorized(self, test_client, payload):
data = payload()
test_client.post('/data', data=data['serialized'])
for key in redis.keys('data:*'):
value = redis.get(key)
data_state = json.loads(value)
rv = test_client.get('/data/meta/{}?wait=1'
.format(data_state['task_id']))
body = flask.json.loads(rv.get_data())
assert rv.status_code == 403
assert 'Access unauthorized.' in body['error']
assert redis.exists(key)
def test_encryption_works(self, test_client, payload):
app.config['FRACTALIS_ENCRYPT_CACHE'] = True
data = payload()
test_client.post('/data?wait=1', data=data['serialized'])
keys = redis.keys('data:*')
for key in keys:
value = redis.get(key)
data_state = json.loads(value)
file_path = data_state['file_path']
with pytest.raises(UnicodeDecodeError):
open(file_path, 'r').readlines()
app.config['FRACTALIS_ENCRYPT_CACHE'] = False
def test_valid_filesystem_before_loaded_on_post(
self, test_client, payload):
data_dir = os.path.join(app.config['FRACTALIS_TMP_DIR'], 'data')
data = payload()
test_client.post('/data', data=data['serialized'])
if os.path.exists(data_dir):
assert len(os.listdir(data_dir)) == 0
keys = redis.keys('data:*')
for key in keys:
value = redis.get(key)
data_state = json.loads(value)
assert not os.path.exists(data_state['file_path'])
def test_valid_redis_after_loaded_on_post(self, test_client, payload):
data = payload()
test_client.post('/data?wait=1', data=data['serialized'])
keys = redis.keys('data:*')
assert len(keys) == data['size']
for key in keys:
value = redis.get(key)
data_state = json.loads(value)
assert 'file_path' in data_state
assert 'label' in data_state
assert 'data_type' in data_state
assert 'meta' in data_state
def test_403_if_no_auth_on_get_meta(self, test_client, payload):
data = payload()
test_client.post('/data?wait=1', data=data['serialized'])
with test_client.session_transaction() as sess:
sess['data_tasks'] = []
for key in redis.keys('data:*'):
value = redis.get(key)
data_state = json.loads(value)
rv = test_client.get('/data/meta/{}?wait=1'
.format(data_state['task_id']))
body = flask.json.loads(rv.get_data())
assert rv.status_code == 403
assert 'Refusing access.' in body['error']
assert redis.exists(key)
assert os.path.exists(data_state['file_path'])
def cleanup_all() -> None:
"""Reset redis, celery and the filesystem. This is only useful for testing
and should !!!NEVER!!! be used for anything else.
"""
celery.control.purge()
for key in redis.keys('data:*'):
value = redis.get(key)
try:
data_state = json.loads(value)
except ValueError:
continue
task_id = data_state.get('task_id')
if task_id is not None:
async_result = celery.AsyncResult(task_id)
if async_result.state == 'SUBMITTED':
async_result.get(propagate=False)
redis.flushall()
tmp_dir = app.config['FRACTALIS_TMP_DIR']
if os.path.exists(tmp_dir):
rmtree(tmp_dir)
assert not os.path.exists(tmp_dir)
