def test_handle_uses_duplicate_of_previous_iteration( self, monkeypatch, redis): descriptor = {'data_type': 'default'} task_ids = self.etlhandler.handle(descriptors=[descriptor, descriptor], data_tasks=[], use_existing=True) assert len(task_ids) == 1 assert len(redis.keys('data:*')) == 1
def test_valid_state_for_finished_etl_on_delete_all( self, test_client, payload): data_dir = os.path.join(app.config['FRACTALIS_TMP_DIR'], 'data') data = payload() test_client.post('/data?wait=1', data=data['serialized']) test_client.delete('/data?wait=1') assert not redis.keys('data:*') assert len(os.listdir(data_dir)) == 0 with test_client.session_transaction() as sess: assert not sess['data_tasks']
def test_valid_response_before_loaded_on_meta(self, test_client, payload): data = payload() test_client.post('/data', data=data['serialized']) for key in redis.keys('data:*'): value = redis.get(key) data_state = json.loads(value) rv = test_client.get('/data/meta/{}'.format(data_state['task_id'])) body = flask.json.loads(rv.get_data()) assert rv.status_code == 200 assert 'features' not in body['meta']
def test_valid_state_for_failed_etl_on_delete(self, test_client, faiload): test_client.post('/data?wait=1', data=faiload['serialized']) for key in redis.keys('data:*'): value = redis.get(key) data_state = json.loads(value) assert not os.path.exists(data_state['file_path']) test_client.delete('/data/{}?wait=1'.format(data_state['task_id'])) assert not redis.exists(key) assert not os.path.exists(data_state['file_path']) with test_client.session_transaction() as sess: assert data_state['task_id'] not in sess['data_tasks']
def test_many_post_and_valid_state(self, test_client, payload): requests = 5 data_dir = os.path.join(app.config['FRACTALIS_TMP_DIR'], 'data') size = 0 for i in range(requests): data = payload() size += data['size'] rv = test_client.post('/data?wait=1', data=data['serialized']) assert rv.status_code == 201 assert len(os.listdir(data_dir)) == size assert len(redis.keys('data:*')) == size
def test_403_if_not_authorized(self, test_client, payload): data = payload() test_client.post('/data', data=data['serialized']) for key in redis.keys('data:*'): value = redis.get(key) data_state = json.loads(value) rv = test_client.get('/data/meta/{}?wait=1' .format(data_state['task_id'])) body = flask.json.loads(rv.get_data()) assert rv.status_code == 403 assert 'Access unauthorized.' in body['error'] assert redis.exists(key)
def test_encryption_works(self, test_client, payload): app.config['FRACTALIS_ENCRYPT_CACHE'] = True data = payload() test_client.post('/data?wait=1', data=data['serialized']) keys = redis.keys('data:*') for key in keys: value = redis.get(key) data_state = json.loads(value) file_path = data_state['file_path'] with pytest.raises(UnicodeDecodeError): open(file_path, 'r').readlines() app.config['FRACTALIS_ENCRYPT_CACHE'] = False
def test_valid_filesystem_before_loaded_on_post( self, test_client, payload): data_dir = os.path.join(app.config['FRACTALIS_TMP_DIR'], 'data') data = payload() test_client.post('/data', data=data['serialized']) if os.path.exists(data_dir): assert len(os.listdir(data_dir)) == 0 keys = redis.keys('data:*') for key in keys: value = redis.get(key) data_state = json.loads(value) assert not os.path.exists(data_state['file_path'])
def test_valid_redis_after_loaded_on_post(self, test_client, payload): data = payload() test_client.post('/data?wait=1', data=data['serialized']) keys = redis.keys('data:*') assert len(keys) == data['size'] for key in keys: value = redis.get(key) data_state = json.loads(value) assert 'file_path' in data_state assert 'label' in data_state assert 'data_type' in data_state assert 'meta' in data_state
def test_403_if_no_auth_on_get_meta(self, test_client, payload): data = payload() test_client.post('/data?wait=1', data=data['serialized']) with test_client.session_transaction() as sess: sess['data_tasks'] = [] for key in redis.keys('data:*'): value = redis.get(key) data_state = json.loads(value) rv = test_client.get('/data/meta/{}?wait=1' .format(data_state['task_id'])) body = flask.json.loads(rv.get_data()) assert rv.status_code == 403 assert 'Refusing access.' in body['error'] assert redis.exists(key) assert os.path.exists(data_state['file_path'])
def cleanup_all() -> None: """Reset redis, celery and the filesystem. This is only useful for testing and should !!!NEVER!!! be used for anything else. """ celery.control.purge() for key in redis.keys('data:*'): value = redis.get(key) try: data_state = json.loads(value) except ValueError: continue task_id = data_state.get('task_id') if task_id is not None: async_result = celery.AsyncResult(task_id) if async_result.state == 'SUBMITTED': async_result.get(propagate=False) redis.flushall() tmp_dir = app.config['FRACTALIS_TMP_DIR'] if os.path.exists(tmp_dir): rmtree(tmp_dir) assert not os.path.exists(tmp_dir)