def test_remove_duplicates_removes_duplicate(self, redis):
descriptor = {'a': {'b': 3}, 'c': 4}
self.etlhandler.create_redis_entry(task_id='456',
file_path='',
descriptor=descriptor,
data_type='')
data_dir = os.path.join(app.config['FRACTALIS_TMP_DIR'], 'data')
os.makedirs(data_dir, exist_ok=True)
Path(os.path.join(data_dir, '456')).touch()
assert redis.exists('data:456')
self.etlhandler.remove_duplicates(data_tasks=['456'],
descriptor=descriptor)
assert not redis.exists('data:456')
def test_session_matched_redis_in_post_big_payload(
self, test_client, payload):
data = payload()
test_client.post('/data', data=data['serialized'])
with test_client.session_transaction() as sess:
for task_id in sess['data_tasks']:
assert redis.exists('data:{}'.format(task_id))
def sanity_check(self):
"""Check whether ETL is still sane and should be continued. E.g. if
redis has been cleared it does not make sense to proceed. Raise an
exception if not sane."""
check_1 = redis.exists('data:{}'.format(self.request.id))
if not check_1:
error = "ETL failed! The associated entry in " \
"Redis has been removed while the ETL was running."
logger.error(error)
raise RuntimeError(error)
def test_valid_state_for_failed_etl_on_delete(self, test_client, faiload):
test_client.post('/data?wait=1', data=faiload['serialized'])
for key in redis.keys('data:*'):
value = redis.get(key)
data_state = json.loads(value)
assert not os.path.exists(data_state['file_path'])
test_client.delete('/data/{}?wait=1'.format(data_state['task_id']))
assert not redis.exists(key)
assert not os.path.exists(data_state['file_path'])
with test_client.session_transaction() as sess:
assert data_state['task_id'] not in sess['data_tasks']
def test_403_if_not_authorized(self, test_client, payload):
data = payload()
test_client.post('/data', data=data['serialized'])
for key in redis.keys('data:*'):
value = redis.get(key)
data_state = json.loads(value)
rv = test_client.get('/data/meta/{}?wait=1'
.format(data_state['task_id']))
body = flask.json.loads(rv.get_data())
assert rv.status_code == 403
assert 'Access unauthorized.' in body['error']
assert redis.exists(key)
def test_403_if_no_auth_on_get_meta(self, test_client, payload):
data = payload()
test_client.post('/data?wait=1', data=data['serialized'])
with test_client.session_transaction() as sess:
sess['data_tasks'] = []
for key in redis.keys('data:*'):
value = redis.get(key)
data_state = json.loads(value)
rv = test_client.get('/data/meta/{}?wait=1'
.format(data_state['task_id']))
body = flask.json.loads(rv.get_data())
assert rv.status_code == 403
assert 'Refusing access.' in body['error']
assert redis.exists(key)
assert os.path.exists(data_state['file_path'])