def test_remove_duplicates_removes_duplicate(self, redis): descriptor = {'a': {'b': 3}, 'c': 4} self.etlhandler.create_redis_entry(task_id='456', file_path='', descriptor=descriptor, data_type='') data_dir = os.path.join(app.config['FRACTALIS_TMP_DIR'], 'data') os.makedirs(data_dir, exist_ok=True) Path(os.path.join(data_dir, '456')).touch() assert redis.exists('data:456') self.etlhandler.remove_duplicates(data_tasks=['456'], descriptor=descriptor) assert not redis.exists('data:456')
def test_session_matched_redis_in_post_big_payload( self, test_client, payload): data = payload() test_client.post('/data', data=data['serialized']) with test_client.session_transaction() as sess: for task_id in sess['data_tasks']: assert redis.exists('data:{}'.format(task_id))
def sanity_check(self): """Check whether ETL is still sane and should be continued. E.g. if redis has been cleared it does not make sense to proceed. Raise an exception if not sane.""" check_1 = redis.exists('data:{}'.format(self.request.id)) if not check_1: error = "ETL failed! The associated entry in " \ "Redis has been removed while the ETL was running." logger.error(error) raise RuntimeError(error)
def test_valid_state_for_failed_etl_on_delete(self, test_client, faiload): test_client.post('/data?wait=1', data=faiload['serialized']) for key in redis.keys('data:*'): value = redis.get(key) data_state = json.loads(value) assert not os.path.exists(data_state['file_path']) test_client.delete('/data/{}?wait=1'.format(data_state['task_id'])) assert not redis.exists(key) assert not os.path.exists(data_state['file_path']) with test_client.session_transaction() as sess: assert data_state['task_id'] not in sess['data_tasks']
def test_403_if_not_authorized(self, test_client, payload): data = payload() test_client.post('/data', data=data['serialized']) for key in redis.keys('data:*'): value = redis.get(key) data_state = json.loads(value) rv = test_client.get('/data/meta/{}?wait=1' .format(data_state['task_id'])) body = flask.json.loads(rv.get_data()) assert rv.status_code == 403 assert 'Access unauthorized.' in body['error'] assert redis.exists(key)
def test_403_if_no_auth_on_get_meta(self, test_client, payload): data = payload() test_client.post('/data?wait=1', data=data['serialized']) with test_client.session_transaction() as sess: sess['data_tasks'] = [] for key in redis.keys('data:*'): value = redis.get(key) data_state = json.loads(value) rv = test_client.get('/data/meta/{}?wait=1' .format(data_state['task_id'])) body = flask.json.loads(rv.get_data()) assert rv.status_code == 403 assert 'Refusing access.' in body['error'] assert redis.exists(key) assert os.path.exists(data_state['file_path'])