def hdfs_file(project_id, cluster, path):
if not current_app.config.get('HDFS_CONFIG'):
raise NotFound('Not Found')
signature = request.args.get('task-signature')
if not signature:
raise Forbidden('No signature')
project = get_project_data(project_id)
timeout = project['info'].get('timeout', ContributionsGuard.STAMP_TTL)
payload = signer.loads(signature, max_age=timeout)
task_id = payload['task_id']
check_allowed(current_user.id, task_id, project, request.path)
client = HDFSKerberos(**current_app.config['HDFS_CONFIG'][cluster])
try:
content = client.get('/{}'.format(path))
project_encryption = project['info'].get('ext_config',
{}).get('encryption', {})
if project_encryption and all(project_encryption.values()):
secret = get_secret_from_vault(project_encryption)
cipher = AESWithGCM(secret)
content = cipher.decrypt(content)
except Exception:
current_app.logger.exception('Project id {} get task file {}'.format(
project_id, path))
raise InternalServerError('An Error Occurred')
return Response(content)
def test_get(self, subprocess):
reader = MagicMock()
reader.read.return_value = 'abc'
reader.__enter__.return_value = reader
def read(*args, **kwargs):
return reader
client = HDFSKerberos('testurl', 'testuser')
with patch.object(client, 'read', read):
assert client.get('test') == 'abc'
def hdfs_file(project_id, cluster, path):
if not current_app.config.get('HDFS_CONFIG'):
raise NotFound('Not Found')
signature = request.args.get('task-signature')
if not signature:
raise Forbidden('No signature')
size_signature = len(signature)
if size_signature > TASK_SIGNATURE_MAX_SIZE:
current_app.logger.exception(
'Project id {}, cluster {} path {} invalid task signature. Signature length {} exceeds max allowed length {}.' \
.format(project_id, cluster, path, size_signature, TASK_SIGNATURE_MAX_SIZE))
raise Forbidden('Invalid signature')
project = get_project_data(project_id)
timeout = project['info'].get('timeout', ContributionsGuard.STAMP_TTL)
payload = signer.loads(signature, max_age=timeout)
task_id = payload['task_id']
try:
check_allowed(
current_user.id, task_id, project,
is_valid_hdfs_url(request.path, request.args.to_dict(flat=False)))
except Exception:
current_app.logger.exception(
'Project id %s not allowed to get file %s %s', project_id, path,
str(request.args))
raise
current_app.logger.info(
"Project id %s, task id %s. Accessing hdfs cluster %s, path %s",
project_id, task_id, cluster, path)
client = HDFSKerberos(**current_app.config['HDFS_CONFIG'][cluster])
offset = request.args.get('offset')
length = request.args.get('length')
try:
offset = int(offset) if offset else None
length = int(length) if length else None
content = client.get('/{}'.format(path), offset=offset, length=length)
project_encryption = get_project_encryption(project)
if project_encryption and all(project_encryption.values()):
secret = get_secret_from_vault(project_encryption)
cipher = AESWithGCM(secret)
content = cipher.decrypt(content)
except Exception:
current_app.logger.exception(
"Project id %s, task id %s, cluster %s, get task file %s, %s",
project_id, task_id, cluster, path, str(request.args))
raise InternalServerError('An Error Occurred')
return Response(content)
def test_put(self, subprocess):
client = HDFSKerberos('testurl', 'testuser')
writer = MagicMock()
with patch.object(client, 'write'):
client.put('test', 'hello')
client.write.assert_called()
def test_kinit_error(self, subprocess):
client = HDFSKerberos('testurl', 'testuser')
subprocess.side_effect = [CalledProcessError('', ''), CalledProcessError('', '')]
assert_raises(CalledProcessError, client.get_ticket)
def test_init_needed(self, subprocess):
client = HDFSKerberos('testurl', 'testuser')
subprocess.side_effect = [CalledProcessError('', ''), 0]
client.get_ticket()
assert subprocess.call_count == 2
def test_init_not_needed(self, subprocess):
client = HDFSKerberos('testurl', 'testuser')
client.get_ticket()
subprocess.return_value = 0
assert subprocess.call_count == 1