def hdfs_file(project_id, cluster, path): if not current_app.config.get('HDFS_CONFIG'): raise NotFound('Not Found') signature = request.args.get('task-signature') if not signature: raise Forbidden('No signature') project = get_project_data(project_id) timeout = project['info'].get('timeout', ContributionsGuard.STAMP_TTL) payload = signer.loads(signature, max_age=timeout) task_id = payload['task_id'] check_allowed(current_user.id, task_id, project, request.path) client = HDFSKerberos(**current_app.config['HDFS_CONFIG'][cluster]) try: content = client.get('/{}'.format(path)) project_encryption = project['info'].get('ext_config', {}).get('encryption', {}) if project_encryption and all(project_encryption.values()): secret = get_secret_from_vault(project_encryption) cipher = AESWithGCM(secret) content = cipher.decrypt(content) except Exception: current_app.logger.exception('Project id {} get task file {}'.format( project_id, path)) raise InternalServerError('An Error Occurred') return Response(content)
def test_get(self, subprocess): reader = MagicMock() reader.read.return_value = 'abc' reader.__enter__.return_value = reader def read(*args, **kwargs): return reader client = HDFSKerberos('testurl', 'testuser') with patch.object(client, 'read', read): assert client.get('test') == 'abc'
def hdfs_file(project_id, cluster, path): if not current_app.config.get('HDFS_CONFIG'): raise NotFound('Not Found') signature = request.args.get('task-signature') if not signature: raise Forbidden('No signature') size_signature = len(signature) if size_signature > TASK_SIGNATURE_MAX_SIZE: current_app.logger.exception( 'Project id {}, cluster {} path {} invalid task signature. Signature length {} exceeds max allowed length {}.' \ .format(project_id, cluster, path, size_signature, TASK_SIGNATURE_MAX_SIZE)) raise Forbidden('Invalid signature') project = get_project_data(project_id) timeout = project['info'].get('timeout', ContributionsGuard.STAMP_TTL) payload = signer.loads(signature, max_age=timeout) task_id = payload['task_id'] try: check_allowed( current_user.id, task_id, project, is_valid_hdfs_url(request.path, request.args.to_dict(flat=False))) except Exception: current_app.logger.exception( 'Project id %s not allowed to get file %s %s', project_id, path, str(request.args)) raise current_app.logger.info( "Project id %s, task id %s. Accessing hdfs cluster %s, path %s", project_id, task_id, cluster, path) client = HDFSKerberos(**current_app.config['HDFS_CONFIG'][cluster]) offset = request.args.get('offset') length = request.args.get('length') try: offset = int(offset) if offset else None length = int(length) if length else None content = client.get('/{}'.format(path), offset=offset, length=length) project_encryption = get_project_encryption(project) if project_encryption and all(project_encryption.values()): secret = get_secret_from_vault(project_encryption) cipher = AESWithGCM(secret) content = cipher.decrypt(content) except Exception: current_app.logger.exception( "Project id %s, task id %s, cluster %s, get task file %s, %s", project_id, task_id, cluster, path, str(request.args)) raise InternalServerError('An Error Occurred') return Response(content)
def test_put(self, subprocess): client = HDFSKerberos('testurl', 'testuser') writer = MagicMock() with patch.object(client, 'write'): client.put('test', 'hello') client.write.assert_called()
def test_kinit_error(self, subprocess): client = HDFSKerberos('testurl', 'testuser') subprocess.side_effect = [CalledProcessError('', ''), CalledProcessError('', '')] assert_raises(CalledProcessError, client.get_ticket)
def test_init_needed(self, subprocess): client = HDFSKerberos('testurl', 'testuser') subprocess.side_effect = [CalledProcessError('', ''), 0] client.get_ticket() assert subprocess.call_count == 2
def test_init_not_needed(self, subprocess): client = HDFSKerberos('testurl', 'testuser') client.get_ticket() subprocess.return_value = 0 assert subprocess.call_count == 1