class TestAes(object):
def setUp(self):
iv_length = 12
tag_length = 16
secret = bytearray('very secret', 'ascii')
self.aes = AESWithGCM(secret, iv_length, tag_length)
def test_aes(self):
text = 'testing simple encrytion'
encrypted = self.aes.encrypt(text)
assert encrypted != text
decrypted = self.aes.decrypt(encrypted)
assert decrypted == text
def test_aes_2(self):
original = 'this is a test string I plan to encrypt'
encrypted = 'DMj4/yC2pgzgAg76TApmk7zVZlaG0B47KASCnS/TqH6fQpA9UaHjmGLHqCfvGVVQcSivX76Oy349QivZjOJ2yfXZRb0='
secret = bytearray('this is my super secret key', 'ascii')
aes = AESWithGCM(secret)
assert aes.decrypt(encrypted) == original
def test_aes_unicode(self):
text = u'∀ z ∈ ℂ, ζ(z) = 0 ⇒ ((z ∈ -2ℕ) ∨ (Re(z) = -½))'
encrypted = self.aes.encrypt(text.encode('utf-8'))
decrypted = self.aes.decrypt(encrypted).decode('utf-8')
assert text == decrypted
def hdfs_file(project_id, cluster, path):
if not current_app.config.get('HDFS_CONFIG'):
raise NotFound('Not Found')
signature = request.args.get('task-signature')
if not signature:
raise Forbidden('No signature')
project = get_project_data(project_id)
timeout = project['info'].get('timeout', ContributionsGuard.STAMP_TTL)
payload = signer.loads(signature, max_age=timeout)
task_id = payload['task_id']
check_allowed(current_user.id, task_id, project, request.path)
client = HDFSKerberos(**current_app.config['HDFS_CONFIG'][cluster])
try:
content = client.get('/{}'.format(path))
project_encryption = project['info'].get('ext_config',
{}).get('encryption', {})
if project_encryption and all(project_encryption.values()):
secret = get_secret_from_vault(project_encryption)
cipher = AESWithGCM(secret)
content = cipher.decrypt(content)
except Exception:
current_app.logger.exception('Project id {} get task file {}'.format(
project_id, path))
raise InternalServerError('An Error Occurred')
return Response(content)
def get_content_and_key_from_s3(s3_bucket, path, conn_name=DEFAULT_CONN,
decrypt=False, secret=None):
_, key = get_s3_bucket_key(s3_bucket, path, conn_name)
content = key.get_contents_as_string()
if decrypt:
if not secret:
secret = app.config.get('FILE_ENCRYPTION_KEY')
cipher = AESWithGCM(secret)
content = cipher.decrypt(content)
return content, key
def get_file_from_s3(s3_bucket, path, conn_name=DEFAULT_CONN, decrypt=False):
temp_file = NamedTemporaryFile()
_, key = get_s3_bucket_key(s3_bucket, path, conn_name)
content = key.get_contents_as_string()
if decrypt:
secret = app.config.get('FILE_ENCRYPTION_KEY')
cipher = AESWithGCM(secret)
content = cipher.decrypt(content)
temp_file.write(content)
temp_file.seek(0)
return temp_file
def hdfs_file(project_id, cluster, path):
if not current_app.config.get('HDFS_CONFIG'):
raise NotFound('Not Found')
signature = request.args.get('task-signature')
if not signature:
raise Forbidden('No signature')
size_signature = len(signature)
if size_signature > TASK_SIGNATURE_MAX_SIZE:
current_app.logger.exception(
'Project id {}, cluster {} path {} invalid task signature. Signature length {} exceeds max allowed length {}.' \
.format(project_id, cluster, path, size_signature, TASK_SIGNATURE_MAX_SIZE))
raise Forbidden('Invalid signature')
project = get_project_data(project_id)
timeout = project['info'].get('timeout', ContributionsGuard.STAMP_TTL)
payload = signer.loads(signature, max_age=timeout)
task_id = payload['task_id']
try:
check_allowed(
current_user.id, task_id, project,
is_valid_hdfs_url(request.path, request.args.to_dict(flat=False)))
except Exception:
current_app.logger.exception(
'Project id %s not allowed to get file %s %s', project_id, path,
str(request.args))
raise
current_app.logger.info(
"Project id %s, task id %s. Accessing hdfs cluster %s, path %s",
project_id, task_id, cluster, path)
client = HDFSKerberos(**current_app.config['HDFS_CONFIG'][cluster])
offset = request.args.get('offset')
length = request.args.get('length')
try:
offset = int(offset) if offset else None
length = int(length) if length else None
content = client.get('/{}'.format(path), offset=offset, length=length)
project_encryption = get_project_encryption(project)
if project_encryption and all(project_encryption.values()):
secret = get_secret_from_vault(project_encryption)
cipher = AESWithGCM(secret)
content = cipher.decrypt(content)
except Exception:
current_app.logger.exception(
"Project id %s, task id %s, cluster %s, get task file %s, %s",
project_id, task_id, cluster, path, str(request.args))
raise InternalServerError('An Error Occurred')
return Response(content)
def encrypted_task_payload(project_id, task_id):
"""Proxy to decrypt encrypted task payload"""
current_app.logger.info(
'Project id {}, task id {}, decrypt task payload.'.format(
project_id, task_id))
signature = request.args.get('task-signature')
if not signature:
current_app.logger.exception(
'Project id {}, task id {} has no signature.'.format(
project_id, task_id))
raise Forbidden('No signature')
size_signature = len(signature)
if size_signature > TASK_SIGNATURE_MAX_SIZE:
current_app.logger.exception(
'Project id {}, task id {} invalid task signature. Signature length {} exceeds max allowed length {}.' \
.format(project_id, task_id, size_signature, TASK_SIGNATURE_MAX_SIZE))
raise Forbidden('Invalid signature')
project = get_project_data(project_id)
if not project:
current_app.logger.exception('Invalid project id {}.'.format(
project_id, task_id))
raise BadRequest('Invalid Project')
timeout = project['info'].get('timeout', ContributionsGuard.STAMP_TTL)
payload = signer.loads(signature, max_age=timeout)
task_id = payload.get('task_id', 0)
validate_task(project, task_id, current_user.id)
## decrypt encrypted task data under private_json__encrypted_payload
try:
secret = get_encryption_key(project)
task = task_repo.get_task(task_id)
content = task.info.get('private_json__encrypted_payload')
if content:
cipher = AESWithGCM(secret)
content = cipher.decrypt(content)
else:
content = ''
except Exception as e:
current_app.logger.exception(
'Project id {} task {} decrypt encrypted data {}'.format(
project_id, task_id, e))
raise InternalServerError('An Error Occurred')
response = Response(content, content_type='application/json')
return response
def test_taskrun_with_encrypted_payload(self, encr_key, upload_from_string,
set_content):
with patch.dict(self.flask_app.config, self.patch_config):
project = ProjectFactory.create()
encryption_key = 'testkey'
encr_key.return_value = encryption_key
aes = AESWithGCM(encryption_key)
content = 'some data'
encrypted_content = aes.encrypt(content)
task = TaskFactory.create(
project=project,
info={'private_json__encrypted_payload': encrypted_content})
self.app.get('/api/project/%s/newtask?api_key=%s' %
(project.id, project.owner.api_key))
taskrun_data = {'another_field': 42}
data = dict(project_id=project.id,
task_id=task.id,
info=taskrun_data)
datajson = json.dumps(data)
url = '/api/taskrun?api_key=%s' % project.owner.api_key
success = self.app.post(url, data=datajson)
assert success.status_code == 200, success.data
set_content.assert_called()
res = json.loads(success.data)
assert len(res['info']) == 2
encrypted_response = res['info'][
'private_json__encrypted_response']
decrypted_content = aes.decrypt(encrypted_response)
assert decrypted_content == json.dumps(
taskrun_data
), "private_json__encrypted_response decrypted data mismatch"
url = res['info']['pyb_answer_url']
args = {
'host': self.host,
'bucket': self.bucket,
'project_id': project.id,
'task_id': task.id,
'user_id': project.owner.id,
'filename': 'pyb_answer.json'
}
expected = 'https://{host}/{bucket}/{project_id}/{task_id}/{user_id}/{filename}'.format(
**args)
assert url == expected, url
def encrypted_file(store, bucket, project_id, path):
"""Proxy encrypted task file in a cloud storage"""
current_app.logger.info('Project id {} decrypt file. {}'.format(
project_id, path))
conn_args = current_app.config.get('S3_TASK_REQUEST', {})
signature = request.args.get('task-signature')
if not signature:
current_app.logger.exception('Project id {} no signature {}'.format(
project_id, path))
raise Forbidden('No signature')
project = get_project_data(project_id)
timeout = project['info'].get('timeout', ContributionsGuard.STAMP_TTL)
payload = signer.loads(signature, max_age=timeout)
task_id = payload['task_id']
check_allowed(current_user.id, task_id, project, request.path)
## download file
try:
key = '/{}/{}'.format(project_id, path)
conn = create_connection(**conn_args)
_bucket = conn.get_bucket(bucket, validate=False)
_key = _bucket.get_key(key, validate=False)
content = _key.get_contents_as_string()
except S3ResponseError as e:
current_app.logger.exception(
'Project id {} get task file {} {}'.format(project_id, path, e))
if e.error_code == 'NoSuchKey':
raise NotFound('File Does Not Exist')
else:
raise InternalServerError('An Error Occurred')
## decyrpt file
secret = current_app.config.get('FILE_ENCRYPTION_KEY')
cipher = AESWithGCM(secret)
decrypted = cipher.decrypt(content)
response = Response(decrypted, content_type=_key.content_type)
response.headers.add('Content-Encoding', _key.content_encoding)
response.headers.add('Content-Disposition', _key.content_disposition)
return response
def test_aes_2(self):
original = 'this is a test string I plan to encrypt'
encrypted = 'DMj4/yC2pgzgAg76TApmk7zVZlaG0B47KASCnS/TqH6fQpA9UaHjmGLHqCfvGVVQcSivX76Oy349QivZjOJ2yfXZRb0='
secret = bytearray('this is my super secret key', 'ascii')
aes = AESWithGCM(secret)
assert aes.decrypt(encrypted) == original
def test_taskrun_with_upload(self, upload_from_string, set_content):
with patch.dict(self.flask_app.config, self.patch_config):
project = ProjectFactory.create()
task = TaskFactory.create(project=project)
self.app.get('/api/project/%s/newtask?api_key=%s' %
(project.id, project.owner.api_key))
data = dict(project_id=project.id,
task_id=task.id,
info={
'test__upload_url': {
'filename': 'hello.txt',
'content': 'abc'
},
'another_field': 42
})
datajson = json.dumps(data)
url = '/api/taskrun?api_key=%s' % project.owner.api_key
success = self.app.post(url, data=datajson)
assert success.status_code == 200, success.data
set_content.assert_called()
res = json.loads(success.data)
assert len(res['info']) == 1
url = res['info']['pyb_answer_url']
args = {
'host': self.host,
'bucket': self.bucket,
'project_id': project.id,
'task_id': task.id,
'user_id': project.owner.id,
'filename': 'pyb_answer.json'
}
expected = 'https://{host}/{bucket}/{project_id}/{task_id}/{user_id}/{filename}'.format(
**args)
assert url == expected, url
aes = AESWithGCM('testkey')
# first call
first_call = set_content.call_args_list[0]
args, kwargs = first_call
encrypted = args[0].read()
content = aes.decrypt(encrypted)
assert encrypted != content
assert content == 'abc'
upload_from_string.assert_called()
args, kwargs = set_content.call_args
content = aes.decrypt(args[0].read())
actual_content = json.loads(content)
args = {
'host': self.host,
'bucket': self.bucket,
'project_id': project.id,
'task_id': task.id,
'user_id': project.owner.id,
'filename': 'hello.txt'
}
expected = 'https://{host}/{bucket}/{project_id}/{task_id}/{user_id}/{filename}'.format(
**args)
assert actual_content['test__upload_url'] == expected
assert actual_content['another_field'] == 42