class TestAes(object): def setUp(self): iv_length = 12 tag_length = 16 secret = bytearray('very secret', 'ascii') self.aes = AESWithGCM(secret, iv_length, tag_length) def test_aes(self): text = 'testing simple encrytion' encrypted = self.aes.encrypt(text) assert encrypted != text decrypted = self.aes.decrypt(encrypted) assert decrypted == text def test_aes_2(self): original = 'this is a test string I plan to encrypt' encrypted = 'DMj4/yC2pgzgAg76TApmk7zVZlaG0B47KASCnS/TqH6fQpA9UaHjmGLHqCfvGVVQcSivX76Oy349QivZjOJ2yfXZRb0=' secret = bytearray('this is my super secret key', 'ascii') aes = AESWithGCM(secret) assert aes.decrypt(encrypted) == original def test_aes_unicode(self): text = u'∀ z ∈ ℂ, ζ(z) = 0 ⇒ ((z ∈ -2ℕ) ∨ (Re(z) = -½))' encrypted = self.aes.encrypt(text.encode('utf-8')) decrypted = self.aes.decrypt(encrypted).decode('utf-8') assert text == decrypted
def hdfs_file(project_id, cluster, path): if not current_app.config.get('HDFS_CONFIG'): raise NotFound('Not Found') signature = request.args.get('task-signature') if not signature: raise Forbidden('No signature') project = get_project_data(project_id) timeout = project['info'].get('timeout', ContributionsGuard.STAMP_TTL) payload = signer.loads(signature, max_age=timeout) task_id = payload['task_id'] check_allowed(current_user.id, task_id, project, request.path) client = HDFSKerberos(**current_app.config['HDFS_CONFIG'][cluster]) try: content = client.get('/{}'.format(path)) project_encryption = project['info'].get('ext_config', {}).get('encryption', {}) if project_encryption and all(project_encryption.values()): secret = get_secret_from_vault(project_encryption) cipher = AESWithGCM(secret) content = cipher.decrypt(content) except Exception: current_app.logger.exception('Project id {} get task file {}'.format( project_id, path)) raise InternalServerError('An Error Occurred') return Response(content)
def get_content_and_key_from_s3(s3_bucket, path, conn_name=DEFAULT_CONN, decrypt=False, secret=None): _, key = get_s3_bucket_key(s3_bucket, path, conn_name) content = key.get_contents_as_string() if decrypt: if not secret: secret = app.config.get('FILE_ENCRYPTION_KEY') cipher = AESWithGCM(secret) content = cipher.decrypt(content) return content, key
def get_file_from_s3(s3_bucket, path, conn_name=DEFAULT_CONN, decrypt=False): temp_file = NamedTemporaryFile() _, key = get_s3_bucket_key(s3_bucket, path, conn_name) content = key.get_contents_as_string() if decrypt: secret = app.config.get('FILE_ENCRYPTION_KEY') cipher = AESWithGCM(secret) content = cipher.decrypt(content) temp_file.write(content) temp_file.seek(0) return temp_file
def hdfs_file(project_id, cluster, path): if not current_app.config.get('HDFS_CONFIG'): raise NotFound('Not Found') signature = request.args.get('task-signature') if not signature: raise Forbidden('No signature') size_signature = len(signature) if size_signature > TASK_SIGNATURE_MAX_SIZE: current_app.logger.exception( 'Project id {}, cluster {} path {} invalid task signature. Signature length {} exceeds max allowed length {}.' \ .format(project_id, cluster, path, size_signature, TASK_SIGNATURE_MAX_SIZE)) raise Forbidden('Invalid signature') project = get_project_data(project_id) timeout = project['info'].get('timeout', ContributionsGuard.STAMP_TTL) payload = signer.loads(signature, max_age=timeout) task_id = payload['task_id'] try: check_allowed( current_user.id, task_id, project, is_valid_hdfs_url(request.path, request.args.to_dict(flat=False))) except Exception: current_app.logger.exception( 'Project id %s not allowed to get file %s %s', project_id, path, str(request.args)) raise current_app.logger.info( "Project id %s, task id %s. Accessing hdfs cluster %s, path %s", project_id, task_id, cluster, path) client = HDFSKerberos(**current_app.config['HDFS_CONFIG'][cluster]) offset = request.args.get('offset') length = request.args.get('length') try: offset = int(offset) if offset else None length = int(length) if length else None content = client.get('/{}'.format(path), offset=offset, length=length) project_encryption = get_project_encryption(project) if project_encryption and all(project_encryption.values()): secret = get_secret_from_vault(project_encryption) cipher = AESWithGCM(secret) content = cipher.decrypt(content) except Exception: current_app.logger.exception( "Project id %s, task id %s, cluster %s, get task file %s, %s", project_id, task_id, cluster, path, str(request.args)) raise InternalServerError('An Error Occurred') return Response(content)
def encrypted_task_payload(project_id, task_id): """Proxy to decrypt encrypted task payload""" current_app.logger.info( 'Project id {}, task id {}, decrypt task payload.'.format( project_id, task_id)) signature = request.args.get('task-signature') if not signature: current_app.logger.exception( 'Project id {}, task id {} has no signature.'.format( project_id, task_id)) raise Forbidden('No signature') size_signature = len(signature) if size_signature > TASK_SIGNATURE_MAX_SIZE: current_app.logger.exception( 'Project id {}, task id {} invalid task signature. Signature length {} exceeds max allowed length {}.' \ .format(project_id, task_id, size_signature, TASK_SIGNATURE_MAX_SIZE)) raise Forbidden('Invalid signature') project = get_project_data(project_id) if not project: current_app.logger.exception('Invalid project id {}.'.format( project_id, task_id)) raise BadRequest('Invalid Project') timeout = project['info'].get('timeout', ContributionsGuard.STAMP_TTL) payload = signer.loads(signature, max_age=timeout) task_id = payload.get('task_id', 0) validate_task(project, task_id, current_user.id) ## decrypt encrypted task data under private_json__encrypted_payload try: secret = get_encryption_key(project) task = task_repo.get_task(task_id) content = task.info.get('private_json__encrypted_payload') if content: cipher = AESWithGCM(secret) content = cipher.decrypt(content) else: content = '' except Exception as e: current_app.logger.exception( 'Project id {} task {} decrypt encrypted data {}'.format( project_id, task_id, e)) raise InternalServerError('An Error Occurred') response = Response(content, content_type='application/json') return response
def test_taskrun_with_encrypted_payload(self, encr_key, upload_from_string, set_content): with patch.dict(self.flask_app.config, self.patch_config): project = ProjectFactory.create() encryption_key = 'testkey' encr_key.return_value = encryption_key aes = AESWithGCM(encryption_key) content = 'some data' encrypted_content = aes.encrypt(content) task = TaskFactory.create( project=project, info={'private_json__encrypted_payload': encrypted_content}) self.app.get('/api/project/%s/newtask?api_key=%s' % (project.id, project.owner.api_key)) taskrun_data = {'another_field': 42} data = dict(project_id=project.id, task_id=task.id, info=taskrun_data) datajson = json.dumps(data) url = '/api/taskrun?api_key=%s' % project.owner.api_key success = self.app.post(url, data=datajson) assert success.status_code == 200, success.data set_content.assert_called() res = json.loads(success.data) assert len(res['info']) == 2 encrypted_response = res['info'][ 'private_json__encrypted_response'] decrypted_content = aes.decrypt(encrypted_response) assert decrypted_content == json.dumps( taskrun_data ), "private_json__encrypted_response decrypted data mismatch" url = res['info']['pyb_answer_url'] args = { 'host': self.host, 'bucket': self.bucket, 'project_id': project.id, 'task_id': task.id, 'user_id': project.owner.id, 'filename': 'pyb_answer.json' } expected = 'https://{host}/{bucket}/{project_id}/{task_id}/{user_id}/{filename}'.format( **args) assert url == expected, url
def encrypted_file(store, bucket, project_id, path): """Proxy encrypted task file in a cloud storage""" current_app.logger.info('Project id {} decrypt file. {}'.format( project_id, path)) conn_args = current_app.config.get('S3_TASK_REQUEST', {}) signature = request.args.get('task-signature') if not signature: current_app.logger.exception('Project id {} no signature {}'.format( project_id, path)) raise Forbidden('No signature') project = get_project_data(project_id) timeout = project['info'].get('timeout', ContributionsGuard.STAMP_TTL) payload = signer.loads(signature, max_age=timeout) task_id = payload['task_id'] check_allowed(current_user.id, task_id, project, request.path) ## download file try: key = '/{}/{}'.format(project_id, path) conn = create_connection(**conn_args) _bucket = conn.get_bucket(bucket, validate=False) _key = _bucket.get_key(key, validate=False) content = _key.get_contents_as_string() except S3ResponseError as e: current_app.logger.exception( 'Project id {} get task file {} {}'.format(project_id, path, e)) if e.error_code == 'NoSuchKey': raise NotFound('File Does Not Exist') else: raise InternalServerError('An Error Occurred') ## decyrpt file secret = current_app.config.get('FILE_ENCRYPTION_KEY') cipher = AESWithGCM(secret) decrypted = cipher.decrypt(content) response = Response(decrypted, content_type=_key.content_type) response.headers.add('Content-Encoding', _key.content_encoding) response.headers.add('Content-Disposition', _key.content_disposition) return response
def test_aes_2(self): original = 'this is a test string I plan to encrypt' encrypted = 'DMj4/yC2pgzgAg76TApmk7zVZlaG0B47KASCnS/TqH6fQpA9UaHjmGLHqCfvGVVQcSivX76Oy349QivZjOJ2yfXZRb0=' secret = bytearray('this is my super secret key', 'ascii') aes = AESWithGCM(secret) assert aes.decrypt(encrypted) == original
def test_taskrun_with_upload(self, upload_from_string, set_content): with patch.dict(self.flask_app.config, self.patch_config): project = ProjectFactory.create() task = TaskFactory.create(project=project) self.app.get('/api/project/%s/newtask?api_key=%s' % (project.id, project.owner.api_key)) data = dict(project_id=project.id, task_id=task.id, info={ 'test__upload_url': { 'filename': 'hello.txt', 'content': 'abc' }, 'another_field': 42 }) datajson = json.dumps(data) url = '/api/taskrun?api_key=%s' % project.owner.api_key success = self.app.post(url, data=datajson) assert success.status_code == 200, success.data set_content.assert_called() res = json.loads(success.data) assert len(res['info']) == 1 url = res['info']['pyb_answer_url'] args = { 'host': self.host, 'bucket': self.bucket, 'project_id': project.id, 'task_id': task.id, 'user_id': project.owner.id, 'filename': 'pyb_answer.json' } expected = 'https://{host}/{bucket}/{project_id}/{task_id}/{user_id}/{filename}'.format( **args) assert url == expected, url aes = AESWithGCM('testkey') # first call first_call = set_content.call_args_list[0] args, kwargs = first_call encrypted = args[0].read() content = aes.decrypt(encrypted) assert encrypted != content assert content == 'abc' upload_from_string.assert_called() args, kwargs = set_content.call_args content = aes.decrypt(args[0].read()) actual_content = json.loads(content) args = { 'host': self.host, 'bucket': self.bucket, 'project_id': project.id, 'task_id': task.id, 'user_id': project.owner.id, 'filename': 'hello.txt' } expected = 'https://{host}/{bucket}/{project_id}/{task_id}/{user_id}/{filename}'.format( **args) assert actual_content['test__upload_url'] == expected assert actual_content['another_field'] == 42