def _process_job_status(self, service, job_doc):
"""
Send job status to DB service.
Include info on prev and next service which DB service
does not know about.
"""
job_doc['current_service'] = self._get_next_service(service)
job_doc['prev_service'] = service
last_service = job_doc.pop('last_service')
notification_email = job_doc.pop('notification_email')
try:
handle_request(
self.database_api_url,
'jobs/',
'put',
job_data=job_doc
)
except Exception as error:
self.log.error('Job status update failed: {}'.format(error))
if notification_email and (last_service == service):
self.send_notification(
job_doc['id'],
notification_email,
job_doc['status'],
job_doc.get('cloud_image_name'),
job_doc['errors']
)
def delete_aliyun_account_for_user(name, user_id):
"""
Delete Aliyun account for user.
"""
data = {
'cloud': 'aliyun',
'account_name': name,
'requesting_user': user_id
}
aliyun_account = get_aliyun_account_for_user(name, user_id)
if aliyun_account:
try:
db.session.delete(aliyun_account)
db.session.commit()
handle_request(
current_app.config['CREDENTIALS_URL'],
'credentials/',
'delete',
job_data=data
)
except Exception:
db.session.rollback()
raise
else:
return 1
else:
return 0
def test_handle_request_failed(mock_requests):
response = MagicMock()
response.status_code = 400
response.reason = 'Not Found'
response.json.return_value = {}
mock_requests.get.return_value = response
with raises(MashException):
handle_request('localhost', '/jobs', 'get')
def update_azure_account_for_user(
account_name,
user_id,
region=None,
credentials=None,
source_container=None,
source_resource_group=None,
source_storage_account=None
):
"""
Update Azure account for user.
"""
account = get_azure_account_by_user(account_name, user_id)
if not account:
return None
if credentials:
data = {
'cloud': 'azure',
'account_name': account_name,
'requesting_user': user_id,
'credentials': credentials
}
try:
handle_request(
current_app.config['CREDENTIALS_URL'],
'credentials/',
'post',
job_data=data
)
except Exception:
raise
if region:
account.region = region
if source_container:
account.source_container = source_container
if source_resource_group:
account.source_resource_group = source_resource_group
if source_storage_account:
account.source_storage_account = source_storage_account
try:
db.session.add(account)
db.session.commit()
except Exception:
db.session.rollback()
raise
return account
def update_aliyun_account_for_user(
account_name,
user_id,
bucket=None,
region=None,
credentials=None,
security_group_id=None,
vswitch_id=None
):
"""
Update an existing Aliyun account.
"""
aliyun_account = get_aliyun_account_for_user(account_name, user_id)
if not aliyun_account:
return None
if credentials:
data = {
'cloud': 'aliyun',
'account_name': account_name,
'requesting_user': user_id,
'credentials': credentials
}
try:
handle_request(
current_app.config['CREDENTIALS_URL'],
'credentials/',
'post',
job_data=data
)
except Exception:
raise
if region:
aliyun_account.region = region
if bucket:
aliyun_account.bucket = bucket
if security_group_id:
aliyun_account.security_group_id = security_group_id
if vswitch_id:
aliyun_account.vswitch_id = vswitch_id
try:
db.session.add(aliyun_account)
db.session.commit()
except Exception:
db.session.rollback()
raise
return aliyun_account
def create_job(data):
"""
Create a new job for user.
"""
if data.get('dry_run'):
return None
job_id = get_new_job_id()
data['job_id'] = job_id
user_id = data['requesting_user']
kwargs = {
'job_id': job_id,
'last_service': data['last_service'],
'utctime': data['utctime'],
'image': data['image'],
'download_url': data['download_url'],
'user_id': user_id,
'state': RUNNING,
'current_service': current_app.config['SERVICE_NAMES'][0]
}
if data['utctime'] != 'now':
kwargs['start_time'] = parser.parse(data['utctime'])
if data.get('cloud_architecture'):
kwargs['cloud_architecture'] = data['cloud_architecture']
if data.get('profile'):
kwargs['profile'] = data['profile']
response = handle_request(current_app.config['DATABASE_API_URL'],
'jobs/',
'post',
job_data=kwargs)
try:
publish('jobcreator', 'job_document', json.dumps(data, sort_keys=True))
except Exception:
try:
handle_request(current_app.config['DATABASE_API_URL'],
'jobs/',
'delete',
job_data={
'job_id': job_id,
'user_id': user_id
})
except Exception:
pass # Attempt to cleanup job in database
raise MashJobException('Failed to initialize job.')
return response.json()
def update_ec2_account_for_user(account_name,
user_id,
additional_regions=None,
credentials=None,
group=None,
region=None,
subnet=None):
"""
Update an existing EC2 account.
"""
ec2_account = get_ec2_account_for_user(account_name, user_id)
if not ec2_account:
return None
if credentials:
data = {
'cloud': 'ec2',
'account_name': account_name,
'requesting_user': user_id,
'credentials': credentials
}
try:
handle_request(current_app.config['CREDENTIALS_URL'],
'credentials/',
'post',
job_data=data)
except Exception:
raise
if group:
ec2_account.group = _get_or_create_ec2_group(group, user_id)
if additional_regions:
for additional_region in additional_regions:
create_new_ec2_region(additional_region['name'],
additional_region['helper_image'],
ec2_account)
if region:
ec2_account.region = region
if subnet:
ec2_account.subnet = subnet
try:
db.session.add(ec2_account)
db.session.commit()
except Exception:
db.session.rollback()
raise
return ec2_account
def delete_user(user_id):
"""
Delete user by id.
If user does not exist return 0.
"""
try:
handle_request(current_app.config['DATABASE_API_URL'],
'users/{user}'.format(user=user_id), 'delete')
except Exception:
return 0
return 1
def update_gce_account_for_user(
account_name,
user_id,
bucket=None,
region=None,
credentials=None,
testing_account=None
):
"""
Update an existing GCE account.
"""
gce_account = get_gce_account_for_user(account_name, user_id)
if not gce_account:
return None
if credentials:
data = {
'cloud': 'gce',
'account_name': account_name,
'requesting_user': user_id,
'credentials': credentials
}
try:
handle_request(
current_app.config['CREDENTIALS_URL'],
'credentials/',
'post',
job_data=data
)
except Exception:
raise
if bucket:
gce_account.bucket = bucket
if region:
gce_account.region = region
if testing_account and gce_account.is_publishing_account:
gce_account.testing_account = testing_account
try:
db.session.add(gce_account)
db.session.commit()
except Exception:
db.session.rollback()
raise
return gce_account
def create_new_aliyun_account(
user_id,
account_name,
bucket_name,
region_name,
credentials,
security_group_id=None,
vswitch_id=None
):
"""
Create a new Aliyun account for user.
"""
data = {
'cloud': 'aliyun',
'account_name': account_name,
'requesting_user': user_id,
'credentials': credentials
}
aliyun_account = AliyunAccount(
name=account_name,
region=region_name,
bucket=bucket_name,
security_group_id=security_group_id,
vswitch_id=vswitch_id,
user_id=user_id
)
if security_group_id:
aliyun_account.security_group_id = security_group_id
if vswitch_id:
aliyun_account.vswitch_id = vswitch_id
try:
handle_request(
current_app.config['CREDENTIALS_URL'],
'credentials/',
'post',
job_data=data
)
db.session.add(aliyun_account)
db.session.commit()
except Exception:
db.session.rollback()
raise
return aliyun_account
def create_new_gce_account(
user_id,
account_name,
bucket,
region_name,
credentials,
testing_account,
is_publishing_account
):
"""
Create a new GCE account for user.
"""
if is_publishing_account and not testing_account:
raise MashDBException(
'Jobs using a GCE publishing account require'
' the use of a test account.'
)
data = {
'cloud': 'gce',
'account_name': account_name,
'requesting_user': user_id,
'credentials': credentials
}
account = GCEAccount(
name=account_name,
bucket=bucket,
region=region_name,
testing_account=testing_account,
is_publishing_account=is_publishing_account,
user_id=user_id
)
try:
handle_request(
current_app.config['CREDENTIALS_URL'],
'credentials/',
'post',
job_data=data
)
db.session.add(account)
db.session.commit()
except Exception:
db.session.rollback()
raise
return account
def add_user(email, password=None):
"""
Add new user to database and set password hash.
If the user or email exists return None.
If password is None, a random password is created, in effect creating
an account that can only be used with oidc authentication.
"""
if password and len(password) < 8:
raise MashException(
'Password too short. Minimum length is 8 characters.')
if not email_in_whitelist(email):
raise MashException(
'Cannot create a user with the provided email. Access denied.')
response = handle_request(current_app.config['DATABASE_API_URL'],
'users/',
'post',
job_data={
'email': email,
'password': password
})
return response.json()
def test_handle_request(mock_requests):
response = MagicMock()
response.status_code = 200
mock_requests.get.return_value = response
result = handle_request('localhost', '/jobs', 'get')
assert result == response
def request_credentials(self, accounts, cloud=None):
"""
Request credentials from credential service.
Only send request if credentials not already populated.
"""
if self.credentials:
return
data = {
'cloud': cloud or self.cloud,
'cloud_accounts': accounts,
'requesting_user': self.requesting_user
}
try:
response = handle_request(
self.config.get_credentials_url(),
'credentials/',
'get',
job_data=data
)
self.credentials = response.json()
except Exception:
raise MashJobException(
'Credentials request failed for accounts: {accounts}'.format(
accounts=', '.join(accounts)
)
)
def get_user_tokens(user_id):
"""
Returns all of the tokens for given user.
"""
response = handle_request(current_app.config['DATABASE_API_URL'],
'tokens/list/{user}'.format(user=user_id), 'get')
return response.json()
def revoke_tokens(user_id):
"""
Revokes (deletes) all tokens for given user.
"""
response = handle_request(current_app.config['DATABASE_API_URL'],
'tokens/list/{user}'.format(user=user_id),
'delete')
return response.json().get('rows_deleted', 0)
def add_token_to_database(encoded_token, user_id):
"""
Add a new token to the database.
"""
decoded_token = decode_token(encoded_token)
jti = decoded_token['jti']
token_type = decoded_token['type']
expires = decoded_token['exp']
handle_request(current_app.config['DATABASE_API_URL'],
'tokens/',
'post',
job_data={
'jti': jti,
'token_type': token_type,
'user_id': user_id,
'expires': expires
})
def get_user_by_id(user_id):
"""
Retrieve user from database if a match exists.
If user does not exist return None.
"""
response = handle_request(current_app.config['DATABASE_API_URL'],
'users/{user}'.format(user=user_id), 'get')
return response.json()
def get_gce_accounts(user_id):
"""
Retrieve all GCE accounts for user.
"""
response = handle_request(current_app.config['DATABASE_API_URL'],
'gce_accounts/list/{user}'.format(user=user_id),
'get')
return response.json()
def create_new_azure_account(
user_id,
account_name,
region_name,
credentials,
source_container,
source_resource_group,
source_storage_account
):
"""
Create a new Azure account for user.
"""
data = {
'cloud': 'azure',
'account_name': account_name,
'requesting_user': user_id,
'credentials': credentials
}
account = AzureAccount(
name=account_name,
region=region_name,
source_container=source_container,
source_resource_group=source_resource_group,
source_storage_account=source_storage_account,
user_id=user_id
)
try:
handle_request(
current_app.config['CREDENTIALS_URL'],
'credentials/',
'post',
job_data=data
)
db.session.add(account)
db.session.commit()
except Exception:
db.session.rollback()
raise
return account
def create_new_ec2_account(user_id, account_name, partition, region_name,
credentials, subnet, group_name,
additional_regions):
"""
Create a new EC2 account for user.
Create a new group and additional regions if necessary.
"""
data = {
'cloud': 'ec2',
'account_name': account_name,
'requesting_user': user_id,
'credentials': credentials
}
account = EC2Account(name=account_name,
partition=partition,
region=region_name,
subnet=subnet,
user_id=user_id)
if group_name:
group = _get_or_create_ec2_group(group_name, user_id)
account.group = group
if additional_regions:
for additional_region in additional_regions:
create_new_ec2_region(additional_region['name'],
additional_region['helper_image'], account)
try:
handle_request(current_app.config['CREDENTIALS_URL'],
'credentials/',
'post',
job_data=data)
db.session.add(account)
db.session.commit()
except Exception:
db.session.rollback()
raise
return account
def change_user_password(email, current_password, new_password):
"""
Change password for user if user exists and existing password matches.
And reset the password to clean so the user can login again.
"""
if len(new_password) < 8:
raise MashException(
'Password too short. Minimum length is 8 characters.')
handle_request(current_app.config['DATABASE_API_URL'],
'users/password/change/{email}'.format(email=email),
'post',
job_data={
'current_password': current_password,
'new_password': new_password
})
current_app.notification_class.send_notification(
password_change_msg_template, '[MASH] Password Changed', email)
def revoke_token_by_jti(jti, user_id):
"""
Revoke token by jti identifier.
"""
response = handle_request(current_app.config['DATABASE_API_URL'],
'tokens/',
'delete',
job_data={
'jti': jti,
'user_id': user_id
})
return response.json()['rows_deleted']
def get_user_by_email(email, create=False):
"""
Retrieve user from database if a match exists.
If the user does not exist and create is True, the user
is created on the fly. Otherwise None is returned.
"""
response = handle_request(current_app.config['DATABASE_API_URL'],
'users/get_user/{email}'.format(email=email),
'get',
job_data={'create': create})
return response.json()
def create_gce_account(user_id, data):
"""
Create a new GCE account for user.
"""
data['user_id'] = user_id
response = handle_request(current_app.config['DATABASE_API_URL'],
'gce_accounts/',
'post',
job_data=data)
return response.json()
def get_jobs(user_id, page=None, per_page=None):
"""
Retrieve all jobs for user.
"""
response = handle_request(current_app.config['DATABASE_API_URL'],
'jobs/list/{user}'.format(user=user_id),
'get',
job_data={
'page': page,
'per_page': per_page
})
return response.json()
def get_token_by_jti(token_jti, user_id):
"""
Get token by jti identifier.
"""
response = handle_request(current_app.config['DATABASE_API_URL'],
'tokens/',
'get',
job_data={
'jti': token_jti,
'user_id': user_id
})
return response.json()
def get_accounts_in_ec2_group(group_name, user_id):
"""
Get an EC2 group for user.
"""
response = handle_request(current_app.config['DATABASE_API_URL'],
'ec2_accounts/group_accounts',
'get',
job_data={
'user_id': user_id,
'group_name': group_name
})
return response.json()
def update_gce_account(account_name, user_id, data):
"""
Update GCE account for user.
"""
data['account_name'] = account_name
data['user_id'] = user_id
response = handle_request(current_app.config['DATABASE_API_URL'],
'gce_accounts/',
'put',
job_data=data)
return response.json()
def delete_gce_account(name, user_id):
"""
Delete GCE account for user.
"""
response = handle_request(current_app.config['DATABASE_API_URL'],
'gce_accounts/',
'delete',
job_data={
'name': name,
'user_id': user_id
})
return response.json()['rows_deleted']
