def main():
"""
mash - obs service application entry point
"""
try:
logging.basicConfig()
log = logging.getLogger('MashService')
log.setLevel(logging.DEBUG)
# run service, enter main loop
OBSImageBuildResultService(
service_exchange='obs',
config=BaseConfig()
)
except MashException as e:
# known exception
log.error('{0}: {1}'.format(type(e).__name__, format(e)))
traceback.print_exc()
sys.exit(1)
except KeyboardInterrupt:
sys.exit(0)
except SystemExit:
# user exception, program aborted by user
sys.exit(0)
except Exception as e:
# exception we did no expect, show python backtrace
log.error('Unexpected error: {0}'.format(e))
traceback.print_exc()
sys.exit(1)
def setup(self):
self.config = BaseConfig(config_file='test/data/mash_config.yaml')
self.credentials = {
'test': {
'signing_key': 'test key',
'fingerprint': 'fake fingerprint'
}
}
job_doc = {
'id': '1',
'last_service': 'create',
'cloud': 'oci',
'requesting_user': '******',
'utctime': 'now',
'region': 'us-phoenix-1',
'account': 'test',
'bucket': 'images',
'cloud_image_name': 'sles-12-sp4-v20180909',
'image_description': 'description 20180909',
'oci_user_id': 'ocid1.user.oc1..',
'tenancy': 'ocid1.tenancy.oc1..',
'compartment_id': 'ocid1.compartment.oc1..',
'operating_system': 'SLES',
'operating_system_version': '12SP2'
}
self.job = OCICreateJob(job_doc, self.config)
self.job._log_callback = Mock()
self.job.credentials = self.credentials
def setup(self):
self.config = BaseConfig(config_file='test/data/mash_config.yaml')
self.credentials = {
'test': {
'access_key': '123456789',
'access_secret': '987654321'
}
}
job_doc = {
'id': '1',
'last_service': 'create',
'cloud': 'aliyun',
'requesting_user': '******',
'utctime': 'now',
'platform': 'SUSE',
'cloud_architecture': 'x86_64',
'region': 'cn-beijing',
'account': 'test',
'bucket': 'images',
'cloud_image_name': 'sles-15-sp2-v20210316',
'image_description': 'great image description',
'disk_size': 20
}
self.job = AliyunCreateJob(job_doc, self.config)
self.job._log_callback = Mock()
self.job.credentials = self.credentials
def main():
"""
mash - publish service application entry point
"""
try:
logging.basicConfig()
log = logging.getLogger('MashService')
log.setLevel(logging.DEBUG)
service_name = 'publish'
# Create job factory
job_factory = BaseJobFactory(service_name=service_name,
job_types={
'azure': AzurePublishJob,
'ec2': EC2PublishJob,
'gce': NoOpJob,
'oci': NoOpJob,
'aliyun': AliyunPublishJob
})
config = BaseConfig()
# run service, enter main loop
ListenerService(service_exchange=service_name,
config=config,
custom_args={
'job_factory':
job_factory,
'thread_pool_count':
config.get_publish_thread_pool_count()
})
except MashException as e:
# known exception
log.error('{0}: {1}'.format(type(e).__name__, format(e)))
traceback.print_exc()
sys.exit(1)
except KeyboardInterrupt:
sys.exit(0)
except SystemExit:
# user exception, program aborted by user
sys.exit(0)
except Exception as e:
# exception we did no expect, show python backtrace
log.error('Unexpected error: {0}'.format(e))
traceback.print_exc()
sys.exit(1)
class Config(object):
def __init__(self, config_file=None, test=False):
self.config = BaseConfig(config_file)
self.service_exchange = 'database'
self.TESTING = test
@property
def AMQP_HOST(self):
return self.config.get_amqp_host()
@property
def AMQP_USER(self):
return self.config.get_amqp_user()
@property
def AMQP_PASS(self):
return self.config.get_amqp_pass()
@property
def LOG_FILE(self):
return self.config.get_log_file(self.service_exchange)
@property
def SQLALCHEMY_TRACK_MODIFICATIONS(self):
return False
@property
def SQLALCHEMY_DATABASE_URI(self):
return self.config.get_database_uri()
@property
def CREDENTIALS_URL(self):
return self.config.get_credentials_url()
def main():
"""
mash - replicate service application entry point
"""
try:
logging.basicConfig()
log = logging.getLogger('MashService')
log.setLevel(logging.INFO)
service_name = 'replicate'
# Create job factory
job_factory = BaseJobFactory(
service_name=service_name,
job_types={
'azure': NoOpJob,
'ec2': EC2ReplicateJob,
'gce': NoOpJob,
'oci': NoOpJob,
'aliyun': AliyunReplicateJob
}
)
# run service, enter main loop
ListenerService(
service_exchange=service_name,
config=BaseConfig(),
custom_args={
'job_factory': job_factory
}
)
except MashException as e:
# known exception
log.error('%s: %s', type(e).__name__, format(e))
traceback.print_exc()
sys.exit(1)
except KeyboardInterrupt:
sys.exit(0)
except SystemExit as e:
# user exception, program aborted by user
sys.exit(e)
except Exception as e:
# exception we did no expect, show python backtrace
log.error('Unexpected error: {0}'.format(e))
traceback.print_exc()
sys.exit(1)
def setup(self):
self.config = BaseConfig(config_file='test/data/mash_config.yaml')
self.credentials = {
'test': {
'type':
'type',
'project_id':
'projectid',
'private_key_id':
'keyid',
'private_key':
'key',
'client_email':
'*****@*****.**',
'client_id':
'a',
'auth_uri':
'https://accounts.google.com/o/oauth2/auth',
'token_uri':
'https://accounts.google.com/o/oauth2/token',
'auth_provider_x509_cert_url':
'https://www.googleapis.com/oauth2/v1/certs',
'client_x509_cert_url':
'https://www.googleapis.com/robot/v1/metadata/x509/'
}
}
job_doc = {
'id': '1',
'last_service': 'create',
'cloud': 'gce',
'requesting_user': '******',
'utctime': 'now',
'family': 'sles-12',
'guest_os_features': ['UEFI_COMPATIBLE'],
'region': 'us-west1-a',
'account': 'test',
'bucket': 'images',
'cloud_image_name': 'sles-12-sp4-v20180909',
'image_description': 'description 20180909'
}
self.job = GCECreateJob(job_doc, self.config)
self.job._log_callback = Mock()
self.job.credentials = self.credentials
def setup(self):
self.config = BaseConfig(
config_file='test/data/mash_config.yaml'
)
self.credentials = {
'test': {
'access_key_id': 'access-key',
'secret_access_key': 'secret-access-key'
}
}
job_doc = {
'cloud_architecture': 'x86_64',
'id': '1',
'last_service': 'upload',
'cloud': 'ec2',
'requesting_user': '******',
'utctime': 'now',
'target_regions': {
'us-east-1': {
'account': 'test',
'helper_image': 'ami-bc5b48d0',
'billing_codes': None,
'use_root_swap': False,
'subnet': 'subnet-123456789'
}
},
'cloud_image_name': 'name',
'image_description': 'description',
'raw_image_upload_type': 's3bucket',
'raw_image_upload_location': 'my-bucket/some-prefix/',
'raw_image_upload_account': 'test'
}
self.job = S3BucketUploadJob(job_doc, self.config)
self.job.status_msg = {
'image_file': 'file.raw.gz',
'cloud_image_name': 'name'
}
self.job.credentials = self.credentials
self.job._log_callback = Mock()
def setup(self):
self.credentials = {
'test': {
'clientId': 'a',
'clientSecret': 'b',
'subscriptionId': 'c',
'tenantId': 'd',
'activeDirectoryEndpointUrl':
'https://login.microsoftonline.com',
'resourceManagerEndpointUrl': 'https://management.azure.com/',
'activeDirectoryGraphResourceId': 'https://graph.windows.net/',
'sqlManagementEndpointUrl':
'https://management.core.windows.net:8443/',
'galleryEndpointUrl': 'https://gallery.azure.com/',
'managementEndpointUrl': 'https://management.core.windows.net/'
}
}
job_doc = {
'id': '1',
'last_service': 'create',
'cloud': 'azure',
'requesting_user': '******',
'utctime': 'now',
'account': 'test',
'resource_group': 'group_name',
'container': 'container',
'storage_account': 'storage',
'region': 'region',
'cloud_image_name': 'name',
'boot_firmware': ['bios', 'uefi']
}
self.config = BaseConfig(config_file='test/data/mash_config.yaml')
self.job = AzureCreateJob(job_doc, self.config)
self.job.credentials = self.credentials
self.job._log_callback = Mock()
def setup(self):
self.config = BaseConfig(config_file='test/data/mash_config.yaml')
self.credentials = {
'test': {
'access_key_id': 'access-key',
'secret_access_key': 'secret-access-key'
}
}
job_doc = {
'cloud_architecture': 'aarch64',
'id': '1',
'last_service': 'create',
'cloud': 'ec2',
'requesting_user': '******',
'utctime': 'now',
'target_regions': {
'us-east-1': {
'account': 'test',
'helper_image': 'ami-bc5b48d0',
'billing_codes': None,
'use_root_swap': False,
'subnet': 'subnet-123456789',
'regions': ['us-east-1', 'us-east-2']
}
},
'cloud_image_name': 'name v{date}',
'image_description': 'description',
'use_build_time': True
}
self.job = EC2CreateJob(job_doc, self.config)
self.job._log_callback = Mock()
self.job.status_msg['image_file'] = 'file'
self.job.status_msg['build_time'] = '1601061355'
self.job.status_msg['source_regions'] = {'us-east-1': 'ami_id'}
self.job.credentials = self.credentials
def setup(self):
self.empty_config = BaseConfig('test/data/empty_mash_config.yaml')
self.config = BaseConfig('test/data/mash_config.yaml')
class TestBaseConfig(object):
def setup(self):
self.empty_config = BaseConfig('test/data/empty_mash_config.yaml')
self.config = BaseConfig('test/data/mash_config.yaml')
def test_get_encryption_keys_file(self):
enc_keys_file = self.empty_config.get_encryption_keys_file()
assert enc_keys_file == '/var/lib/mash/encryption_keys'
def test_get_jwt_algorithm(self):
algorithm = self.empty_config.get_jwt_algorithm()
assert algorithm == 'HS256'
def test_get_jwt_secret(self):
secret = self.config.get_jwt_secret()
assert secret == 'abc123'
def test_get_jwt_secret_empty(self):
msg = 'jwt_secret must be in config file.'
with raises(MashConfigException) as error:
self.empty_config.get_jwt_secret()
assert msg == str(error.value)
def test_get_cloud_data(self):
data = self.config.get_cloud_data()
assert data['ec2']['regions']['aws-cn'] == ['cn-north-1']
assert data['ec2']['helper_images']['cn-north-1'] == 'ami-bcc45885'
with raises(MashConfigException) as error:
self.empty_config.get_cloud_data()
assert str(error.value) == \
'cloud data must be provided in config file.'
def test_get_services_names(self):
# Services requiring credentials
expected = [
'upload', 'create', 'test', 'raw_image_upload',
'replicate', 'publish', 'deprecate'
]
services = self.empty_config.get_service_names(
credentials_required=True
)
assert expected == services
# All services
expected = ['obs'] + expected
services = self.empty_config.get_service_names()
assert expected == services
def test_get_ssh_private_key_file(self):
assert self.config.get_ssh_private_key_file() == \
'/var/lib/mash/ssh_key'
with raises(MashConfigException) as error:
self.empty_config.get_ssh_private_key_file()
assert str(error.value) == \
'ssh_private_key_file is required in MASH configuration file.'
def test_get_amqp_host(self):
host = self.empty_config.get_amqp_host()
assert host == 'localhost'
def test_get_amqp_user(self):
user = self.empty_config.get_amqp_user()
assert user == 'guest'
def test_get_amqp_pass(self):
password = self.empty_config.get_amqp_pass()
assert password == 'guest'
def test_get_smtp_host(self):
host = self.empty_config.get_smtp_host()
assert host == 'localhost'
def test_get_smtp_port(self):
port = self.empty_config.get_smtp_port()
assert port == 25
def test_get_smtp_ssl(self):
ssl = self.empty_config.get_smtp_ssl()
assert not ssl
def test_get_smtp_user(self):
user = self.config.get_smtp_user()
assert user == '*****@*****.**'
with raises(MashConfigException) as error:
self.empty_config.get_smtp_user()
msg = 'smtp_user is required in MASH configuration file.'
assert str(error.value) == msg
def test_get_smtp_pass(self):
password = self.config.get_smtp_pass()
assert password == 'super.secret'
password = self.empty_config.get_smtp_pass()
assert password is None
def test_get_notification_subject(self):
subject = self.empty_config.get_notification_subject()
assert subject == '[MASH] Job Status Update'
def test_get_job_dir(self):
assert self.config.get_job_directory('test') == \
'/tmp/jobs/test_jobs/'
assert self.empty_config.get_job_directory('test') == \
'/var/lib/mash/test_jobs/'
def test_get_log_dir(self):
assert self.config.get_log_directory() == '/tmp/log/'
assert self.empty_config.get_log_directory() == '/var/log/mash/'
@patch.object(BaseConfig, 'get_log_directory')
def test_get_job_log_file(self, mock_get_log_dir):
mock_get_log_dir.return_value = '/var/log/mash/'
assert self.empty_config.get_job_log_file('1234') == \
'/var/log/mash/jobs/1234.log'
def test_get_credentials_url(self):
assert self.config.get_credentials_url() == 'http://*****:*****@patch.object(BaseConfig, 'get_auth_methods', lambda x: ['oauth2'])
def test_get_oauth2_client_id(self):
with raises(MashConfigException):
self.config.get_oauth2_client_id()
@patch.object(BaseConfig, 'get_auth_methods', lambda x: ['oauth2'])
def test_get_oauth2_client_secret(self):
with raises(MashConfigException):
self.config.get_oauth2_client_secret()
@patch.object(BaseConfig, 'get_auth_methods', lambda x: ['oauth2'])
def test_get_oauth2_provider_url(self):
with raises(MashConfigException):
self.config.get_oauth2_provider_url()
@patch.object(BaseConfig, 'get_auth_methods', lambda x: ['oauth2'])
def test_get_oauth2_redirect_ports(self):
with raises(MashConfigException):
self.config.get_oauth2_redirect_ports()
def test_get_database_api_url(self):
assert self.config.get_database_api_url() == 'http://localhost:5057/'
assert self.empty_config.get_database_api_url() == \
'http://localhost:5007/'
def __init__(self, config_file=None, test=False):
self.config = BaseConfig(config_file)
self.service_exchange = 'database'
self.TESTING = test