def test_awsconfig():
"""Test AWSConfig defaults"""
REGION = 'test-region'
cfg = AWSConfig(region=REGION)
assert cfg.region == REGION
assert not cfg.vpc
assert not cfg.subnet
assert not cfg.security_group
assert not cfg.key_pair
assert not cfg.job_role
assert not cfg.instance_role
assert not cfg.batch_service_role
assert not cfg.spot_fleet_role
errors = []
cfg.validate(errors, ElbCommand.SUBMIT)
assert not errors
def test_clusterconfig_from_configparser():
"""Test ClusterConfig initialized from a ConfigParser object"""
RESULTS = 's3://test-bucket'
NAME = 'test-name'
MACHINE_TYPE = 'test-machine-type'
PD_SIZE = 'test-pd-size'
NUM_CPUS = 123
NUM_NODES = 5000
MIN_NODES = 12
MAX_NODES = 999
USE_PREEMPTIBLE = 'Yes'
DISK_TYPE = 'test-disk-type'
IOPS = 987
BID_PERC = 45
LABELS = 'test-labels'
USE_LOCAL_SSD = 'yes'
ENABLE_STACKDRIVER = 'true'
confpars = configparser.ConfigParser()
confpars[CFG_CLUSTER] = {
CFG_CLUSTER_NAME: NAME,
CFG_CLUSTER_MACHINE_TYPE: MACHINE_TYPE,
CFG_CLUSTER_PD_SIZE: PD_SIZE,
CFG_CLUSTER_NUM_CPUS: str(NUM_CPUS),
CFG_CLUSTER_NUM_NODES: str(NUM_NODES),
CFG_CLUSTER_MIN_NODES: str(MIN_NODES),
CFG_CLUSTER_MAX_NODES: str(MAX_NODES),
CFG_CLUSTER_USE_PREEMPTIBLE: USE_PREEMPTIBLE,
CFG_CLUSTER_DISK_TYPE: DISK_TYPE,
CFG_CLUSTER_PROVISIONED_IOPS: IOPS,
CFG_CLUSTER_BID_PERCENTAGE: BID_PERC,
CFG_CLUSTER_LABELS: LABELS,
CFG_CLUSTER_EXP_USE_LOCAL_SSD: USE_LOCAL_SSD,
CFG_CLUSTER_ENABLE_STACKDRIVER: ENABLE_STACKDRIVER
}
confpars[CFG_BLAST] = {CFG_BLAST_RESULTS: RESULTS}
cfg = ClusterConfig.create_from_cfg(
confpars, cloud_provider=AWSConfig(region='test-region'))
assert cfg.name == NAME
assert cfg.machine_type == MACHINE_TYPE
assert cfg.pd_size == PD_SIZE
assert cfg.num_cpus == NUM_CPUS
assert cfg.num_nodes == NUM_NODES
assert cfg.min_nodes == MIN_NODES
assert cfg.max_nodes == MAX_NODES
assert cfg.use_preemptible == True
assert cfg.disk_type == DISK_TYPE
assert cfg.iops == IOPS
assert cfg.bid_percentage == BID_PERC
assert cfg.labels == LABELS
assert cfg.use_local_ssd == True
assert cfg.enable_stackdriver == True
errors = []
# caused by use_local_ssd == True
with pytest.raises(NotImplementedError):
cfg.validate(errors, ElbCommand.SUBMIT)
assert not errors
def test_awsconfig_from_configparser_missing():
"""Test missing required parameters are reported when initializing
GCPConfig from a ConfigParser object"""
REQUIRED_PARAMS = [CFG_CP_AWS_REGION]
with pytest.raises(ValueError) as err:
cfg = AWSConfig.create_from_cfg(configparser.ConfigParser())
for param in REQUIRED_PARAMS:
assert 'Missing ' + param in str(err.value)
def test_clusterconfig_from_configparser_missing():
"""Test ClusterConfig initialization from a ConfigParser object with
missing required parameters"""
REQUIRED_PARAMS = [CFG_BLAST_RESULTS]
with pytest.raises(ValueError) as err:
cfg = ClusterConfig.create_from_cfg(
configparser.ConfigParser(),
cloud_provider=AWSConfig(region='test-region'))
for param in REQUIRED_PARAMS:
assert 'Missing ' + param in str(err.value)
def test_blastconfig_from_configparser_missing():
"""Test BlastConfig initialization from a ConfigParser object with missing
required parameters"""
REQUIRED_PARAMS = [CFG_BLAST_PROGRAM, CFG_BLAST_DB, CFG_BLAST_QUERY]
with pytest.raises(ValueError) as err:
cfg = BlastConfig.create_from_cfg(
configparser.ConfigParser(),
cloud_provider=AWSConfig(region='test-region'),
machine_type='test-machine-type')
for param in REQUIRED_PARAMS:
assert 'Missing ' + param in str(err.value)
def test_blastconfig_from_configparser():
"""Test BlastConfig initialized from a ConfigParser object"""
PROGRAM = 'blastn'
DB = 'test-db'
QUERIES = 'test-queries'
DB_SOURCE = 'GCP'
BATCH_LEN = 5000
OPTIONS = f'test options -outfmt {ELB_DFLT_OUTFMT}'
MEM_REQUEST = '1.3G'
MEM_LIMIT = '21.9G'
DB_MEM_MARGIN = 91.6
confpars = configparser.ConfigParser()
confpars[CFG_BLAST] = {
CFG_BLAST_PROGRAM: PROGRAM,
CFG_BLAST_DB: DB,
CFG_BLAST_QUERY: QUERIES,
CFG_BLAST_DB_SRC: DB_SOURCE,
CFG_BLAST_BATCH_LEN: str(BATCH_LEN),
CFG_BLAST_OPTIONS: OPTIONS,
CFG_BLAST_MEM_REQUEST: MEM_REQUEST,
CFG_BLAST_MEM_LIMIT: MEM_LIMIT,
CFG_BLAST_DB_MEM_MARGIN: str(DB_MEM_MARGIN)
}
cfg = BlastConfig.create_from_cfg(
confpars,
cloud_provider=AWSConfig(region='test-region'),
machine_type='test-machine-type')
assert cfg.program == PROGRAM
assert cfg.db == DB
assert cfg.queries_arg == QUERIES
assert cfg.db_source == DBSource[DB_SOURCE]
assert cfg.batch_len == BATCH_LEN
assert not cfg.queries
assert cfg.options == OPTIONS
assert cfg.mem_limit == MEM_LIMIT
assert cfg.mem_request == MEM_REQUEST
# taxid list is later parsed from BLAST options
assert not cfg.taxidlist
assert cfg.db_mem_margin == DB_MEM_MARGIN
errors = []
cfg.validate(errors, ElbCommand.SUBMIT)
assert not errors
def test_blastconfig_from_configparser_errors():
"""Test that incorrect parameter values in ConfigParser are properly
reported"""
PROGRAM = 'some-program'
DB_SOURCE = 'some-db-source'
BATCH_LEN = -5
MEM_LIMIT = '5'
MEM_REQUEST = -5
DB_MEM_MARGIN = 'margin'
confpars = configparser.ConfigParser()
confpars[CFG_BLAST] = {
CFG_BLAST_PROGRAM: PROGRAM,
CFG_BLAST_DB: 'some-db',
CFG_BLAST_QUERY: 'some-query',
CFG_BLAST_DB_SRC: DB_SOURCE,
CFG_BLAST_BATCH_LEN: str(BATCH_LEN),
CFG_BLAST_MEM_LIMIT: str(MEM_LIMIT),
CFG_BLAST_MEM_REQUEST: str(MEM_REQUEST),
CFG_BLAST_DB_MEM_MARGIN: str(DB_MEM_MARGIN)
}
with pytest.raises(ValueError) as err:
cfg = BlastConfig.create_from_cfg(
confpars,
cloud_provider=AWSConfig(region='test-region'),
machine_type='some-machine-type')
# test that each invalid parameter value is reported
errors = str(err.value).split('\n')
for key in [
CFG_BLAST_PROGRAM, CFG_BLAST_DB_SRC, CFG_BLAST_BATCH_LEN,
CFG_BLAST_MEM_LIMIT, CFG_BLAST_MEM_REQUEST, CFG_BLAST_DB_MEM_MARGIN
]:
assert [
message for message in errors
if key in message and 'invalid value' in message
and confpars[CFG_BLAST][key] in message
]
def test_clusterconfig_aws():
"""Test ClusterConfig defaults for GCP"""
RESULTS = CloudURI('s3://test-results')
aws_cfg = AWSConfig(region='test-region')
cfg = ClusterConfig(cloud_provider=aws_cfg, results=RESULTS)
assert cfg.name.startswith('elasticblast')
assert cfg.results == RESULTS
assert cfg.machine_type == ELB_DFLT_AWS_MACHINE_TYPE
assert cfg.pd_size == ELB_DFLT_AWS_PD_SIZE
assert cfg.num_cpus == get_instance_props(aws_cfg, cfg.machine_type).ncpus
assert cfg.num_nodes == ELB_DFLT_NUM_NODES
assert not cfg.min_nodes
assert not cfg.max_nodes
assert not cfg.use_preemptible
assert cfg.disk_type == ELB_DFLT_AWS_DISK_TYPE
assert not cfg.iops
assert cfg.bid_percentage == int(ELB_DFLT_AWS_SPOT_BID_PERCENTAGE)
assert not cfg.labels
assert not cfg.use_local_ssd
assert not cfg.enable_stackdriver
errors = []
cfg.validate(errors, ElbCommand.SUBMIT)
assert not errors
def test_awsconfig_from_configparser():
"""Test AWSConfig initialized from a ConfigParser object"""
REGION = 'test-region'
VPC = 'test-vpc'
SUBNET = 'test-subnet'
SECURITY_GROUP = 'test-security-group'
KEY_PAIR = 'test-key-pair'
JOB_ROLE = 'test-job-role'
INSTANCE_ROLE = 'test-instance-role'
BATCH_SERV_ROLE = 'test-batch-service-role'
SPOT_FLEET_ROLE = 'test-spot-fleet-role'
confpars = configparser.ConfigParser()
confpars[CFG_CLOUD_PROVIDER] = {
CFG_CP_AWS_REGION: REGION,
CFG_CP_AWS_VPC: VPC,
CFG_CP_AWS_SUBNET: SUBNET,
CFG_CP_AWS_SECURITY_GROUP: SECURITY_GROUP,
CFG_CP_AWS_KEY_PAIR: KEY_PAIR,
CFG_CP_AWS_JOB_ROLE: JOB_ROLE,
CFG_CP_AWS_INSTANCE_ROLE: INSTANCE_ROLE,
CFG_CP_AWS_BATCH_SERVICE_ROLE: BATCH_SERV_ROLE,
CFG_CP_AWS_SPOT_FLEET_ROLE: SPOT_FLEET_ROLE
}
cfg = AWSConfig.create_from_cfg(confpars)
assert cfg.region == REGION
assert cfg.vpc == VPC
assert cfg.subnet == SUBNET
assert cfg.security_group == SECURITY_GROUP
assert cfg.key_pair == KEY_PAIR
assert cfg.job_role == JOB_ROLE
assert cfg.instance_role == INSTANCE_ROLE
assert cfg.batch_service_role == BATCH_SERV_ROLE
assert cfg.spot_fleet_role == SPOT_FLEET_ROLE
errors = []
cfg.validate(errors, ElbCommand.SUBMIT)
assert not errors