def take_action(self, parsed_args):
self.log.info("Setting up pancancer config files.")
config_path = vars(parsed_args)["config_path"]
force_config = vars(parsed_args)["force_config"]
self.log.debug("force_config:" + str(force_config))
if config_path != None:
# if the user specifies a path to an existing file, they can use that instead of the interactive process.
self.log.debug("path to config file: " + config_path)
process_config.main(config_path)
else:
# if the user did not specify a config, ask them questions and then WRITE/update a config!
pancancer_config_path = os.path.expanduser("~/.pancancer/simple_pancancer_config.json")
config_data = {}
if os.path.isfile(pancancer_config_path):
# Read the pancancer_config.json file if it exists
with open(pancancer_config_path, "r") as pancancer_config_file:
config_data = json.load(pancancer_config_file)
pancancer_config = {}
aws_config_path = os.path.expanduser("~/.aws/config")
aws_key = ""
aws_secret_key = ""
prev_aws_key = ""
prev_aws_secret_key = ""
if os.path.isfile(aws_config_path):
# read the file, get the values
aws_config = configparser.ConfigParser()
aws_config.read(aws_config_path)
prev_aws_key = aws_config["default"]["aws_access_key_id"]
prev_aws_secret_key = aws_config["default"]["aws_secret_access_key"]
# AWS Key and Secret Key should only be asked for if for some reason they are NOT available (they *should* be, but stuff happens...)
if force_config or prev_aws_key.strip() == "" or prev_aws_secret_key.strip() == "":
aws_key = self._get_config_value(aws_key, prev_aws_key, "What is your AWS Key")
aws_secret_key = self._get_config_value(
aws_secret_key, prev_aws_secret_key, "What is your AWS Secret Key"
)
# If the AWS keys weren't available, ~/.aws/config is either not present or missing values, so re-write it.
aws_config = configparser.ConfigParser()
aws_config["default"] = {"aws_access_key_id": aws_key, "aws_secret_access_key": aws_secret_key}
with open(aws_config_path, "w") as aws_configfile:
aws_config.write(aws_configfile, space_around_delimiters=False)
else:
aws_key = prev_aws_key
aws_secret_key = prev_aws_secret_key
# also: get fleet name, PEM key path, and key name from env. If everything went smoothly, these values should be in
# /opt/from_host/config/pancancer.config
# Actually, fleet name should be coming from process_config.py, not here, since it's passed into the container as an environment variable.
# If it's NOT there, we should askt he user for these values.
bootstrap_config_path = "/opt/from_host/config/pancancer.config"
pem_key_path = ""
prev_pem_key_path = ""
key_name = ""
prev_key_name = ""
prev_security_group = ""
security_group = ""
prev_fleet_size = ""
fleet_size = ""
if os.path.isfile(bootstrap_config_path):
self.log.debug("bootstrap config file exists at: " + bootstrap_config_path)
# load simple config file (it should have been generated by install_bootrap) and get values.
H = dict(line.strip().split("=") for line in open(bootstrap_config_path))
if "PEM_PATH" in H:
prev_pem_key_path = H["PEM_PATH"]
if "KEY_NAME" in H:
prev_key_name = H["KEY_NAME"]
# Fleet size could be defined in the bootstrap config file which should come from the host machine, or it could be defined in the pancancer config file.
# If there's nothing in pancancer config, we'll check in the bootstrap config, and if nothing there, default to 1
if "max_fleet_size" in config_data:
prev_fleet_size = config_data["max_fleet_size"]
else:
if "FLEET_SIZE" in H:
prev_fleet_size = H["FLEET_SIZE"]
if str(prev_fleet_size).strip() == "":
prev_fleet_size = 1
if force_config:
fleet_size = self._get_config_value(
fleet_size,
prev_fleet_size,
"How many VMs do you want in your fleet",
alt_condition=lambda x: x.strip() == "" or x.isdigit() == False or int(x) <= 0,
)
else:
fleet_size = prev_fleet_size
if "security_group" in config_data:
prev_security_group = config_data["security_group"]
else:
if "SECURITY_GROUP" in H:
prev_security_group = H["SECURITY_GROUP"]
if prev_security_group.strip() == "":
prev_security_group = "default"
if force_config:
security_group = self._get_config_value(
security_group, prev_security_group, "What AWS Security Group should the VMs belong to"
)
else:
security_group = prev_security_group
if prev_pem_key_path.strip() != "":
pem_key_path = prev_pem_key_path
if prev_key_name.strip() != "":
key_name = prev_key_name
else:
self.log.debug(
"bootstrap config file cannot be found at: "
+ bootstrap_config_path
+ " so a new one will be written."
)
# Only ask the use about these if one or the other does not exist (indicates there might be a config value/file problem somewhere)
if force_config or prev_pem_key_path.strip() == "" or prev_key_name.strip() == "":
pem_key_path = self._get_config_value(
pem_key_path, prev_pem_key_path, "What is the path to the AWS pem key file that your new VMs to use"
)
key_name = self._get_config_value(key_name, prev_key_name, "What is the Name of this key in AWS")
# Now write the bootstrap config file, but only if we had to ask the user for help filling in the details.
with open(bootstrap_config_path, "w") as bootstrap_config:
bootstrap_config.write("PEM_PATH=" + pem_key_path + "\n")
bootstrap_config.write("KEY_NAME=" + key_name + "\n")
bootstrap_config.write("FLEET_NAME=" + os.environ["FLEET_NAME"] + "\n")
# Now, write the simple JSON config that will be used for the rest of pancancer system.
pancancer_config = {
"max_fleet_size": fleet_size,
"path_to_key": pem_key_path,
"name_of_key": key_name,
"aws_secret_key": aws_secret_key,
"security_group": security_group,
"aws_key": aws_key,
}
if not os.path.exists(os.path.expanduser("~/.pancancer")):
os.makedirs(os.path.expanduser("~/.pancancer"))
with open(pancancer_config_path, "w") as simple_pancancer_config:
simple_pancancer_config.write(str(json.dumps(pancancer_config, sort_keys=True, indent=4)))
self.log.info("Your new pancancer system config file has been written to " + pancancer_config_path)
self.log.info('The next time you want to run "pancancer sysconfig", you can use this file like this: ')
self.log.info("pancancer sysconfig --config " + pancancer_config_path)
# Now that a config is written, USE IT!!
process_config.main(pancancer_config_path)
def take_action(self, parsed_args):
self.log.info('Setting up pancancer config files.')
config_path=vars(parsed_args)['config_path']
force_config=vars(parsed_args)['force_config']
self.log.debug('force_config:' + str(force_config))
if config_path != None:
# if the user specifies a path to an existing file, they can use that instead of the interactive process.
self.log.debug('path to config file: ' + config_path)
process_config.main(config_path)
else:
# if the user did not specify a config, ask them questions and then WRITE/update a config!
pancancer_config_path=os.path.expanduser('~/.pancancer/simple_pancancer_config.json')
config_data={}
if os.path.isfile(pancancer_config_path):
#Read the pancancer_config.json file if it exists
with open(pancancer_config_path,'r') as pancancer_config_file:
config_data = json.load(pancancer_config_file)
pancancer_config={}
aws_config_path = os.path.expanduser('~/.aws/config')
aws_key=''
aws_secret_key=''
prev_aws_key=''
prev_aws_secret_key=''
if os.path.isfile(aws_config_path):
# read the file, get the values
aws_config = configparser.ConfigParser()
aws_config.read(aws_config_path)
prev_aws_key = aws_config['default']['aws_access_key_id']
prev_aws_secret_key = aws_config['default']['aws_secret_access_key']
# AWS Key and Secret Key should only be asked for if for some reason they are NOT available (they *should* be, but stuff happens...)
if force_config or prev_aws_key.strip() == '' or prev_aws_secret_key.strip() == '':
aws_key = self._get_config_value(aws_key,prev_aws_key,'What is your AWS Key')
aws_secret_key = self._get_config_value(aws_secret_key,prev_aws_secret_key,'What is your AWS Secret Key')
# If the AWS keys weren't available, ~/.aws/config is either not present or missing values, so re-write it.
aws_config = configparser.ConfigParser()
aws_config['default'] = {'aws_access_key_id':aws_key, 'aws_secret_access_key':aws_secret_key}
with open(aws_config_path,'w') as aws_configfile:
aws_config.write(aws_configfile,space_around_delimiters=False)
else:
aws_key = prev_aws_key
aws_secret_key = prev_aws_secret_key
# also: get fleet name, PEM key path, and key name from env. If everything went smoothly, these values should be in
# /opt/from_host/config/pancancer.config
# Actually, fleet name should be coming from process_config.py, not here, since it's passed into the container as an environment variable.
# If it's NOT there, we should askt he user for these values.
bootstrap_config_path='/opt/from_host/config/pancancer.config'
pem_key_path=''
prev_pem_key_path=''
key_name=''
prev_key_name=''
prev_security_group = ''
security_group = ''
prev_fleet_size = ''
fleet_size=''
if os.path.isfile(bootstrap_config_path):
self.log.debug('bootstrap config file exists at: '+bootstrap_config_path)
# load simple config file (it should have been generated by install_bootrap) and get values.
H = dict(line.strip().split('=') for line in open(bootstrap_config_path))
if 'PEM_PATH' in H:
prev_pem_key_path = H['PEM_PATH']
if 'KEY_NAME' in H:
prev_key_name = H['KEY_NAME']
# Fleet size could be defined in the bootstrap config file which should come from the host machine, or it could be defined in the pancancer config file.
# If there's nothing in pancancer config, we'll check in the bootstrap config, and if nothing there, default to 1
if 'max_fleet_size' in config_data:
prev_fleet_size = config_data['max_fleet_size']
else:
if 'FLEET_SIZE' in H:
prev_fleet_size = H['FLEET_SIZE']
if str(prev_fleet_size).strip() == '':
prev_fleet_size = 1
if force_config:
fleet_size = self._get_config_value(fleet_size, prev_fleet_size, 'How many VMs do you want in your fleet', alt_condition = lambda x: x.strip()=='' or x.isdigit() == False or int(x)<=0 )
else:
fleet_size = prev_fleet_size
if 'security_group' in config_data:
prev_security_group = config_data['security_group']
else:
if 'SECURITY_GROUP' in H:
prev_security_group = H['SECURITY_GROUP']
if prev_security_group.strip() == '':
prev_security_group = 'default'
if force_config:
security_group = self._get_config_value(security_group, prev_security_group,'What AWS Security Group should the VMs belong to')
else:
security_group = prev_security_group
if prev_pem_key_path.strip() != '':
pem_key_path = prev_pem_key_path
if prev_key_name.strip() != '':
key_name = prev_key_name
else:
self.log.debug('bootstrap config file cannot be found at: '+bootstrap_config_path+' so a new one will be written.')
# Only ask the use about these if one or the other does not exist (indicates there might be a config value/file problem somewhere)
if force_config or prev_pem_key_path.strip()=='' or prev_key_name.strip()=='' :
pem_key_path = self._get_config_value(pem_key_path, prev_pem_key_path, 'What is the path to the AWS pem key file that your new VMs to use')
key_name = self._get_config_value(key_name, prev_key_name, 'What is the Name of this key in AWS')
#Now write the bootstrap config file, but only if we had to ask the user for help filling in the details.
with open(bootstrap_config_path,'w') as bootstrap_config:
bootstrap_config.write('PEM_PATH='+pem_key_path+'\n')
bootstrap_config.write('KEY_NAME='+key_name+'\n')
bootstrap_config.write('FLEET_NAME='+os.environ['FLEET_NAME']+'\n')
# Now, write the simple JSON config that will be used for the rest of pancancer system.
pancancer_config = { 'max_fleet_size':fleet_size, 'path_to_key': pem_key_path,
'name_of_key':key_name, 'aws_secret_key':aws_secret_key,
'security_group': security_group, 'aws_key':aws_key }
if not os.path.exists(os.path.expanduser('~/.pancancer')):
os.makedirs(os.path.expanduser('~/.pancancer'))
with open(pancancer_config_path,'w') as simple_pancancer_config:
simple_pancancer_config.write(str(json.dumps(pancancer_config,sort_keys=True, indent=4) ))
self.log.info('Your new pancancer system config file has been written to '+pancancer_config_path)
self.log.info('The next time you want to run \"pancancer sysconfig\", you can use this file like this: ')
self.log.info('pancancer sysconfig --config '+pancancer_config_path)
# Now that a config is written, USE IT!!
process_config.main(pancancer_config_path)
def take_action(self, parsed_args):
self.log.info('Setting up pancancer config files.')
config_path=vars(parsed_args)['config_path']
force_config=vars(parsed_args)['force_config']
self.log.debug('force_config:' + str(force_config))
if config_path != None:
# if the user specifies a path to an existing file, they can use that instead of the interactive process.
self.log.debug('path to config file: ' + config_path)
process_config.main(config_path)
else:
# if the user did not specify a config, ask them questions and then WRITE/update a config!
pancancer_config_path=os.path.expanduser('~/.pancancer/simple_pancancer_config.json')
config_data={}
if os.path.isfile(pancancer_config_path):
#Read the pancancer_config.json file if it exists
with open(pancancer_config_path,'r') as pancancer_config_file:
config_data = json.load(pancancer_config_file)
pancancer_config={}
aws_config_path = os.path.expanduser('~/.aws/config')
aws_key=''
aws_secret_key=''
prev_aws_key=''
prev_aws_secret_key=''
if os.path.isfile(aws_config_path):
# read the file, get the values
aws_config = configparser.ConfigParser()
aws_config.read(aws_config_path)
prev_aws_key = aws_config['default']['aws_access_key_id']
prev_aws_secret_key = aws_config['default']['aws_secret_access_key']
# also: get fleet name, PEM key path, and key name from env. If everything went smoothly, these values should be in
# /opt/from_host/config/pancancer.config
# Actually, fleet name should be coming from process_config.py, not here, since it's passed into the container as an environment variable.
# If it's NOT there, we should askt he user for these values.
bootstrap_config_path='/opt/from_host/config/pancancer.config'
pem_key_path=''
prev_pem_key_path=''
key_name=''
spot_price = ''
prev_key_name=''
security_group = ''
#prev_fleet_size = ''
fleet_size=''
workflow_listing_url=''
cloud_env=''
bootstrap_config_exists = False
if os.path.isfile(bootstrap_config_path):
self.log.debug('bootstrap config file exists at: '+bootstrap_config_path)
bootstrap_config_exists = True
# load simple config file (it should have been generated by install_bootrap) and get values.
H = dict(line.strip().split('=',maxsplit=1) for line in open(bootstrap_config_path))
#Now, need to strip the leading and trailing double-quotes
for k in H:
H[k] = H[k].lstrip('"').rstrip('"')
cloud_env = self._ask_question_or_set_to_prev(force_config, 'CLOUD_ENV', H, 'cloud_env', config_data, 'What Cloud Environment (must be one of: AWS, Azure, OpenStack) are you working in', alt_condition = lambda x: x.strip() =='AWS' or x.strip() =='Azure' or x.strip() =='OpenStack')
if 'PEM_PATH' in H:
prev_pem_key_path = H['PEM_PATH']
if 'KEY_NAME' in H:
prev_key_name = H['KEY_NAME']
# pull the workflow URL from the config
if 'WORKFLOW_LISTING_URL' in H:
workflow_listing_url = H['WORKFLOW_LISTING_URL']
fleet_size = self._ask_question_or_set_to_prev(force_config, 'FLEET_SIZE', H, 'max_fleet_size', config_data, 'How many VMs do you want in your fleet', alt_condition = lambda x: x.strip()!='' and x.isdigit() and int(x)>0)
if prev_pem_key_path.strip() != '':
pem_key_path = prev_pem_key_path
if prev_key_name.strip() != '':
key_name = prev_key_name
if cloud_env.upper() == 'AWS':
# Ask AWS questions
aws_secret_key, aws_key, security_group, spot_price = self._ask_AWS_questions(force_config, config_data, aws_config_path, aws_key, aws_secret_key, prev_aws_key, prev_aws_secret_key, aws_config, H)
elif cloud_env.upper() == 'AZURE':
# Ask Azure questions
az_subscription_id, az_storage_account, az_storage_account_key, az_ad_user, az_ad_password, az_tenant_id, az_client_id, az_virtual_network, az_location = self._ask_Azure_questions(force_config, config_data, H)
elif cloud_env.upper() == 'OPENSTACK':
# Ask OpenStack questions
os_username, os_password, security_group, os_endpoint, os_region, os_zone, os_network_id = self._ask_OpenStack_questions(force_config, config_data, H)
else:
print('Unknown cloud environment: \''+cloud_env+'\'')
else:
self.log.debug('bootstrap config file cannot be found at: '+bootstrap_config_path+' so a new one will be written.')
# Only ask the use about these if one or the other does not exist (indicates there might be a config value/file problem somewhere)
if force_config or prev_pem_key_path.strip()=='' or prev_key_name.strip()=='' or not bootstrap_config_exists:
pem_key_path = self._get_config_value( prev_pem_key_path, 'What is the path to the pem key file that your new VMs to use')
key_name = self._get_config_value( prev_key_name, 'What is the Name of this key')
# Now (re)write the bootstrap config file if the user forced sysconfig, OR if the key name or path were missing (because those should never be missing).
with open(bootstrap_config_path,'w') as bootstrap_config:
bootstrap_config.write('PEM_PATH='+pem_key_path+'\n')
bootstrap_config.write('KEY_NAME='+key_name+'\n')
bootstrap_config.write('FLEET_NAME='+os.environ['FLEET_NAME']+'\n')
bootstrap_config.write('WORKFLOW_LISTING_URL='+workflow_listing_url+'\n')
bootstrap_config.write('SECURITY_GROUP='+security_group+'\n')
if cloud_env.upper() == 'AZURE':
bootstrap_config.write('AZURE_SUBSCRIPTION='+az_subscription_id+'\n')
bootstrap_config.write('AZURE_STORAGE_ACCOUNT='+az_storage_account+'\n')
bootstrap_config.write('AZURE_STORAGE_ACCOUNT_KEY='+az_storage_account_key+'\n')
bootstrap_config.write('AZURE_AD_USER='******'\n')
bootstrap_config.write('AZURE_AD_PASSWD='+az_ad_password+'\n')
bootstrap_config.write('AZURE_AD_TENANT='+az_tenant_id+'\n')
bootstrap_config.write('AZURE_AD_CLIENT='+az_client_id+'\n')
elif cloud_env.upper() == 'OPENSTACK':
bootstrap_config.write('OS_USERNAME='******'\n')
bootstrap_config.write('OS_PASSWORD='******'\n')
bootstrap_config.write('OS_ENDPOINT='+os_endpoint+'\n')
bootstrap_config.write('OS_REGION='+os_region+'\n')
bootstrap_config.write('OS_ZONE='+os_zone+'\n')
# Write the simple JSON config that will be used for the rest of pancancer system.
# This JSON file will be used as the input to the template file "panancer_config.mustache".
pancancer_config = { 'max_fleet_size':fleet_size, 'path_to_key': pem_key_path,
'name_of_key':key_name, 'security_group': security_group, 'cloud_env':cloud_env,
'workflow_listing_url':workflow_listing_url}
if cloud_env == 'AWS':
pancancer_config.update( { 'aws_secret_key':aws_secret_key,
'aws_key':aws_key,
'spot_price': spot_price} )
elif cloud_env == 'Azure':
pancancer_config.update( { 'az_subscription_id': az_subscription_id, 'az_storage_account': az_storage_account,
'az_storage_account_key': az_storage_account_key, 'az_ad_user': az_ad_user,
'az_ad_password': az_ad_password, 'az_ad_tenant_id': az_tenant_id,
'az_ad_client_id': az_client_id, 'az_virtual_network': az_virtual_network,
'az_location': az_location, } )
elif cloud_env == 'OpenStack':
pancancer_config.update( {'os_username': os_username, 'os_password': os_password,
'os_endpoint': os_endpoint, 'os_region': os_region,
'os_zone': os_zone, 'os_network_id': os_network_id } )
if not os.path.exists(os.path.expanduser('~/.pancancer')):
os.makedirs(os.path.expanduser('~/.pancancer'))
with open(pancancer_config_path,'w') as simple_pancancer_config:
simple_pancancer_config.write(str(json.dumps(pancancer_config,sort_keys=True, indent=4) ))
self.log.info('Your new pancancer system config file has been written to '+pancancer_config_path)
self.log.info('The next time you want to run \"pancancer sysconfig\", you can use this file like this: ')
self.log.info('pancancer sysconfig --config '+pancancer_config_path)
if cloud_env =='AWS':
# Copy the AWS config file to ~/.gnos, because some workflows may need it to access S3 and it's just easier to do this automatically
# then to tell the user to do it. We don't copy to ~/.aws because currently, the ~/.gnos directory and its contents get copied to the
# workers and ~/.gnos on the worker will get mounted into the running seqware container. In the future, it may be possible to copy ~/.aws
# to the workers and have the seqware container look at that as well as ~/.gnos ...
shutil.copy2(aws_config_path,os.path.expanduser('~/.gnos/config'))
# Now that a config is written, USE IT!!
process_config.main(pancancer_config_path)
