def setup(path=None): """Setup cloudscheduler using config file. setup will look for a configuration file specified on the command line, or in ~/.cloudscheduler.conf or /etc/cloudscheduler.conf """ global condor_webservice_url global condor_collector_url global condor_retrieval_method global condor_q_command global condor_status_command global condor_status_master_command global condor_off_command global condor_on_command global ssh_path global openssl_path global condor_context_file global condor_host global condor_host_on_vm global vm_lifetime global cert_file global key_file global cert_file_on_vm global key_file_on_vm global ca_root_certs global ca_signing_policies global cloudscheduler_ssh_key global cloud_resource_config global image_attach_device global scratch_attach_device global info_server_port global admin_server_port global workspace_path global persistence_file global user_limit_file global job_ban_timeout global ban_tracking global ban_file global ban_min_track global ban_failrate_threshold global polling_error_threshold global condor_register_time_limit global graceful_shutdown global graceful_shutdown_method global retire_before_lifetime global retire_before_lifetime_factor global retire_missing_vms global clean_shutdown_idle global getclouds global scheduling_metric global scheduling_algorithm global job_distribution_type global high_priority_job_support global high_priority_job_weight global cpu_distribution_weight global memory_distribution_weight global storage_distribution_weight global cleanup_interval global vm_poller_interval global job_poller_interval global machine_poller_interval global scheduler_interval global job_proxy_refresher_interval global job_proxy_renewal_threshold global vm_proxy_refresher_interval global vm_proxy_renewal_threshold global vm_proxy_shutdown_threshold global vm_connection_fail_threshold global vm_start_running_timeout global vm_idle_threshold global max_starting_vm global proxy_cache_dir global myproxy_logon_command global override_vmtype global vm_reqs_from_condor_reqs global adjust_insufficient_resources global default_VMType global default_VMNetwork global default_VMCPUArch global default_VMName global default_VMLoc global default_VMAMI global default_VMMem global default_VMCPUCores global default_VMStorage global default_VMInstanceType global default_VMMaximumPrice global default_VMProxyNonBoot global log_level global log_location global log_stdout global log_max_size global log_format global use_pyopenssl homedir = os.path.expanduser('~') # Find config file if not path: if os.path.exists(homedir + "/.cloudscheduler/cloud_scheduler.conf"): path = homedir + "/.cloudscheduler/cloud_scheduler.conf" elif os.path.exists("/etc/cloudscheduler/cloud_scheduler.conf"): path = "/etc/cloudscheduler/cloud_scheduler.conf" else: print >> sys.stderr, "Configuration file problem: There doesn't " \ "seem to be a configuration file. " \ "You can specify one with the --config-file parameter, " \ "or put one in ~/.cloudscheduler/cloud_scheduler.conf or "\ "/etc/cloudscheduler/cloud_scheduler.conf" sys.exit(1) # Read config file config_file = ConfigParser.ConfigParser() try: config_file.read(path) except IOError: print >> sys.stderr, "Configuration file problem: There was a " \ "problem reading %s. Check that it is readable," \ "and that it exists. " % path raise except ConfigParser.ParsingError: print >> sys.stderr, "Configuration file problem: Couldn't " \ "parse your file. Check for spaces before or after variables." raise except: print "Configuration file problem: There is something wrong with " \ "your config file." raise if config_file.has_option("global", "condor_retrieval_method"): condor_retrieval_method = config_file.get("global", "condor_retrieval_method") if config_file.has_option("global", "condor_q_command"): condor_q_command = config_file.get("global", "condor_q_command") if config_file.has_option("global", "condor_off_command"): condor_off_command = config_file.get("global", "condor_off_command") if config_file.has_option("global", "condor_on_command"): condor_on_command = config_file.get("global", "condor_on_command") if config_file.has_option("global", "ssh_path"): ssh_path = config_file.get("global", "ssh_path") if config_file.has_option("global", "openssl_path"): openssl_path = config_file.get("global", "openssl_path") if config_file.has_option("global", "condor_status_command"): condor_status_command = config_file.get("global", "condor_status_command") if config_file.has_option("global", "condor_status_master_command"): condor_status_master_command = config_file.get( "global", "condor_status_master_command") if config_file.has_option("global", "condor_webservice_url"): condor_webservice_url = config_file.get("global", "condor_webservice_url") if config_file.has_option("global", "condor_collector_url"): condor_collector_url = config_file.get("global", "condor_collector_url") if config_file.has_option("global", "condor_host_on_vm"): condor_host_on_vm = config_file.get("global", "condor_host_on_vm") if config_file.has_option("global", "condor_context_file"): condor_context_file = config_file.get("global", "condor_context_file") if config_file.has_option("global", "vm_lifetime"): try: vm_lifetime = config_file.getint("global", "vm_lifetime") except ValueError: print "Configuration file problem: vm_lifetime must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "cert_file"): cert_file = config_file.get("global", "cert_file") if config_file.has_option("global", "key_file"): key_file = config_file.get("global", "key_file") if config_file.has_option("global", "cert_file_on_vm"): cert_file_on_vm = config_file.get("global", "cert_file_on_vm") if config_file.has_option("global", "key_file_on_vm"): key_file_on_vm = config_file.get("global", "key_file_on_vm") if config_file.has_option("global", "ca_root_certs"): ca_root_certs = config_file.get("global", "ca_root_certs").split(',') if config_file.has_option("global", "ca_signing_policies"): ca_signing_policies = config_file.get("global", "ca_signing_policies").split(',') if config_file.has_option("global", "cloudscheduler_ssh_key"): cloudscheduler_ssh_key = config_file.get("global", "cloudscheduler_ssh_key") if config_file.has_option("global", "cloud_resource_config"): cloud_resource_config = config_file.get("global", "cloud_resource_config") if config_file.has_option("global", "image_attach_device"): image_attach_device = config_file.get("global", "image_attach_device") if config_file.has_option("global", "scratch_attach_device"): scratch_attach_device = config_file.get("global", "scratch_attach_device") if config_file.has_option("global", "info_server_port"): try: info_server_port = config_file.getint("global", "info_server_port") except ValueError: print "Configuration file problem: info_server_port must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "admin_server_port"): try: info_server_port = config_file.getint("global", "admin_server_port") except ValueError: print "Configuration file problem: admin_server_port must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "workspace_path"): workspace_path = config_file.get("global", "workspace_path") if config_file.has_option("global", "persistence_file"): persistence_file = config_file.get("global", "persistence_file") if config_file.has_option("global", "user_limit_file"): user_limit_file = config_file.get("global", "user_limit_file") if config_file.has_option("global", "job_ban_timeout"): try: job_ban_timeout = 60 * config_file.getint("global", "job_ban_timeout") except ValueError: print "Configuration file problem: job_ban_timeout must be an " \ "integer value in minutes." sys.exit(1) if config_file.has_option("global", "ban_file"): ban_file = config_file.get("global", "ban_file") if config_file.has_option("global", "polling_error_threshold"): try: polling_error_threshold = config_file.getint( "global", "polling_error_threshold") except ValueError: print "Configuration file problem: polling_error_threshold must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "ban_failrate_threshold"): try: ban_failrate_threshold = config_file.getfloat( "global", "ban_failrate_threshold") if ban_failrate_threshold == 0: print "Please use a float value (0, 1.0]" sys.exit(1) except ValueError: print "Configuration file problem: ban_failrate_threshold must be an " \ "float value." sys.exit(1) if config_file.has_option("global", "ban_min_track"): try: ban_min_track = config_file.getint("global", "ban_min_track") except ValueError: print "Configuration file problem: ban_min_track must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "condor_register_time_limit"): try: condor_register_time_limit = 60 * config_file.getint( "global", "condor_register_time_limit") except ValueError: print "Configuration file problem: condor_register_time_limit must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "ban_tracking"): try: ban_tracking = config_file.getboolean("global", "ban_tracking") except ValueError: print "Configuration file problem: ban_tracking must be an " \ "boolean value." sys.exit(1) if config_file.has_option("global", "graceful_shutdown"): try: graceful_shutdown = config_file.getboolean("global", "graceful_shutdown") except ValueError: print "Configuration file problem: graceful_shutdown must be an " \ "boolean value." sys.exit(1) if config_file.has_option("global", "graceful_shutdown_method"): graceful_shutdown_method = config_file.get("global", "graceful_shutdown_method") if config_file.has_option("global", "retire_before_lifetime"): try: retire_before_lifetime = config_file.getboolean( "global", "retire_before_lifetime") except ValueError: print "Configuration file problem: retire_before_lifetime must be an " \ "boolean value." sys.exit(1) if config_file.has_option("global", "retire_before_lifetime_factor"): try: retire_before_lifetime_factor = config_file.getfloat( "global", "retire_before_lifetime_factor") if retire_before_lifetime_factor < 1.0: print "Please use a float value (1.0, X] for the retire_before_lifetime_factor" sys.exit(1) except ValueError: print "Configuration file problem: retire_before_lifetime_factor must be a " \ "float value." sys.exit(1) if config_file.has_option("global", "retire_missing_vms"): try: retire_missing_vms = config_file.getboolean( "global", "retire_missing_vms") except ValueError: print "Configuration file problem: retire_missing_vms must be an " \ "boolean value." sys.exit(1) if config_file.has_option("global", "clean_shutdown_idle"): try: clean_shutdown_idle = config_file.getboolean( "global", "clean_shutdown_idle") except ValueError: print "Configuration file problem: clean_shutdown_idle must be an " \ "boolean value." sys.exit(1) if config_file.has_option("global", "getclouds"): try: getclouds = config_file.getboolean("global", "getclouds") except ValueError: print "Configuration file problem: getclouds must be an " \ "boolean value." sys.exit(1) if config_file.has_option("global", "scheduling_metric"): scheduling_metric = config_file.get("global", "scheduling_metric") if config_file.has_option("global", "job_distribution_type"): job_distribution_type = config_file.get("global", "job_distribution_type") if config_file.has_option("global", "memory_distribution_weight"): try: memory_distribution_weight = config_file.getfloat( "global", "memory_distribution_weight") if ban_failrate_threshold <= 0: print "Please use a float value (0, x]" sys.exit(1) except ValueError: print "Configuration file problem: memory_distribution_weight must be an " \ "float value." sys.exit(1) if config_file.has_option("global", "cpu_distribution_weight"): try: cpu_distribution_weight = config_file.getfloat( "global", "cpu_distribution_weight") if ban_failrate_threshold <= 0: print "Please use a float value (0, x]" sys.exit(1) except ValueError: print "Configuration file problem: cpu_distribution_weight must be an " \ "float value." sys.exit(1) if config_file.has_option("global", "storage_distribution_weight"): try: storage_distribution_weight = config_file.getfloat( "global", "storage_distribution_weight") if ban_failrate_threshold <= 0: print "Please use a float value (0, x]" sys.exit(1) except ValueError: print "Configuration file problem: storage_distribution_weight must be an " \ "float value." sys.exit(1) if config_file.has_option("global", "scheduling_algorithm"): scheduling_algorithm = config_file.get("global", "scheduling_algorithm") if config_file.has_option("global", "high_priority_job_support"): try: high_priority_job_support = config_file.getboolean( "global", "high_priority_job_support") except ValueError: print "Configuration file problem: high_priority_job_support must be an " \ "boolean value." sys.exit(1) if config_file.has_option("global", "high_priority_job_weight"): try: high_priority_job_weight = config_file.getint( "global", "high_priority_job_weight") except ValueError: print "Configuration file problem: high_priority_job_weight must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "scheduler_interval"): try: scheduler_interval = config_file.getint("global", "scheduler_interval") except ValueError: print "Configuration file problem: scheduler_interval must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "vm_poller_interval"): try: vm_poller_interval = config_file.getint("global", "vm_poller_interval") except ValueError: print "Configuration file problem: vm_poller_interval must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "job_poller_interval"): try: job_poller_interval = config_file.getint("global", "job_poller_interval") except ValueError: print "Configuration file problem: job_poller_interval must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "machine_poller_interval"): try: machine_poller_interval = config_file.getint( "global", "machine_poller_interval") except ValueError: print "Configuration file problem: machine_poller_interval must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "cleanup_interval"): try: cleanup_interval = config_file.getint("global", "cleanup_interval") except ValueError: print "Configuration file problem: cleanup_interval must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "job_proxy_refresher_interval"): try: job_proxy_refresher_interval = config_file.getint( "global", "job_proxy_refresher_interval") except ValueError: print "Configuration file problem: job_proxy_refresher_interval must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "job_proxy_renewal_threshold"): try: job_proxy_renewal_threshold = config_file.getint( "global", "job_proxy_renewal_threshold") except ValueError: print "Configuration file problem: job_proxy_renewal_threshold must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "vm_proxy_refresher_interval"): try: vm_proxy_refresher_interval = config_file.getint( "global", "vm_proxy_refresher_interval") except ValueError: print "Configuration file problem: vm_proxy_refresher_interval must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "vm_proxy_renewal_threshold"): try: vm_proxy_renewal_threshold = config_file.getint( "global", "vm_proxy_renewal_threshold") except ValueError: print "Configuration file problem: vm_proxy_renewal_threshold must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "vm_proxy_shutdown_threshold"): try: vm_proxy_shutdown_threshold = config_file.getint( "global", "vm_proxy_shutdown_threshold") except ValueError: print "Configuration file problem: vm_proxy_shutdown_threshold must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "vm_connection_fail_threshold"): try: vm_connection_fail_threshold = config_file.getint( "global", "vm_connection_fail_threshold") except ValueError: print "Configuration file problem: vm_connection_fail_threshold must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "vm_idle_threshold"): try: vm_idle_threshold = config_file.getint("global", "vm_idle_threshold") except ValueError: print "Configuration file problem: vm_idle_threshold must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "vm_start_running_timeout"): try: vm_start_running_timeout = config_file.getint( "global", "vm_start_running_timeout") except ValueError: print "Configuration file problem: vm_start_running_timeout must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "max_starting_vm"): try: max_starting_vm = config_file.getint("global", "max_starting_vm") if max_starting_vm < -1: max_starting_vm = -1 except ValueError: print "Configuration file problem: max_starting_vm must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "max_destroy_threads"): try: max_destroy_threads = config_file.getint("global", "max_destroy_threads") if max_destroy_threads <= 0: max_destroy_threads = 1 except ValueError: print "Configuration file problem: max_destroy_threads must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "proxy_cache_dir"): proxy_cache_dir = config_file.get("global", "proxy_cache_dir") if config_file.has_option("global", "myproxy_logon_command"): myproxy_logon_command = config_file.get("global", "myproxy_logon_command") if config_file.has_option("global", "override_vmtype"): try: override_vmtype = config_file.getboolean("global", "override_vmtype") except ValueError: print "Configuration file problem: override_vmtype must be a" \ " Boolean value." if config_file.has_option("global", "vm_reqs_from_condor_reqs"): try: vm_reqs_from_condor_reqs = config_file.getboolean( "global", "vm_reqs_from_condor_reqs") except ValueError: print "Configuration file problem: vm_reqs_from_condor_reqs must be a" \ " Boolean value." if config_file.has_option("global", "adjust_insufficient_resources"): try: adjust_insufficient_resources = config_file.getboolean( "global", "adjust_insufficient_resources") except ValueError: print "Configuration file problem: adjust_insufficient_resources must be a" \ " Boolean value." if config_file.has_option("global", "connection_fail_disable_time"): try: connection_fail_disable_time = config_file.getint( "global", "connection_fail_disable_time") except ValueError: print "Configuration file problem: connection_fail_disable_time must be an " \ "integer value." sys.exit(1) # Default Logging options if config_file.has_option("logging", "log_level"): log_level = config_file.get("logging", "log_level") if config_file.has_option("logging", "log_location"): log_location = os.path.expanduser( config_file.get("logging", "log_location")) if config_file.has_option("logging", "log_stdout"): try: log_stdout = config_file.getboolean("logging", "log_stdout") except ValueError: print "Configuration file problem: log_stdout must be a" \ " Boolean value." if config_file.has_option("logging", "log_max_size"): try: log_max_size = config_file.getint("logging", "log_max_size") except ValueError: print "Configuration file problem: log_max_size must be an " \ "integer value in bytes." sys.exit(1) if config_file.has_option("logging", "log_format"): log_format = config_file.get("logging", "log_format") # Default Job options if config_file.has_option("job", "default_VMType"): default_VMType = config_file.get("job", "default_VMType") if config_file.has_option("job", "default_VMNetwork"): default_VMNetwork = config_file.get("job", "default_VMNetwork") if config_file.has_option("job", "default_VMCPUArch"): default_VMCPUArch = config_file.get("job", "default_VMCPUArch") if config_file.has_option("job", "default_VMName"): default_VMName = config_file.get("job", "default_VMName") if config_file.has_option("job", "default_VMLoc"): default_VMLoc = config_file.get("job", "default_VMLoc") if config_file.has_option("job", "default_VMAMI"): default_VMAMI = config_file.get("job", "default_VMAMI") if config_file.has_option("job", "default_VMMem"): try: default_VMMem = config_file.getint("job", "default_VMMem") except ValueError: print "Configuration file problem: default_VMMem must be an " \ "integer value." sys.exit(1) if config_file.has_option("job", "default_VMCPUCores"): try: default_VMCPUCores = config_file.getint("job", "default_VMCPUCores") except ValueError: print "Configuration file problem: default_VMCPUCores must be an " \ "integer value." sys.exit(1) if config_file.has_option("job", "default_VMStorage"): try: default_VMStorage = config_file.getint("job", "default_VMStorage") except ValueError: print "Configuration file problem: default_VMStorage must be an " \ "integer value." sys.exit(1) if config_file.has_option("job", "default_VMInstanceType"): default_VMInstanceType = config_file.get("job", "default_VMInstanceType") if config_file.has_option("job", "default_VMMaximumPrice"): try: default_VMMaximumPrice = config_file.getint( "job", "default_VMMaximumPrice") except ValueError: print "Configuration file problem: default_VMMaximumPrice must be an " \ "integer value." sys.exit(1) if config_file.has_option("job", "default_VMProxyNonBoot"): try: default_VMProxyNonBoot = config_file.getboolean( "global", "default_VMProxyNonBoot") except ValueError: print "Configuration file problem: default_VMProxyNonBoot must be a" \ " Boolean value." # Derived options if condor_host_on_vm: condor_host = condor_host_on_vm else: condor_host = utilities.get_hostname_from_url(condor_webservice_url) if config_file.has_option("global", "use_pyopenssl"): use_pyopenssl = config_file.getboolean("global", "use_pyopenssl")
def setup(path=None): global condor_webservice_url global condor_collector_url global condor_retrieval_method global condor_q_command global condor_status_command global condor_context_file global condor_host global condor_host_on_vm global vm_lifetime global cert_file global key_file global cert_file_on_vm global key_file_on_vm global ca_root_certs global ca_signing_policies global cloudscheduler_ssh_key global cloud_resource_config global image_attach_device global scratch_attach_device global info_server_port global workspace_path global persistence_file global ban_tracking global ban_file global ban_min_track global ban_failrate_threshold global polling_error_threshold global condor_register_time_limit global graceful_shutdown global graceful_shutdown_method global getclouds global scheduling_metric global high_priority_job_support global high_priority_job_weight global cleanup_interval global vm_poller_interval global job_poller_interval global machine_poller_interval global scheduler_interval global log_level global log_location global log_stdout global log_max_size global log_format homedir = os.path.expanduser('~') # Find config file if not path: if os.path.exists(homedir + "/.cloudscheduler/cloud_scheduler.conf"): path = homedir + "/.cloudscheduler/cloud_scheduler.conf" elif os.path.exists("/etc/cloudscheduler/cloud_scheduler.conf"): path = "/etc/cloudscheduler/cloud_scheduler.conf" else: print >> sys.stderr, "Configuration file problem: There doesn't " \ "seem to be a configuration file. " \ "You can specify one with the --config-file parameter, " \ "or put one in ~/.cloudscheduler/cloud_scheduler.conf or "\ "/etc/cloudscheduler/cloud_scheduler.conf" sys.exit(1) # Read config file config_file = ConfigParser.ConfigParser() try: config_file.read(path) except IOError: print >> sys.stderr, "Configuration file problem: There was a " \ "problem reading %s. Check that it is readable," \ "and that it exists. " % path raise except ConfigParser.ParsingError: print >> sys.stderr, "Configuration file problem: Couldn't " \ "parse your file. Check for spaces before or after variables." raise except: print "Configuration file problem: There is something wrong with " \ "your config file." raise if config_file.has_option("global", "condor_retrieval_method"): condor_retrieval_method = config_file.get("global", "condor_retrieval_method") if config_file.has_option("global", "condor_q_command"): condor_q_command = config_file.get("global", "condor_q_command") if config_file.has_option("global", "condor_status_command"): condor_status_command = config_file.get("global", "condor_status_command") if config_file.has_option("global", "condor_webservice_url"): condor_webservice_url = config_file.get("global", "condor_webservice_url") if config_file.has_option("global", "condor_collector_url"): condor_collector_url = config_file.get("global", "condor_collector_url") if config_file.has_option("global", "condor_host_on_vm"): condor_host_on_vm = config_file.get("global", "condor_host_on_vm") if config_file.has_option("global", "condor_context_file"): condor_context_file = config_file.get("global", "condor_context_file") if config_file.has_option("global", "vm_lifetime"): try: vm_lifetime = config_file.getint("global", "vm_lifetime") except ValueError: print "Configuration file problem: vm_lifetime must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "cert_file"): cert_file = config_file.get("global", "cert_file") if config_file.has_option("global", "key_file"): key_file = config_file.get("global", "key_file") if config_file.has_option("global", "cert_file_on_vm"): cert_file_on_vm = config_file.get("global", "cert_file_on_vm") if config_file.has_option("global", "key_file_on_vm"): key_file_on_vm = config_file.get("global", "key_file_on_vm") if config_file.has_option("global", "ca_root_certs"): ca_root_certs = config_file.get("global", "ca_root_certs").split(',') if config_file.has_option("global", "ca_signing_policies"): ca_signing_policies = config_file.get("global", "ca_signing_policies").split(',') if config_file.has_option("global", "cloudscheduler_ssh_key"): cloudscheduler_ssh_key = config_file.get("global", "cloudscheduler_ssh_key") if config_file.has_option("global", "cloud_resource_config"): cloud_resource_config = config_file.get("global", "cloud_resource_config") if config_file.has_option("global", "image_attach_device"): image_attach_device = config_file.get("global", "image_attach_device") if config_file.has_option("global", "scratch_attach_device"): scratch_attach_device = config_file.get("global", "scratch_attach_device") if config_file.has_option("global", "info_server_port"): try: info_server_port = config_file.getint("global", "info_server_port") except ValueError: print "Configuration file problem: info_server_port must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "workspace_path"): workspace_path = config_file.get("global", "workspace_path") if config_file.has_option("global", "persistence_file"): persistence_file = config_file.get("global", "persistence_file") if config_file.has_option("global", "ban_file"): ban_file = config_file.get("global", "ban_file") if config_file.has_option("global", "polling_error_threshold"): try: polling_error_threshold = config_file.getint("global", "polling_error_threshold") except ValueError: print "Configuration file problem: polling_error_threshold must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "ban_failrate_threshold"): try: ban_failrate_threshold = config_file.getfloat("global", "ban_failrate_threshold") if ban_failrate_threshold == 0: print "Please use a float value (0, 1.0]" sys.exit(1) except ValueError: print "Configuration file problem: ban_failrate_threshold must be an " \ "float value." sys.exit(1) if config_file.has_option("global", "ban_min_track"): try: ban_min_track = config_file.getint("global", "ban_min_track") except ValueError: print "Configuration file problem: ban_min_track must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "condor_register_time_limit"): try: condor_register_time_limit = 60*config_file.getint("global", "condor_register_time_limit") except ValueError: print "Configuration file problem: condor_register_time_limit must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "ban_tracking"): ban_tracking = config_file.getboolean("global", "ban_tracking") if config_file.has_option("global", "graceful_shutdown"): graceful_shutdown = config_file.getboolean("global", "graceful_shutdown") if config_file.has_option("global", "graceful_shutdown_method"): graceful_shutdown_method = config_file.get("global", "graceful_shutdown_method") if config_file.has_option("global", "getclouds"): getclouds = config_file.getboolean("global", "getclouds") if config_file.has_option("global", "scheduling_metric"): scheduling_metric = config_file.get("global", "scheduling_metric") if config_file.has_option("global", "high_priority_job_support"): high_priority_job_support = config_file.getboolean("global", "high_priority_job_support") if config_file.has_option("global", "high_priority_job_weight"): try: high_priority_job_weight = config_file.getint("global", "high_priority_job_weight") except ValueError: print "Configuration file problem: high_priority_job_weight must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "scheduler_interval"): try: scheduler_interval = config_file.getint("global", "scheduler_interval") except ValueError: print "Configuration file problem: scheduler_interval must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "vm_poller_interval"): try: vm_poller_interval = config_file.getint("global", "vm_poller_interval") except ValueError: print "Configuration file problem: vm_poller_interval must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "job_poller_interval"): try: job_poller_interval = config_file.getint("global", "job_poller_interval") except ValueError: print "Configuration file problem: job_poller_interval must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "machine_poller_interval"): try: machine_poller_interval = config_file.getint("global", "machine_poller_interval") except ValueError: print "Configuration file problem: machine_poller_interval must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "cleanup_interval"): try: cleanup_interval = config_file.getint("global", "cleanup_interval") except ValueError: print "Configuration file problem: cleanup_interval must be an " \ "integer value." sys.exit(1) if config_file.has_option("logging", "log_level"): log_level = config_file.get("logging", "log_level") if config_file.has_option("logging", "log_location"): log_location = os.path.expanduser(config_file.get("logging", "log_location")) if config_file.has_option("logging", "log_stdout"): log_stdout = config_file.getboolean("logging", "log_stdout") if config_file.has_option("logging", "log_max_size"): try: log_max_size = config_file.getint("logging", "log_max_size") except ValueError: print "Configuration file problem: log_max_size must be an " \ "integer value in bytes." sys.exit(1) if config_file.has_option("logging", "log_format"): log_format = config_file.get("logging", "log_format") # Derived options if condor_host_on_vm: condor_host = condor_host_on_vm else: condor_host = utilities.get_hostname_from_url(condor_webservice_url)
def setup(path=None): """Setup cloudscheduler using config file. setup will look for a configuration file specified on the command line, or in ~/.cloudscheduler.conf or /etc/cloudscheduler.conf """ global condor_webservice_url global condor_collector_url global condor_retrieval_method global condor_q_command global condor_status_command global condor_off_command global condor_on_command global ssh_path global openssl_path global condor_context_file global condor_host global condor_host_on_vm global vm_lifetime global cert_file global key_file global cert_file_on_vm global key_file_on_vm global ca_root_certs global ca_signing_policies global cloudscheduler_ssh_key global cloud_resource_config global image_attach_device global scratch_attach_device global info_server_port global workspace_path global persistence_file global job_ban_timeout global ban_tracking global ban_file global ban_min_track global ban_failrate_threshold global polling_error_threshold global condor_register_time_limit global graceful_shutdown global graceful_shutdown_method global retire_before_lifetime global retire_before_lifetime_factor global getclouds global scheduling_metric global scheduling_algorithm global job_distribution_type global high_priority_job_support global high_priority_job_weight global cpu_distribution_weight global memory_distribution_weight global storage_distribution_weight global cleanup_interval global vm_poller_interval global job_poller_interval global machine_poller_interval global scheduler_interval global job_proxy_refresher_interval global job_proxy_renewal_threshold global vm_proxy_refresher_interval global vm_proxy_renewal_threshold global vm_proxy_shutdown_threshold global vm_connection_fail_threshold global vm_idle_threshold global proxy_cache_dir global myproxy_logon_command global override_vmtype global default_VMType global default_VMNetwork global default_VMCPUArch global default_VMName global default_VMLoc global default_VMAMI global default_VMMem global default_VMCPUCores global default_VMStorage global default_VMInstanceType global default_VMMaximumPrice global log_level global log_location global log_stdout global log_max_size global log_format global use_pyopenssl homedir = os.path.expanduser('~') # Find config file if not path: if os.path.exists(homedir + "/.cloudscheduler/cloud_scheduler.conf"): path = homedir + "/.cloudscheduler/cloud_scheduler.conf" elif os.path.exists("/etc/cloudscheduler/cloud_scheduler.conf"): path = "/etc/cloudscheduler/cloud_scheduler.conf" else: print >> sys.stderr, "Configuration file problem: There doesn't " \ "seem to be a configuration file. " \ "You can specify one with the --config-file parameter, " \ "or put one in ~/.cloudscheduler/cloud_scheduler.conf or "\ "/etc/cloudscheduler/cloud_scheduler.conf" sys.exit(1) # Read config file config_file = ConfigParser.ConfigParser() try: config_file.read(path) except IOError: print >> sys.stderr, "Configuration file problem: There was a " \ "problem reading %s. Check that it is readable," \ "and that it exists. " % path raise except ConfigParser.ParsingError: print >> sys.stderr, "Configuration file problem: Couldn't " \ "parse your file. Check for spaces before or after variables." raise except: print "Configuration file problem: There is something wrong with " \ "your config file." raise if config_file.has_option("global", "condor_retrieval_method"): condor_retrieval_method = config_file.get("global", "condor_retrieval_method") if config_file.has_option("global", "condor_q_command"): condor_q_command = config_file.get("global", "condor_q_command") if config_file.has_option("global", "condor_off_command"): condor_off_command = config_file.get("global", "condor_off_command") if config_file.has_option("global", "condor_on_command"): condor_on_command = config_file.get("global", "condor_on_command") if config_file.has_option("global", "ssh_path"): ssh_path = config_file.get("global", "ssh_path") if config_file.has_option("global", "openssl_path"): openssl_path = config_file.get("global", "openssl_path") if config_file.has_option("global", "condor_status_command"): condor_status_command = config_file.get("global", "condor_status_command") if config_file.has_option("global", "condor_webservice_url"): condor_webservice_url = config_file.get("global", "condor_webservice_url") if config_file.has_option("global", "condor_collector_url"): condor_collector_url = config_file.get("global", "condor_collector_url") if config_file.has_option("global", "condor_host_on_vm"): condor_host_on_vm = config_file.get("global", "condor_host_on_vm") if config_file.has_option("global", "condor_context_file"): condor_context_file = config_file.get("global", "condor_context_file") if config_file.has_option("global", "vm_lifetime"): try: vm_lifetime = config_file.getint("global", "vm_lifetime") except ValueError: print "Configuration file problem: vm_lifetime must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "cert_file"): cert_file = config_file.get("global", "cert_file") if config_file.has_option("global", "key_file"): key_file = config_file.get("global", "key_file") if config_file.has_option("global", "cert_file_on_vm"): cert_file_on_vm = config_file.get("global", "cert_file_on_vm") if config_file.has_option("global", "key_file_on_vm"): key_file_on_vm = config_file.get("global", "key_file_on_vm") if config_file.has_option("global", "ca_root_certs"): ca_root_certs = config_file.get("global", "ca_root_certs").split(',') if config_file.has_option("global", "ca_signing_policies"): ca_signing_policies = config_file.get("global", "ca_signing_policies").split(',') if config_file.has_option("global", "cloudscheduler_ssh_key"): cloudscheduler_ssh_key = config_file.get("global", "cloudscheduler_ssh_key") if config_file.has_option("global", "cloud_resource_config"): cloud_resource_config = config_file.get("global", "cloud_resource_config") if config_file.has_option("global", "image_attach_device"): image_attach_device = config_file.get("global", "image_attach_device") if config_file.has_option("global", "scratch_attach_device"): scratch_attach_device = config_file.get("global", "scratch_attach_device") if config_file.has_option("global", "info_server_port"): try: info_server_port = config_file.getint("global", "info_server_port") except ValueError: print "Configuration file problem: info_server_port must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "workspace_path"): workspace_path = config_file.get("global", "workspace_path") if config_file.has_option("global", "persistence_file"): persistence_file = config_file.get("global", "persistence_file") if config_file.has_option("global", "job_ban_timeout"): try: job_ban_timeout = 60 * config_file.getint("global", "job_ban_timeout") except ValueError: print "Configuration file problem: job_ban_timeout must be an " \ "integer value in minutes." sys.exit(1) if config_file.has_option("global", "ban_file"): ban_file = config_file.get("global", "ban_file") if config_file.has_option("global", "polling_error_threshold"): try: polling_error_threshold = config_file.getint("global", "polling_error_threshold") except ValueError: print "Configuration file problem: polling_error_threshold must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "ban_failrate_threshold"): try: ban_failrate_threshold = config_file.getfloat("global", "ban_failrate_threshold") if ban_failrate_threshold == 0: print "Please use a float value (0, 1.0]" sys.exit(1) except ValueError: print "Configuration file problem: ban_failrate_threshold must be an " \ "float value." sys.exit(1) if config_file.has_option("global", "ban_min_track"): try: ban_min_track = config_file.getint("global", "ban_min_track") except ValueError: print "Configuration file problem: ban_min_track must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "condor_register_time_limit"): try: condor_register_time_limit = 60*config_file.getint("global", "condor_register_time_limit") except ValueError: print "Configuration file problem: condor_register_time_limit must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "ban_tracking"): ban_tracking = config_file.getboolean("global", "ban_tracking") if config_file.has_option("global", "graceful_shutdown"): graceful_shutdown = config_file.getboolean("global", "graceful_shutdown") if config_file.has_option("global", "graceful_shutdown_method"): graceful_shutdown_method = config_file.get("global", "graceful_shutdown_method") if config_file.has_option("global", "retire_before_lifetime"): retire_before_lifetime = config_file.getboolean("global", "retire_before_lifetime") if config_file.has_option("global", "retire_before_lifetime_factor"): try: retire_before_lifetime_factor = config_file.getfloat("global", "retire_before_lifetime_factor") if retire_before_lifetime_factor < 1.0: print "Please use a float value (1.0, X] for the retire_before_lifetime_factor" sys.exit(1) except ValueError: print "Configuration file problem: retire_before_lifetime_factor must be a " \ "float value." sys.exit(1) if config_file.has_option("global", "getclouds"): getclouds = config_file.getboolean("global", "getclouds") if config_file.has_option("global", "scheduling_metric"): scheduling_metric = config_file.get("global", "scheduling_metric") if config_file.has_option("global", "job_distribution_type"): job_distribution_type = config_file.get("global", "job_distribution_type") if config_file.has_option("global", "memory_distribution_weight"): try: memory_distribution_weight = config_file.getfloat("global", "memory_distribution_weight") if ban_failrate_threshold <= 0: print "Please use a float value (0, x]" sys.exit(1) except ValueError: print "Configuration file problem: memory_distribution_weight must be an " \ "float value." sys.exit(1) if config_file.has_option("global", "cpu_distribution_weight"): try: cpu_distribution_weight = config_file.getfloat("global", "cpu_distribution_weight") if ban_failrate_threshold <= 0: print "Please use a float value (0, x]" sys.exit(1) except ValueError: print "Configuration file problem: cpu_distribution_weight must be an " \ "float value." sys.exit(1) if config_file.has_option("global", "storage_distribution_weight"): try: storage_distribution_weight = config_file.getfloat("global", "storage_distribution_weight") if ban_failrate_threshold <= 0: print "Please use a float value (0, x]" sys.exit(1) except ValueError: print "Configuration file problem: storage_distribution_weight must be an " \ "float value." sys.exit(1) if config_file.has_option("global", "scheduling_algorithm"): scheduling_algorithm = config_file.get("global", "scheduling_algorithm") if config_file.has_option("global", "high_priority_job_support"): high_priority_job_support = config_file.getboolean("global", "high_priority_job_support") if config_file.has_option("global", "high_priority_job_weight"): try: high_priority_job_weight = config_file.getint("global", "high_priority_job_weight") except ValueError: print "Configuration file problem: high_priority_job_weight must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "scheduler_interval"): try: scheduler_interval = config_file.getint("global", "scheduler_interval") except ValueError: print "Configuration file problem: scheduler_interval must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "vm_poller_interval"): try: vm_poller_interval = config_file.getint("global", "vm_poller_interval") except ValueError: print "Configuration file problem: vm_poller_interval must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "job_poller_interval"): try: job_poller_interval = config_file.getint("global", "job_poller_interval") except ValueError: print "Configuration file problem: job_poller_interval must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "machine_poller_interval"): try: machine_poller_interval = config_file.getint("global", "machine_poller_interval") except ValueError: print "Configuration file problem: machine_poller_interval must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "cleanup_interval"): try: cleanup_interval = config_file.getint("global", "cleanup_interval") except ValueError: print "Configuration file problem: cleanup_interval must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "job_proxy_refresher_interval"): try: job_proxy_refresher_interval = config_file.getint("global", "job_proxy_refresher_interval") except ValueError: print "Configuration file problem: job_proxy_refresher_interval must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "job_proxy_renewal_threshold"): try: job_proxy_renewal_threshold = config_file.getint("global", "job_proxy_renewal_threshold") except ValueError: print "Configuration file problem: job_proxy_renewal_threshold must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "vm_proxy_refresher_interval"): try: vm_proxy_refresher_interval = config_file.getint("global", "vm_proxy_refresher_interval") except ValueError: print "Configuration file problem: vm_proxy_refresher_interval must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "vm_proxy_renewal_threshold"): try: vm_proxy_renewal_threshold = config_file.getint("global", "vm_proxy_renewal_threshold") except ValueError: print "Configuration file problem: vm_proxy_renewal_threshold must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "vm_proxy_shutdown_threshold"): try: vm_proxy_shutdown_threshold = config_file.getint("global", "vm_proxy_shutdown_threshold") except ValueError: print "Configuration file problem: vm_proxy_shutdown_threshold must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "vm_connection_fail_threshold"): try: vm_connection_fail_threshold = config_file.getint("global", "vm_connection_fail_threshold") except ValueError: print "Configuration file problem: vm_connection_fail_threshold must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "vm_idle_threshold"): try: vm_idle_threshold = config_file.getint("global", "vm_idle_threshold") except ValueError: print "Configuration file problem: vm_idle_threshold must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "proxy_cache_dir"): proxy_cache_dir = config_file.get("global", "proxy_cache_dir") if config_file.has_option("global", "myproxy_logon_command"): myproxy_logon_command = config_file.get("global", "myproxy_logon_command") if config_file.has_option("global", "override_vmtype"): override_vmtype = config_file.getboolean("global", "override_vmtype") if config_file.has_option("logging", "log_level"): log_level = config_file.get("logging", "log_level") if config_file.has_option("logging", "log_location"): log_location = os.path.expanduser(config_file.get("logging", "log_location")) if config_file.has_option("logging", "log_stdout"): log_stdout = config_file.getboolean("logging", "log_stdout") if config_file.has_option("logging", "log_max_size"): try: log_max_size = config_file.getint("logging", "log_max_size") except ValueError: print "Configuration file problem: log_max_size must be an " \ "integer value in bytes." sys.exit(1) if config_file.has_option("logging", "log_format"): log_format = config_file.get("logging", "log_format") # Default Job options if config_file.has_option("job", "default_VMType"): default_VMType = config_file.get("job", "default_VMType") if config_file.has_option("job", "default_VMNetwork"): default_VMNetwork = config_file.get("job", "default_VMNetwork") if config_file.has_option("job", "default_VMCPUArch"): default_VMCPUArch = config_file.get("job", "default_VMCPUArch") if config_file.has_option("job", "default_VMName"): default_VMName = config_file.get("job", "default_VMName") if config_file.has_option("job", "default_VMLoc"): default_VMLoc = config_file.get("job", "default_VMLoc") if config_file.has_option("job", "default_VMAMI"): default_VMAMI = config_file.get("job", "default_VMAMI") if config_file.has_option("job", "default_VMMem"): try: default_VMMem = config_file.getint("job", "default_VMMem") except ValueError: print "Configuration file problem: default_VMMem must be an " \ "integer value." sys.exit(1) if config_file.has_option("job", "default_VMCPUCores"): try: default_VMCPUCores = config_file.getint("job", "default_VMCPUCores") except ValueError: print "Configuration file problem: default_VMCPUCores must be an " \ "integer value." sys.exit(1) if config_file.has_option("job", "default_VMStorage"): try: default_VMStorage = config_file.getint("job", "default_VMStorage") except ValueError: print "Configuration file problem: default_VMStorage must be an " \ "integer value." sys.exit(1) if config_file.has_option("job", "default_VMInstanceType"): default_VMInstanceType = config_file.get("job", "default_VMInstanceType") if config_file.has_option("job", "default_VMMaximumPrice"): try: default_VMMaximumPrice = config_file.getint("job", "default_VMMaximumPrice") except ValueError: print "Configuration file problem: default_VMMaximumPrice must be an " \ "integer value." sys.exit(1) # Derived options if condor_host_on_vm: condor_host = condor_host_on_vm else: condor_host = utilities.get_hostname_from_url(condor_webservice_url) if config_file.has_option("global", "use_pyopenssl"): use_pyopenssl = config_file.getboolean("global", "use_pyopenssl")
def setup(path=None): """Setup cloudscheduler using config file. setup will look for a configuration file specified on the command line, or in ~/.cloudscheduler.conf or /etc/cloudscheduler.conf """ global condor_webservice_url global condor_collector_url global condor_retrieval_method global condor_q_command global condor_status_command global condor_status_master_command global condor_hold_command global condor_release_command global condor_off_command global condor_on_command global condor_advertise_command global ssh_path global openssl_path global condor_context_file global condor_host global condor_host_on_vm global vm_lifetime global cert_file global key_file global cert_file_on_vm global key_file_on_vm global ca_root_certs global ca_signing_policies global cloudscheduler_ssh_key global cloud_resource_config global info_server_port global admin_server_port global persistence_file global user_limit_file global target_cloud_alias_file global job_ban_timeout global ban_tracking global ban_file global ban_min_track global ban_failrate_threshold global polling_error_threshold global condor_register_time_limit global graceful_shutdown global graceful_shutdown_method global retire_before_lifetime global retire_before_lifetime_factor global cleanup_missing_vms global clean_shutdown_idle global getclouds global scheduling_metric global scheduling_algorithm global job_distribution_type global high_priority_job_support global high_priority_job_weight global cpu_distribution_weight global memory_distribution_weight global storage_distribution_weight global cleanup_interval global vm_poller_interval global job_poller_interval global machine_poller_interval global scheduler_interval global job_proxy_refresher_interval global job_proxy_renewal_threshold global vm_proxy_refresher_interval global vm_proxy_renewal_threshold global vm_proxy_shutdown_threshold global vm_connection_fail_threshold global vm_start_running_timeout global vm_idle_threshold global max_starting_vm global max_keepalive global proxy_cache_dir global myproxy_logon_command global override_vmtype global vm_reqs_from_condor_reqs global adjust_insufficient_resources global use_cloud_init global default_yaml global validate_yaml global retire_reallocate global default_VMType global default_VMNetwork global default_VMName global default_VMLoc global default_VMAMI global default_VMMem global default_VMCPUCores global default_VMStorage global default_VMInstanceType global default_VMInstanceTypeList global default_VMMaximumPrice global default_VMProxyNonBoot global default_VMUserData global default_TargetClouds global default_VMAMIConfig global default_VMInjectCA global default_VMJobPerCore global log_level global log_location global log_location_cloud_admin global admin_log_comments global log_stdout global log_syslog global log_max_size global log_format global monitor_url global use_pyopenssl homedir = os.path.expanduser('~') # Find config file if not path: if os.path.exists(homedir + "/.cloudscheduler/cloud_scheduler.conf"): path = homedir + "/.cloudscheduler/cloud_scheduler.conf" elif os.path.exists("/etc/cloudscheduler/cloud_scheduler.conf"): path = "/etc/cloudscheduler/cloud_scheduler.conf" elif os.path.exists("/usr/local/share/cloud-scheduler/cloud_scheduler.conf"): path = "/usr/local/share/cloud-scheduler/cloud_scheduler.conf" else: #print >> sys.stderr, "Configuration file problem: There doesn't " \ # "seem to be a configuration file. " \ # "You can specify one with the --config-file parameter, " \ # "or put one in ~/.cloudscheduler/cloud_scheduler.conf or "\ # "/etc/cloudscheduler/cloud_scheduler.conf "\ # "Running in full default value mode." return # Read config file config_file = ConfigParser.ConfigParser() try: config_file.read(path) except IOError: print >> sys.stderr, "Configuration file problem: There was a " \ "problem reading %s. Check that it is readable," \ "and that it exists. " % path raise except ConfigParser.ParsingError: print >> sys.stderr, "Configuration file problem: Couldn't " \ "parse your file. Check for spaces before or after variables." raise except: print "Configuration file problem: There is something wrong with " \ "your config file." raise if config_file.has_option("global", "condor_retrieval_method"): condor_retrieval_method = config_file.get("global", "condor_retrieval_method") if config_file.has_option("global", "condor_q_command"): condor_q_command = config_file.get("global", "condor_q_command") if config_file.has_option("global", "condor_off_command"): condor_off_command = config_file.get("global", "condor_off_command") if config_file.has_option("global", "condor_on_command"): condor_on_command = config_file.get("global", "condor_on_command") if config_file.has_option("global", "ssh_path"): ssh_path = config_file.get("global", "ssh_path") if config_file.has_option("global", "openssl_path"): openssl_path = config_file.get("global", "openssl_path") if config_file.has_option("global", "condor_status_command"): condor_status_command = config_file.get("global", "condor_status_command") if config_file.has_option("global", "condor_status_master_command"): condor_status_master_command = config_file.get("global", "condor_status_master_command") if config_file.has_option("global", "condor_hold_command"): condor_hold_command = config_file.get("global", "condor_hold_command") if config_file.has_option("global", "condor_release_command"): condor_release_command = config_file.get("global", "condor_release_command") if config_file.has_option("global", "condor_advertise_command"): condor_advertise_command = config_file.get("global", "condor_advertise_command") if config_file.has_option("global", "condor_webservice_url"): condor_webservice_url = config_file.get("global", "condor_webservice_url") if config_file.has_option("global", "condor_collector_url"): condor_collector_url = config_file.get("global", "condor_collector_url") if config_file.has_option("global", "condor_host_on_vm"): condor_host_on_vm = config_file.get("global", "condor_host_on_vm") if config_file.has_option("global", "condor_context_file"): condor_context_file = config_file.get("global", "condor_context_file") if config_file.has_option("global", "vm_lifetime"): try: vm_lifetime = config_file.getint("global", "vm_lifetime") except ValueError: print "Configuration file problem: vm_lifetime must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "cert_file"): cert_file = config_file.get("global", "cert_file") if config_file.has_option("global", "key_file"): key_file = config_file.get("global", "key_file") if config_file.has_option("global", "cert_file_on_vm"): cert_file_on_vm = config_file.get("global", "cert_file_on_vm") if config_file.has_option("global", "key_file_on_vm"): key_file_on_vm = config_file.get("global", "key_file_on_vm") if config_file.has_option("global", "ca_root_certs"): ca_root_certs = config_file.get("global", "ca_root_certs").split(',') if config_file.has_option("global", "ca_signing_policies"): ca_signing_policies = config_file.get("global", "ca_signing_policies").split(',') if config_file.has_option("global", "cloudscheduler_ssh_key"): cloudscheduler_ssh_key = config_file.get("global", "cloudscheduler_ssh_key") if config_file.has_option("global", "cloud_resource_config"): cloud_resource_config = config_file.get("global", "cloud_resource_config") if config_file.has_option("global", "info_server_port"): try: info_server_port = config_file.getint("global", "info_server_port") except ValueError: print "Configuration file problem: info_server_port must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "admin_server_port"): try: info_server_port = config_file.getint("global", "admin_server_port") except ValueError: print "Configuration file problem: admin_server_port must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "persistence_file"): persistence_file = config_file.get("global", "persistence_file") if config_file.has_option("global", "user_limit_file"): user_limit_file = config_file.get("global", "user_limit_file") if config_file.has_option("global", "target_cloud_alias_file"): target_cloud_alias_file = config_file.get("global", "target_cloud_alias_file") if config_file.has_option("global", "job_ban_timeout"): try: job_ban_timeout = 60 * config_file.getint("global", "job_ban_timeout") except ValueError: print "Configuration file problem: job_ban_timeout must be an " \ "integer value in minutes." sys.exit(1) if config_file.has_option("global", "ban_file"): ban_file = config_file.get("global", "ban_file") if config_file.has_option("global", "polling_error_threshold"): try: polling_error_threshold = config_file.getint("global", "polling_error_threshold") except ValueError: print "Configuration file problem: polling_error_threshold must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "ban_failrate_threshold"): try: ban_failrate_threshold = config_file.getfloat("global", "ban_failrate_threshold") if ban_failrate_threshold == 0: print "Please use a float value (0, 1.0]" sys.exit(1) except ValueError: print "Configuration file problem: ban_failrate_threshold must be an " \ "float value." sys.exit(1) if config_file.has_option("global", "ban_min_track"): try: ban_min_track = config_file.getint("global", "ban_min_track") except ValueError: print "Configuration file problem: ban_min_track must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "condor_register_time_limit"): try: condor_register_time_limit = 60*config_file.getint("global", "condor_register_time_limit") except ValueError: print "Configuration file problem: condor_register_time_limit must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "ban_tracking"): try: ban_tracking = config_file.getboolean("global", "ban_tracking") except ValueError: print "Configuration file problem: ban_tracking must be an " \ "boolean value." sys.exit(1) if config_file.has_option("global", "graceful_shutdown"): try: graceful_shutdown = config_file.getboolean("global", "graceful_shutdown") except ValueError: print "Configuration file problem: graceful_shutdown must be an " \ "boolean value." sys.exit(1) if config_file.has_option("global", "graceful_shutdown_method"): graceful_shutdown_method = config_file.get("global", "graceful_shutdown_method") if config_file.has_option("global", "retire_before_lifetime"): try: retire_before_lifetime = config_file.getboolean("global", "retire_before_lifetime") except ValueError: print "Configuration file problem: retire_before_lifetime must be an " \ "boolean value." sys.exit(1) if config_file.has_option("global", "retire_before_lifetime_factor"): try: retire_before_lifetime_factor = config_file.getfloat("global", "retire_before_lifetime_factor") if retire_before_lifetime_factor < 1.0: print "Please use a float value (1.0, X] for the retire_before_lifetime_factor" sys.exit(1) except ValueError: print "Configuration file problem: retire_before_lifetime_factor must be a " \ "float value." sys.exit(1) if config_file.has_option("global", "cleanup_missing_vms"): try: cleanup_missing_vms = config_file.getboolean("global", "cleanup_missing_vms") except ValueError: print "Configuration file problem: cleanup_missing_vms must be an " \ "boolean value." sys.exit(1) if config_file.has_option("global", "clean_shutdown_idle"): try: clean_shutdown_idle = config_file.getboolean("global", "clean_shutdown_idle") except ValueError: print "Configuration file problem: clean_shutdown_idle must be an " \ "boolean value." sys.exit(1) if config_file.has_option("global", "getclouds"): try: getclouds = config_file.getboolean("global", "getclouds") except ValueError: print "Configuration file problem: getclouds must be an " \ "boolean value." sys.exit(1) if config_file.has_option("global", "scheduling_metric"): scheduling_metric = config_file.get("global", "scheduling_metric") if config_file.has_option("global", "job_distribution_type"): job_distribution_type = config_file.get("global", "job_distribution_type") if config_file.has_option("global", "memory_distribution_weight"): try: memory_distribution_weight = config_file.getfloat("global", "memory_distribution_weight") if ban_failrate_threshold <= 0: print "Please use a float value (0, x]" sys.exit(1) except ValueError: print "Configuration file problem: memory_distribution_weight must be an " \ "float value." sys.exit(1) if config_file.has_option("global", "cpu_distribution_weight"): try: cpu_distribution_weight = config_file.getfloat("global", "cpu_distribution_weight") if ban_failrate_threshold <= 0: print "Please use a float value (0, x]" sys.exit(1) except ValueError: print "Configuration file problem: cpu_distribution_weight must be an " \ "float value." sys.exit(1) if config_file.has_option("global", "storage_distribution_weight"): try: storage_distribution_weight = config_file.getfloat("global", "storage_distribution_weight") if ban_failrate_threshold <= 0: print "Please use a float value (0, x]" sys.exit(1) except ValueError: print "Configuration file problem: storage_distribution_weight must be an " \ "float value." sys.exit(1) if config_file.has_option("global", "scheduling_algorithm"): scheduling_algorithm = config_file.get("global", "scheduling_algorithm") if config_file.has_option("global", "high_priority_job_support"): try: high_priority_job_support = config_file.getboolean("global", "high_priority_job_support") except ValueError: print "Configuration file problem: high_priority_job_support must be an " \ "boolean value." sys.exit(1) if config_file.has_option("global", "high_priority_job_weight"): try: high_priority_job_weight = config_file.getint("global", "high_priority_job_weight") except ValueError: print "Configuration file problem: high_priority_job_weight must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "scheduler_interval"): try: scheduler_interval = config_file.getint("global", "scheduler_interval") except ValueError: print "Configuration file problem: scheduler_interval must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "vm_poller_interval"): try: vm_poller_interval = config_file.getint("global", "vm_poller_interval") except ValueError: print "Configuration file problem: vm_poller_interval must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "job_poller_interval"): try: job_poller_interval = config_file.getint("global", "job_poller_interval") except ValueError: print "Configuration file problem: job_poller_interval must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "machine_poller_interval"): try: machine_poller_interval = config_file.getint("global", "machine_poller_interval") except ValueError: print "Configuration file problem: machine_poller_interval must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "cleanup_interval"): try: cleanup_interval = config_file.getint("global", "cleanup_interval") except ValueError: print "Configuration file problem: cleanup_interval must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "job_proxy_refresher_interval"): try: job_proxy_refresher_interval = config_file.getint("global", "job_proxy_refresher_interval") except ValueError: print "Configuration file problem: job_proxy_refresher_interval must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "job_proxy_renewal_threshold"): try: job_proxy_renewal_threshold = config_file.getint("global", "job_proxy_renewal_threshold") except ValueError: print "Configuration file problem: job_proxy_renewal_threshold must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "vm_proxy_refresher_interval"): try: vm_proxy_refresher_interval = config_file.getint("global", "vm_proxy_refresher_interval") except ValueError: print "Configuration file problem: vm_proxy_refresher_interval must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "vm_proxy_renewal_threshold"): try: vm_proxy_renewal_threshold = config_file.getint("global", "vm_proxy_renewal_threshold") except ValueError: print "Configuration file problem: vm_proxy_renewal_threshold must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "vm_proxy_shutdown_threshold"): try: vm_proxy_shutdown_threshold = config_file.getint("global", "vm_proxy_shutdown_threshold") except ValueError: print "Configuration file problem: vm_proxy_shutdown_threshold must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "vm_connection_fail_threshold"): try: vm_connection_fail_threshold = config_file.getint("global", "vm_connection_fail_threshold") except ValueError: print "Configuration file problem: vm_connection_fail_threshold must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "vm_idle_threshold"): try: vm_idle_threshold = config_file.getint("global", "vm_idle_threshold") except ValueError: print "Configuration file problem: vm_idle_threshold must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "vm_start_running_timeout"): try: vm_start_running_timeout = config_file.getint("global", "vm_start_running_timeout") except ValueError: print "Configuration file problem: vm_start_running_timeout must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "max_starting_vm"): try: max_starting_vm = config_file.getint("global", "max_starting_vm") if max_starting_vm < -1: max_starting_vm = -1 except ValueError: print "Configuration file problem: max_starting_vm must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "max_keepalive"): try: max_keepalive = config_file.getint("global", "max_keepalive") * 60 if max_keepalive < 0: max_keepalive = 0 except ValueError: print "Configuration file problem: max_keepalive must be an " \ "integer value(# of minutes)." sys.exit(1) if config_file.has_option("global", "max_destroy_threads"): try: max_destroy_threads = config_file.getint("global", "max_destroy_threads") if max_destroy_threads <= 0: max_destroy_threads = 1 except ValueError: print "Configuration file problem: max_destroy_threads must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "proxy_cache_dir"): proxy_cache_dir = config_file.get("global", "proxy_cache_dir") if config_file.has_option("global", "myproxy_logon_command"): myproxy_logon_command = config_file.get("global", "myproxy_logon_command") if config_file.has_option("global", "override_vmtype"): try: override_vmtype = config_file.getboolean("global", "override_vmtype") except ValueError: print "Configuration file problem: override_vmtype must be a" \ " Boolean value." if config_file.has_option("global", "vm_reqs_from_condor_reqs"): try: vm_reqs_from_condor_reqs = config_file.getboolean("global", "vm_reqs_from_condor_reqs") except ValueError: print "Configuration file problem: vm_reqs_from_condor_reqs must be a" \ " Boolean value." if config_file.has_option("global", "adjust_insufficient_resources"): try: adjust_insufficient_resources = config_file.getboolean("global", "adjust_insufficient_resources") except ValueError: print "Configuration file problem: adjust_insufficient_resources must be a" \ " Boolean value." if config_file.has_option("global", "connection_fail_disable_time"): try: connection_fail_disable_time = config_file.getint("global", "connection_fail_disable_time") except ValueError: print "Configuration file problem: connection_fail_disable_time must be an " \ "integer value." sys.exit(1) if config_file.has_option("global", "use_cloud_init"): try: use_cloud_init = config_file.getboolean("global", "use_cloud_init") except ValueError: print "Configuration file problem: use_cloud_init must be a" \ " Boolean value." if config_file.has_option("global", "default_yaml"): default_yaml = config_file.get("global", "default_yaml") if config_file.has_option("global", "validate_yaml"): try: validate_yaml = config_file.getboolean("global", "validate_yaml") except ValueError: print "Configuration file problem: validate_yaml must be a" \ " Boolean value." if config_file.has_option("global", "retire_reallocate"): try: retire_reallocate = config_file.getboolean("global", "retire_reallocate") except ValueError: print "Configuration file problem: retire_reallocate must be a" \ " Boolean value." # Default Logging options if config_file.has_option("logging", "log_level"): log_level = config_file.get("logging", "log_level") if config_file.has_option("logging", "log_location"): log_location = os.path.expanduser(config_file.get("logging", "log_location")) if config_file.has_option("logging", "log_location_cloud_admin"): log_location_cloud_admin = os.path.expanduser(config_file.get("logging", "log_location_cloud_admin")) if config_file.has_option("logging", "admin_log_comments"): try: admin_log_comments = config_file.getboolean("logging", "admin_log_comments") except ValueError: print "Configuration file problem: admin_log_comments must be a" \ " Boolean value." if config_file.has_option("logging", "log_stdout"): try: log_stdout = config_file.getboolean("logging", "log_stdout") except ValueError: print "Configuration file problem: log_stdout must be a" \ " Boolean value." if config_file.has_option("logging", "log_syslog"): try: log_syslog = config_file.getboolean("logging", "log_syslog") except ValueError: print "Configuration file problem: log_syslog must be a" \ " Boolean value." if config_file.has_option("logging", "log_max_size"): try: log_max_size = config_file.getint("logging", "log_max_size") except ValueError: print "Configuration file problem: log_max_size must be an " \ "integer value in bytes." sys.exit(1) if config_file.has_option("logging", "log_format"): log_format = config_file.get("logging", "log_format", raw=True) # Default Job options if config_file.has_option("job", "default_VMType"): default_VMType = config_file.get("job", "default_VMType") if config_file.has_option("job", "default_VMNetwork"): default_VMNetwork = config_file.get("job", "default_VMNetwork") if config_file.has_option("job", "default_VMName"): default_VMName = config_file.get("job", "default_VMName") if config_file.has_option("job", "default_VMAMI"): default_VMAMI = config_file.get("job", "default_VMAMI") if config_file.has_option("job", "default_VMMem"): try: default_VMMem = config_file.getint("job", "default_VMMem") except ValueError: print "Configuration file problem: default_VMMem must be an " \ "integer value." sys.exit(1) if config_file.has_option("job", "default_VMCPUCores"): try: default_VMCPUCores = config_file.getint("job", "default_VMCPUCores") except ValueError: print "Configuration file problem: default_VMCPUCores must be an " \ "integer value." sys.exit(1) if config_file.has_option("job", "default_VMStorage"): try: default_VMStorage = config_file.getint("job", "default_VMStorage") except ValueError: print "Configuration file problem: default_VMStorage must be an " \ "integer value." sys.exit(1) if config_file.has_option("job", "default_VMInstanceType"): default_VMInstanceType = config_file.get("job", "default_VMInstanceType") if config_file.has_option("job", "default_VMInstanceTypeList"): default_VMInstanceTypeList = config_file.get("job", "default_VMInstanceTypeList") if config_file.has_option("job", "default_VMMaximumPrice"): try: default_VMMaximumPrice = config_file.getfloat("job", "default_VMMaximumPrice") except ValueError: print "Configuration file problem: default_VMMaximumPrice must be an " \ "floating point value." sys.exit(1) if config_file.has_option("job", "default_VMProxyNonBoot"): try: default_VMProxyNonBoot = config_file.getboolean("global", "default_VMProxyNonBoot") except ValueError: print "Configuration file problem: default_VMProxyNonBoot must be a" \ " Boolean value." if config_file.has_option("job", "default_VMUserData"): default_VMUserData = config_file.get("job", "default_VMUserData").replace(' ', '').strip('"').split(',') if config_file.has_option("job", "default_TargetClouds"): default_TargetClouds = config_file.get("job", "default_TargetClouds") if config_file.has_option("job", "default_VMAMIConfig"): default_VMAMIConfig = config_file.get("job", "default_VMAMIConfig") if config_file.has_option("job", "default_VMInjectCA"): try: default_VMInjectCA = config_file.getboolean("job", "default_VMInjectCA") except ValueError: print "Configuration file problem: default_VMInjectCA must be a" \ " Boolean value." if config_file.has_option("job", "default_VMJobPerCore"): try: default_VMJobPerCore = config_file.getboolean("job", "default_VMJobPerCore") except ValueError: print "Configuration file problem: default_VMJobPerCore must be a" \ " Boolean value." # Derived options if condor_host_on_vm: condor_host = condor_host_on_vm else: condor_host = utilities.get_hostname_from_url(condor_webservice_url) if config_file.has_option("global", "use_pyopenssl"): use_pyopenssl = config_file.getboolean("global", "use_pyopenssl") if config_file.has_option("global", "monitor_url"): monitor_url = config_file.get("global", "monitor_url")