def init_defaults(self):
"""
Populates the defaults for all the factory configuration values.
"""
self.init_support_defaults()
# Defaults for allowing frontends in a whitelist
# in the factory config (per entry point)
self.allow_defaults = cWParams.commentedOrderedDict()
self.allow_defaults["name"] = (None, "string", "frontend name", None)
self.allow_defaults["security_class"] = ("All", "string", "security class", None)
self.max_job_frontend_defaults=cWParams.commentedOrderedDict()
self.max_job_frontend_defaults["name"]=(None,"string","frontend name",None)
self.max_job_frontend_defaults["security_class"]=("All","string","security class",None)
self.max_job_frontend_defaults["held"]=('1000',"nr","Maximum number of held glideins (for this frontend) before forcing the cleanup.",None)
self.max_job_frontend_defaults["idle"]=('2000',"nr","Maximum number of idle glideins (for this frontend) allowed.",None)
self.max_job_frontend_defaults["running"]=('10000',"nr","Maximum number of concurrent glideins (per frontend) that can be submitted",None)
# publishing specific to factory
self.attr_defaults["publish"] = ("True", "Bool", "Should it be published by the factory?", None)
self.attr_defaults["const"] = ("True", "Bool", "Should it be constant? (Else it can be overriden by the frontend. Used only if parameter is True.)", None)
self.infosys_defaults = cWParams.commentedOrderedDict()
self.infosys_defaults["type"] = (None, "RESS|BDII", "Type of information system", None)
self.infosys_defaults["server"] = (None, "host", "Location of the infosys server", None)
self.infosys_defaults["ref"] = (None, "id", "Referenced for the entry point in the infosys", None)
self.mongroup_defaults = cWParams.commentedOrderedDict()
self.mongroup_defaults["group_name"] = (None, "groupname", "Name of the monitoring group", None)
entry_config_defaults = cWParams.commentedOrderedDict()
entry_config_max_jobs_defaults=cWParams.commentedOrderedDict()
entry_config_max_jobs_defaults["running"]=('10000',"nr","Maximum number of concurrent glideins (per frontend) that can be submitted.",None)
entry_config_max_jobs_defaults["idle"]=('2000',"nr","Maximum number of idle glideins (per frontend) allowed.",None)
entry_config_max_jobs_defaults["held"]=('1000',"nr","Maximum number of held glideins (per frontend) before forcing the cleanup.",None)
entry_config_defaults['max_jobs']=entry_config_max_jobs_defaults
entry_config_max_jobs_defaults=cWParams.commentedOrderedDict()
entry_config_max_jobs_defaults["running"]=('10000',"nr","Maximum number of concurrent glideins (per frontend) that can be submitted.",None)
entry_config_max_jobs_defaults["idle"]=('2000',"nr","Maximum number of idle glideins (per frontend) allowed.",None)
entry_config_max_jobs_defaults["held"]=('1000',"nr","Maximum number of held glideins (per frontend) before forcing the cleanup.",None)
entry_config_max_jobs_defaults["max_job_frontends"]=(xmlParse.OrderedDict(),'Dictionary of frontends',"Each frontend entry contains",self.max_job_frontend_defaults)
entry_config_defaults['max_jobs']=entry_config_max_jobs_defaults
entry_config_max_jobs_defaults = cWParams.commentedOrderedDict()
entry_config_max_jobs_defaults["running"] = ('10000', "nr", "Maximum number of concurrent glideins (per frontend) that can be submitted.", None)
entry_config_max_jobs_defaults["idle"] = ('2000', "nr", "Maximum number of idle glideins (per frontend) allowed.", None)
entry_config_max_jobs_defaults["held"] = ('1000', "nr", "Maximum number of held glideins (per frontend) before forcing the cleanup.", None)
entry_config_defaults['max_jobs'] = entry_config_max_jobs_defaults
entry_config_restrictions_defaults=cWParams.commentedOrderedDict()
entry_config_restrictions_defaults["require_voms_proxy"]=("False","Bool","Whether this entry point requires a voms proxy",None)
entry_config_defaults['restrictions']=entry_config_restrictions_defaults
entry_config_queue_defaults = cWParams.commentedOrderedDict()
entry_config_queue_defaults["max_per_cycle"] = ['100', "nr", "Maximum number of jobs affected per cycle.", None]
entry_config_queue_defaults["sleep"] = ['0.2', "seconds", "Sleep between interactions with the schedd.", None]
entry_config_defaults['submit'] = copy.deepcopy(entry_config_queue_defaults)
entry_config_defaults['submit']['cluster_size'] = ['10', "nr", "Max number of jobs submitted in a single transaction.", None]
entry_config_defaults['remove'] = copy.deepcopy(entry_config_queue_defaults)
entry_config_defaults['remove']['max_per_cycle'][0] = '5'
entry_config_defaults['release'] = copy.deepcopy(entry_config_queue_defaults)
entry_config_defaults['release']['max_per_cycle'][0] = '20'
# not exported and order does not matter, can stay a regular dictionary
sub_defaults = {'attrs':(xmlParse.OrderedDict(), 'Dictionary of attributes', "Each attribute entry contains", self.attr_defaults),
'files':([], 'List of files', "Each file entry contains", self.file_defaults),
'infosys_refs':([], 'List of information system references', "Each reference points to this entry", self.infosys_defaults),
'monitorgroups':([], 'List of monitoring groups', "Each group entry belongs to", self.mongroup_defaults)}
self.entry_defaults = cWParams.commentedOrderedDict()
self.entry_defaults["gatekeeper"] = (None, 'gatekeeper', 'Grid gatekeeper/resource', None)
self.entry_defaults["gridtype"] = ('gt2', 'grid_type', 'Condor Grid type', None)
self.entry_defaults["trust_domain"] = ('OSG', 'trust_domain', 'Entry trust domain', None)
self.entry_defaults["auth_method"] = ('grid_proxy', 'auth_method', 'Type of auth method this entry supports', None)
self.entry_defaults["vm_id"] = (None, 'vm_id', 'VM id this entry supports', None)
self.entry_defaults["vm_type"] = (None, 'vm_type', 'VM type this entry supports', None)
self.entry_defaults["rsl"] = (None, 'RSL', 'Globus gt2 RSL option', None)
self.entry_defaults['schedd_name'] = (None, "ScheddName", "Which schedd to use (Overrides the global one if specified)", None)
self.entry_defaults["work_dir"] = (".", ".|Condor|OSG|TMPDIR", "Where to start glidein", None)
self.entry_defaults['proxy_url'] = (None, 'proxy_url', "Squid cache to use", None)
self.entry_defaults['verbosity'] = ('std', 'std|nodebug|fast', "Verbosity level and timeout setting", None)
self.entry_defaults["enabled"] = ("True", "Bool", "Is this entry enabled?", None)
self.entry_defaults["config"] = entry_config_defaults
self.entry_defaults["attrs"] = sub_defaults['attrs']
self.entry_defaults["files"] = sub_defaults['files']
self.entry_defaults["infosys_refs"] = sub_defaults['infosys_refs']
self.entry_defaults["downtimes"] = self.downtimes_defaults
self.entry_defaults["monitorgroups"] = copy.deepcopy(sub_defaults['monitorgroups'])
self.entry_defaults["allow_frontends"] = (xmlParse.OrderedDict(), 'Dictionary of frontends', "Each frontend entry contains", self.allow_defaults);
###############################
# Start defining the defaults
self.defaults["factory_name"] = (socket.gethostname(), 'ID', 'Factory name', None)
self.defaults["glidein_name"] = (None, 'ID', 'Glidein name', None)
self.defaults['schedd_name'] = ("schedd_glideins@%s" % socket.gethostname(), "ScheddName", "Which schedd to use, can be a comma separated list", None)
submit_defaults = cWParams.commentedOrderedDict()
submit_defaults["base_dir"] = ("%s/glideinsubmit" % os.environ["HOME"], "base_dir", "Submit base dir", None)
submit_defaults["base_log_dir"] = ("%s/glideinlog" % os.environ["HOME"], "log_dir", "Submit base log dir", None)
submit_defaults["base_client_log_dir"] = ("%s/glideclientlog" % os.environ["HOME"], "client_dir", "Base dir for client logs, needs a user_<uid> subdir per frontend user", None)
submit_defaults["base_client_proxies_dir"] = ("%s/glideclientproxies" % os.environ["HOME"], "client_dir", "Base dir for client proxies, needs a user_<uid> subdir per frontend user", None)
self.defaults["submit"] = submit_defaults
one_log_retention_defaults = cWParams.commentedOrderedDict()
one_log_retention_defaults["min_days"] = ["3.0", "days", "Min number of days the logs must be preserved (even if they use too much space)", None]
one_log_retention_defaults["max_days"] = ["7.0", "days", "Max number of days the logs should be preserved", None]
one_log_retention_defaults["max_mbytes"] = ["100.0", "Mbytes", "Max number of Mbytes the logs can use", None]
log_retention_defaults = cWParams.commentedOrderedDict()
log_retention_defaults["logs"] = copy.deepcopy(one_log_retention_defaults)
log_retention_defaults["job_logs"] = copy.deepcopy(one_log_retention_defaults)
log_retention_defaults["job_logs"]["min_days"][0] = "2.0"
self.defaults['advertise_with_tcp'] = ('False', 'Bool', 'Should condor_advertise use TCP connections?', None)
self.defaults['advertise_with_multiple'] = ('False', 'Bool', 'Should condor_advertise use -multiple?', None)
log_retention_defaults["summary_logs"] = copy.deepcopy(one_log_retention_defaults)
log_retention_defaults["summary_logs"]["max_days"][0] = "31.0"
log_retention_defaults["condor_logs"] = copy.deepcopy(one_log_retention_defaults)
log_retention_defaults["condor_logs"]["max_days"][0] = "14.0"
self.defaults["log_retention"] = log_retention_defaults
self.defaults['loop_delay'] = ('60', 'seconds', 'Number of seconds between iterations', None)
self.defaults['advertise_delay'] = ('5', 'NR', 'Advertize evert NR loops', None)
self.defaults['restart_attempts'] = ('3', 'NR', 'Max allowed NR restarts every restart_interval before shutting down', None)
self.defaults['restart_interval'] = ('1800', 'NR', 'Time interval NR sec which allow max restart attempts', None)
stage_defaults = cWParams.commentedOrderedDict()
stage_defaults["base_dir"] = ("/var/www/html/glidefactory/stage", "base_dir", "Stage base dir", None)
stage_defaults["web_base_url"] = ("http://%s/glidefactory/stage" % socket.gethostname(), 'base_url', 'Base Web server URL', None)
stage_defaults["use_symlink"] = ("True", "Bool", "Can I symlink stage dir from submit dir?", None)
self.defaults["stage"] = stage_defaults
self.monitor_defaults["base_dir"] = ("/var/www/html/glidefactory/monitor", "base_dir", "Monitoring base dir", None)
self.defaults["monitor"] = self.monitor_defaults
self.frontend_sec_class_defaults = cWParams.commentedOrderedDict()
self.frontend_sec_class_defaults["username"] = (None, 'username', 'UNIX ID to be used for this security class', None)
self.frontend_defaults = cWParams.commentedOrderedDict()
self.frontend_defaults["identity"] = (None, 'identity', 'Authenticated Identity', None)
self.frontend_defaults["security_classes"] = (xmlParse.OrderedDict(), "Dictionary of security class maps", "Each mapping contains", self.frontend_sec_class_defaults)
security_default=cWParams.commentedOrderedDict()
security_default["pub_key"]=("RSA","None|RSA","Type of public key system used for secure message passing",None)
security_default["reuse_oldkey_onstartup_gracetime"]=("900","seconds","Time in sec old key can be used to decrypt requests from frontend",None)
security_default["key_length"]=("2048","bits","Key length in bits",None)
security_default["frontends"]=(xmlParse.OrderedDict(),"Dictionary of frontend","Each frontend contains",self.frontend_defaults)
self.defaults["security"] = security_default
condor_defaults = cWParams.commentedOrderedDict()
condor_defaults["os"] = ("default", "osname", "Operating System (like linux-rhel3)", None)
condor_defaults["arch"] = ("default", "arch", "Architecture (like x86)", None)
condor_defaults["version"] = ("default", "arch", "Architecture (like x86)", None)
condor_defaults["tar_file"] = (None, "fname", "Tarball containing condor binaries (overrides base_dir if defined)", None)
condor_defaults["base_dir"] = (None, "base_dir", "Condor distribution base dir (used only if tar_file undefined)", None)
self.defaults["condor_tarballs"] = ([], 'List of condor tarballs', "Each entry contains", condor_defaults)
self.defaults["downtimes"] = self.downtimes_defaults
self.defaults["attrs"] = sub_defaults['attrs']
self.defaults["files"] = copy.deepcopy(sub_defaults['files'])
# ordering is specific to global section of factory
self.defaults["files"][3]["after_entry"] = ("False", 'Bool', 'Should this file be loaded after the entry ones?', None)
self.defaults["entries"] = (xmlParse.OrderedDict(), "Dictionary of entries", "Each entry contains", self.entry_defaults)
return
def init_defaults(self):
self.init_support_defaults()
# VO scripts should start after the factory has been set completely up
# but there could be exceptions
self.file_defaults["after_entry"]=("True",'Bool','Should this file be loaded after the factory entry ones?',None)
# publishing specific to frontend
self.attr_defaults["type"]=["string","string|int|expr","What kind on data is value. (if expr, a python expression with access to frontend and glidein dictionaries)",None]
group_config_defaults=cWParams.commentedOrderedDict()
group_config_running_defaults=cWParams.commentedOrderedDict()
group_config_running_defaults["max"]=['10000',"nr_jobs","What is the max number of running glideins I want to get to",None]
group_config_running_defaults["relative_to_queue"]=['1.15',"fraction","Max relative to number of matching jobs in the queue.",None]
group_config_defaults['running_glideins_per_entry']=group_config_running_defaults
group_config_running_total_defaults=cWParams.commentedOrderedDict()
group_config_running_total_defaults["max"]=['100000',"nr_jobs","What is the max number of running glideins I want to get to - globally",None]
group_config_running_total_defaults["curb"]=['90000',"nr_jobs","When should I start curbing glidein submission",None]
group_config_defaults['running_glideins_total']=group_config_running_total_defaults
group_config_idle_defaults=cWParams.commentedOrderedDict()
group_config_idle_defaults["max"]=['100',"nr_jobs","How much pressure should I apply to the entry points",None]
group_config_idle_defaults["reserve"]=['5',"nr_jobs","How much to overcommit.",None]
group_config_defaults['idle_glideins_per_entry']=group_config_idle_defaults
group_config_vms_defaults=cWParams.commentedOrderedDict()
group_config_vms_defaults["max"]=['100',"nr_jobs","How many idle VMs should I tollerate, before stopping submitting glideins",None]
group_config_vms_defaults["curb"]=['5',"nr_jobs","How many idle VMs should I tollerate, before starting to curb submissions.",None]
group_config_defaults['idle_vms_per_entry']=group_config_vms_defaults
# not exported and order does not matter, can stay a regular dictionary
sub_defaults={'attrs':(xmlParse.OrderedDict(),'Dictionary of attributes',"Each attribute group contains",self.attr_defaults),
'files':([],'List of files',"Each file group contains",self.file_defaults)}
query_attrs_defaults=cWParams.commentedOrderedDict()
query_attrs_defaults['type']=('string','string|int|real|bool','Attribute type',None)
fj_match_defaults=cWParams.commentedOrderedDict()
fj_match_defaults["query_expr"]=['True','CondorExpr','Expression for selecting user jobs',None]
fj_match_defaults["match_attrs"]=(xmlParse.OrderedDict(),"Dictionary of ClassAd attributes","Each attribute contains",query_attrs_defaults)
collector_defaults=cWParams.commentedOrderedDict()
collector_defaults["node"]=(None,"nodename","Factory collector node name (for example, fg2.my.org:9999)",None)
collector_defaults["DN"]=(None,"dn","Factory collector distinguised name (subject) (for example, /DC=org/DC=myca/OU=Services/CN=fg2.my.org)",None)
collector_defaults["factory_identity"]=("*****@*****.**","authenticated_identity","What is the AuthenticatedIdentity of the factory at the WMS collector",None)
collector_defaults["my_identity"]=("*****@*****.**","authenticated_identity","What is the AuthenticatedIdentity of my proxy at the WMS collector",None)
factory_match_defaults=copy.deepcopy(fj_match_defaults)
factory_match_defaults["collectors"]=([],"List of factory collectors","Each collector contains",collector_defaults)
schedd_defaults=cWParams.commentedOrderedDict()
schedd_defaults["fullname"]=(None,"name","User schedd name (for example, [email protected])",None)
schedd_defaults["DN"]=(None,"dn","User schedd distinguised name (subject) (for example, /DC=org/DC=myca/OU=Services/CN=sb1.my.org)",None)
job_match_defaults=copy.deepcopy(fj_match_defaults)
job_match_defaults["schedds"]=([],"List of user schedds","Each schedd contains",schedd_defaults)
match_defaults=cWParams.commentedOrderedDict()
match_defaults["factory"]=factory_match_defaults
match_defaults["job"]=job_match_defaults
match_defaults["match_expr"]=('True','PythonExpr', 'Python expression for matching jobs to factory entries with access to job and glidein dictionaries',None)
match_defaults["start_expr"]=('True','CondorExpr', 'Condor expression for matching jobs to glideins at runtime',None)
proxy_defaults=cWParams.commentedOrderedDict()
proxy_defaults["absfname"]=(None,"fname","x509 proxy file name (see also pool_idx_list)",None)
proxy_defaults["keyabsfname"]=(None,"fname","for key files, file name of the key pair",None)
proxy_defaults["pilotabsfname"]=(None,"fname","to specify a different pilot proxy instead of using submit proxy",None)
proxy_defaults["type"]=("grid_proxy","proxy_type","Type of credential: grid_proxy,cert_pair,key_pair,username_password",None)
proxy_defaults["trust_domain"]=("OSG","grid_type","Trust Domain",None)
proxy_defaults["creation_script"]=(None,"command","Script to re-create credential",None)
proxy_defaults["update_frequency"]=(None,"int","Update proxy when there is this much time left",None)
proxy_defaults["vm_id"]=(None,"vm_id","VM Id",None)
proxy_defaults["vm_type"]=(None,"vm_type","VM Type",None)
proxy_defaults["pool_idx_len"]=(None,"boolean","Adds leading zeros to the suffix so all filenames the same length",None)
proxy_defaults["pool_idx_list"]=(None,"string","List of indices, can include ranges of indices",None)
proxy_defaults["security_class"]=(None,"id","Proxies in the same security class can potentially access each other (Default: proxy_nr)",None)
security_defaults=cWParams.commentedOrderedDict()
security_defaults["proxy_selection_plugin"]=(None,"proxy_name","Which proxy selection plugin should I use (ProxyAll if None)",None)
security_defaults["credentials"]=([],'List of credentials',"Each proxy element contains",proxy_defaults)
security_defaults["security_name"]=(None,"frontend_name","What name will we advertize for security purposes?",None)
self.group_defaults=cWParams.commentedOrderedDict()
self.group_defaults["match"]=match_defaults
self.group_defaults["enabled"]=("True","Bool","Is this group enabled?",None)
self.group_defaults["config"]=group_config_defaults
self.group_defaults["attrs"]=sub_defaults['attrs']
self.group_defaults["files"]=sub_defaults['files']
self.group_defaults["security"]=copy.deepcopy(security_defaults)
###############################
# Start defining the defaults
self.defaults["frontend_name"]=(socket.gethostname(),'ID', 'VO Frontend name',None)
self.defaults['frontend_versioning'] = ('True', 'Bool', 'Should we create versioned subdirectories of the type frontend_$frontend_name?', None)
self.defaults['frontend_monitor_index_page'] = ('True', 'Bool', 'Should we create an index.html in the monitoring web directory?',None)
work_defaults=cWParams.commentedOrderedDict()
work_defaults["base_dir"]=("%s/frontstage"%os.environ["HOME"],"base_dir","Frontend base dir",None)
work_defaults["base_log_dir"]=("%s/frontlogs"%os.environ["HOME"],"log_dir","Frontend base log dir",None)
self.defaults["work"]=work_defaults
process_log_defaults=cWParams.commentedOrderedDict()
process_log_defaults["min_days"] = ["3.0","days","Min number of days the logs must be preserved (even if they use too much space)",None]
process_log_defaults["max_days"] = ["7.0","days","Max number of days the logs should be preserved",None]
process_log_defaults["max_mbytes"] = ["100.0","Mbytes","Max number of Mbytes the logs can use",None]
process_log_defaults['extension'] = ["all", "string", "name of the log extention", None]
process_log_defaults['msg_types'] = ["INFO, WARN, ERR", "string", "types of log messages", None]
log_retention_defaults = cWParams.commentedOrderedDict()
log_retention_defaults["process_logs"] = ([], 'Dictionary of log types', "Each log corresponds to a log file", copy.deepcopy(process_log_defaults))
self.defaults["log_retention"] = log_retention_defaults
monitor_footer_defaults=cWParams.commentedOrderedDict()
monitor_footer_defaults["display_txt"] = ["", "string", "what will be displayed at the bottom of the monitoring page", None]
monitor_footer_defaults["href_link"] = ["", "string", "where to link to", None]
self.defaults["monitor_footer"] = monitor_footer_defaults
self.defaults['loop_delay']=('60','seconds', 'Number of seconds between iterations',None)
self.defaults['advertise_delay']=('5','NR', 'Advertize evert NR loops',None)
self.defaults['advertise_with_tcp']=('True','Bool', 'Should condor_advertise use TCP connections?',None)
self.defaults['advertise_with_multiple']=('True','Bool', 'Should condor_advertise use -multiple?',None)
self.defaults['restart_attempts']=('3','NR', 'Max allowed NR restarts every restart_interval before shutting down',None)
self.defaults['restart_interval']=('1800','NR', 'Time interval NR sec which allow max restart attempts',None)
stage_defaults=cWParams.commentedOrderedDict()
stage_defaults["base_dir"]=("/var/www/html/vofrontend/stage","base_dir","Stage base dir",None)
stage_defaults["web_base_url"]=("http://%s/vofrontend/stage"%socket.gethostname(),'base_url','Base Web server URL',None)
stage_defaults["use_symlink"]=("True","Bool","Can I symlink stage dir from work dir?",None)
self.defaults["stage"]=stage_defaults
self.monitor_defaults["base_dir"]=("/var/www/html/vofrontend/monitor","base_dir","Monitoring base dir",None)
self.monitor_defaults["web_base_url"]=(None,"web_base_url","Monitoring base dir",None)
self.defaults["monitor"]=self.monitor_defaults
pool_collector_defaults=cWParams.commentedOrderedDict()
pool_collector_defaults["node"]=(None,"nodename","Pool collector node name (for example, col1.my.org:9999)",None)
pool_collector_defaults["DN"]=(None,"dn","Factory collector distinguised name (subject) (for example, /DC=org/DC=myca/OU=Services/CN=col1.my.org)",None)
pool_collector_defaults["secondary"]=("False","Bool","Secondary nodes will be used by glideins, if present",None)
pool_collector_defaults["group"]=("default","string","Collector group name useful to group HA setup",None)
self.defaults["collectors"]=([],'List of pool collectors',"Each proxy collector contains",pool_collector_defaults)
self.defaults["security"]=copy.deepcopy(security_defaults)
self.defaults["security"]["classad_proxy"]=(None,"fname","File name of the proxy used for talking to the WMS collector",None)
self.defaults["security"]["proxy_DN"]=(None,"dn","Distinguised name (subject) of the proxy (for example, /DC=org/DC=myca/OU=Services/CN=fe1.my.org)",None)
self.defaults["security"]["sym_key"]=("aes_256_cbc","sym_algo","Type of symetric key system used for secure message passing",None)
self.defaults["match"]=copy.deepcopy(match_defaults)
# change default match value
# by default we want to look only for vanilla universe jobs that are not monitoring jobs
self.defaults["match"]["job"]["query_expr"][0]='(JobUniverse==5)&&(GLIDEIN_Is_Monitor =!= TRUE)&&(JOB_Is_Monitor =!= TRUE)'
self.defaults["attrs"]=sub_defaults['attrs']
self.defaults["files"]=copy.deepcopy(sub_defaults['files'])
# ordering is specific to global section of factory
self.defaults["files"][3]["after_group"]=("False",'Bool','Should this file be loaded after the group ones?',None)
self.defaults["groups"]=(xmlParse.OrderedDict(),"Dictionary of groups","Each group contains",self.group_defaults)
return
