def initDefaults(self):
self._add('client_host', self.client_host,
"Hostname for the client to connect to", True, None)
self._add(
'server_secure_port', Conf.getDefaultServerSecurePort(),
"Port number the server uses for communication from servers ",
True, None, '\d+')
self._add(
'client_secure_port', Conf.getDefaultClientSecurePort(),
"Port number the server listens on for communication from clients",
True, None, '\d+')
self._add('private_key', '',
"Port number for the client to connect to https", True, None)
self._add('public_key', '',
"Port number for the client to connect to https", True, None)
self._add('cert', '', "Port number for the client to connect to https",
True, None)
self._add('ca_cert', '',
"Port number for the client to connect to https", True, None)
self._add('plugin_path',
"",
"Colon-separated list of directories to search for plugins",
True,
writable=False)
self._add(
'local_executables_dir',
"executables",
"Directory containing executables for the run client. Part of executables_path",
False,
relTo='conf_dir',
writable=False)
self._add(
'global_executables_dir',
"executables",
"The directory containing executables for the run client. Part of executables_path",
False,
relTo='global_dir',
writable=False)
self._add('executables_path',
"",
"Colon-separated directory list to search for executables",
True,
writable=False)
# the worker's run directory should NEVER be fixed relative to
# anything else; instead, it should just run in the current directory
self._add(
'run_dir', #os.path.join(os.environ["HOME"],
"cpc-worker-workload",
"The run directory for the run client",
True,
writable=False)
def __init__(self, userSpecifiedPath=None):
"""
Can_create allows for the initialization of an empty configuration file
"""
if self.exists():
return
# call parent constructor with right file name.
try:
Conf.__init__(self, name="clientconfig.cfg",
userSpecifiedPath=userSpecifiedPath)
self.lock = threading.RLock()
self.initDefaults()
self._tryRead()
self._loadDefaultServer();
except NoConfError as e:
self._initClientConf()
def create_and_start_server(name="test_server",
clientSecurePort=None,
serverSecurePort=None):
if (clientSecurePort == None):
clientSecurePort = Conf.getDefaultClientSecurePort()
if (serverSecurePort == None):
serverSecurePort = Conf.getDefaultServerSecurePort()
setup_server(name=name)
configureServerPorts(name, clientSecurePort, serverSecurePort)
run_server_command("config keep_alive_interval "
"%s" % 3, name) #every 3 seconds
run_server_command("config reconnect_interval "
"%s" % 3, name) #every 3 seconds
setLogToTrace(name)
generate_bundle(name)
start_server(name)
def create_and_start_server(name="test_server",clientSecurePort=None,
serverSecurePort=None):
if (clientSecurePort==None):
clientSecurePort = Conf.getDefaultClientSecurePort()
if (serverSecurePort==None):
serverSecurePort = Conf.getDefaultServerSecurePort()
setup_server(name=name)
configureServerPorts(name,clientSecurePort,serverSecurePort)
run_server_command("config keep_alive_interval "
"%s"%3,name) #every 3 seconds
run_server_command("config reconnect_interval "
"%s"%3,name) #every 3 seconds
setLogToTrace(name)
generate_bundle(name)
start_server(name)
def initDefaults(self):
self._add('client_host', self.client_host,
"Hostname for the client to connect to", True,None)
self._add('server_secure_port', Conf.getDefaultServerSecurePort(),
"Port number the server uses for communication from servers ",
True,None,'\d+')
self._add('client_secure_port', Conf.getDefaultClientSecurePort(),
"Port number the server listens on for communication from clients",
True,None,'\d+')
self._add('private_key', '',
"Port number for the client to connect to https", True, None)
self._add('public_key', '',
"Port number for the client to connect to https", True, None)
self._add('cert', '',
"Port number for the client to connect to https", True, None)
self._add('ca_cert', '',
"Port number for the client to connect to https", True, None)
self._add('plugin_path', "",
"Colon-separated list of directories to search for plugins",
True, writable=False)
self._add('local_executables_dir', "executables",
"Directory containing executables for the run client. Part of executables_path",
False,
relTo='conf_dir', writable=False)
self._add('global_executables_dir', "executables",
"The directory containing executables for the run client. Part of executables_path",
False,
relTo='global_dir', writable=False)
self._add('executables_path', "",
"Colon-separated directory list to search for executables",
True, writable=False)
# the worker's run directory should NEVER be fixed relative to
# anything else; instead, it should just run in the current directory
self._add('run_dir', #os.path.join(os.environ["HOME"],
"cpc-worker-workload",
"The run directory for the run client",
True, writable=False)
def initDefaults(self):
conf_base.Conf.initDefaults(self)
server_host = ''
self._add('server_host', server_host, "Address the server listens on",
True)
self._add('server_fqdn', socket.getfqdn(), "Manually specified fqdn",
True)
self._add(
'server_secure_port', Conf.getDefaultServerSecurePort(),
"Port number the server uses for communication from servers ",
True, None, '\d+')
self._add(
'client_secure_port', Conf.getDefaultClientSecurePort(),
"Port number the server listens on for communication from clients",
True, None, '\d+')
self._add('nodes', Nodes(), "List of nodes connected to this server",
False)
self._add('revoked_nodes', Nodes(), "List of revoked nodes", False)
self._add('node_connect_requests', Nodes(),
"List of nodes requesting to connect to this server", False)
self._add('sent_node_connect_requests', Nodes(),
"List of connect requests sent", False)
self._add('project_file',
"projects.xml",
"Projects file name (relative to conf_dir)",
relTo='conf_dir')
self._add('state_save_interval',
240,
"Time in seconds between state saves",
True,
validation='\d+')
self._add(
'import_path', "",
"Colon-separated list of directories to search for imports, in addition to cpc/lib, .copernicus/lib and .copernicus/<hostname>/lib",
True)
self._add('mode', 'prod', "The run mode of the server", True, None,
None, ['trace', 'debug', 'prod'])
self._add('profiling', 'false',
"Profile the server CPU usage using yappi (ver >= 0.82)",
True, None, None, ['false', 'true'])
# run options
self._add('run_dir', None,
"Base directory of all files produced by running projects.",
True)
# log options
self._add('log_dir',
"log",
"Directory containing logs",
True,
relTo='conf_dir')
self._add('server_log_file',
"server.log",
"The server log file",
False,
relTo='log_dir')
self._add('error_log_file',
"error.log",
"The error log file",
False,
relTo='log_dir')
# heartbeat options
self._add('heartbeat_time',
120,
"Time in seconds between heartbeats",
True,
validation='\d+')
self._add('heartbeat_file',
"heartbeatlist.xml",
"Heartbeat monitor list",
False,
relTo='conf_dir')
# Task exec queue size. If it exceeds this size, the dataflow
# propagation blocks.
self._add('task_queue_size',
1024,
"Dataflow execution task queue size",
True,
validation='\d+')
#static configuration
self._add('web_root', 'web',
"The directory where html,js and css files are located")
# assets
self._add(
'local_assets_dir',
"local_assets",
"Directory containing local assets such as command output files",
True,
relTo='conf_dir')
self._add('server_cores',
-1,
"Number of cores to use on the server (for OpenMP tasks).",
userSettable=True,
validation='\d+')
self._add(
'num_persistent_connections',
5, "Number of persistent connection to establish for each trusted "
"server",
userSettable=True)
self._add(
'keep_alive_interval',
60,
"Keep alive interval of server connections,value is in minutes",
userSettable=True)
self._add('reconnect_interval',
300,
"Interval between trying to reestablish failed connections ,"
"value is in seconds",
userSettable=True)
self._add('server_verification',True,
"By default servers should always require ssl certificate from both directions" \
"setting this to true will let the sending server to use the client port and disregard" \
"certificate checks. This should only be used in very rare circumstances, for example when debugging" \
"ssl incombatibilites between machines " ,writable=False
,userSettable=False)
dn = os.path.dirname(sys.argv[0])
self.execBasedir = ''
if dn != "":
self.execBasedir = os.path.abspath(dn)
self._add('exec_base_dir', self.execBasedir,
'executable base directory')
# make child processes inherit our path
if os.environ.has_key('PYTHONPATH'):
os.environ['PYTHONPATH'] += ":%s" % self.execBasedir
else:
os.environ['PYTHONPATH'] = self.execBasedir
def __init__(self,
userSpecifiedPath=None,
create=False,
fqdn=socket.getfqdn()):
# check whether the object is already initialized
if not create:
if self.exists():
return
# call parent constructor with right file name.
Conf.__init__(self,
name='client.cnx',
userSpecifiedPath=userSpecifiedPath)
if create:
# create an empty conf without any values.
self.conf = dict()
self.client_host = fqdn
self.server_secure_port = Conf.getDefaultServerSecurePort()
self.client_secure_port = Conf.getDefaultClientSecurePort()
self.privateKey = ''
self.publicKey = ''
self.cert = ''
self.CAcert = ''
self.initDefaults()
# TODO: make it a regular Lock() - for now this might reduce the
# chances of a deadlock
self.lock = threading.RLock()
#worker specific
dn = os.path.dirname(sys.argv[0])
self.execBasedir = ''
if dn != "":
self.execBasedir = os.path.abspath(dn)
self._add('exec_base_dir',
self.execBasedir,
'executable base directory',
writable=False)
self.tempfiles = dict()
#if conffile:
self._tryRead()
'''
the private key, cert and
ca cert need to be provided as filepaths to the ssl connection object
So we create tempfiles for them here
'''
privKeyTempFile = tempfile.NamedTemporaryFile(delete=False)
privKeyTempFile.write(self.get('private_key'))
privKeyTempFile.seek(0)
self.tempfiles['private_key'] = privKeyTempFile
privKeyTempFile.close()
certTempFile = tempfile.NamedTemporaryFile(delete=False)
certTempFile.write(self.get('cert'))
certTempFile.seek(0)
self.tempfiles['cert'] = certTempFile
certTempFile.close()
caCertTempFile = tempfile.NamedTemporaryFile(delete=False)
caCertTempFile.write(self.get('ca_cert'))
caCertTempFile.seek(0)
self.tempfiles['ca_cert'] = caCertTempFile
caCertTempFile.close()
def __init__(self, userSpecifiedPath=None, create=False,
fqdn=socket.getfqdn()):
# check whether the object is already initialized
if not create:
if self.exists():
return
# call parent constructor with right file name.
Conf.__init__(self, name='client.cnx',
userSpecifiedPath=userSpecifiedPath)
if create:
# create an empty conf without any values.
self.conf = dict()
self.client_host = fqdn
self.server_secure_port = Conf.getDefaultServerSecurePort()
self.client_secure_port = Conf.getDefaultClientSecurePort()
self.privateKey = ''
self.publicKey = ''
self.cert = ''
self.CAcert = ''
self.initDefaults()
# TODO: make it a regular Lock() - for now this might reduce the
# chances of a deadlock
self.lock = threading.RLock()
#worker specific
dn = os.path.dirname(sys.argv[0])
self.execBasedir = ''
if dn != "":
self.execBasedir = os.path.abspath(dn)
self._add('exec_base_dir', self.execBasedir,
'executable base directory', writable=False)
self.tempfiles = dict()
#if conffile:
self._tryRead()
'''
the private key, cert and
ca cert need to be provided as filepaths to the ssl connection object
So we create tempfiles for them here
'''
privKeyTempFile = tempfile.NamedTemporaryFile(delete=False)
privKeyTempFile.write(self.get('private_key'))
privKeyTempFile.seek(0)
self.tempfiles['private_key'] = privKeyTempFile
privKeyTempFile.close()
certTempFile = tempfile.NamedTemporaryFile(delete=False)
certTempFile.write(self.get('cert'))
certTempFile.seek(0)
self.tempfiles['cert'] = certTempFile
certTempFile.close()
caCertTempFile = tempfile.NamedTemporaryFile(delete=False)
caCertTempFile.write(self.get('ca_cert'))
caCertTempFile.seek(0)
self.tempfiles['ca_cert'] = caCertTempFile
caCertTempFile.close()
def initDefaults(self):
conf_base.Conf.initDefaults(self)
server_host = ''
self._add('server_host', server_host,
"Address the server listens on", True)
self._add('server_fqdn', socket.getfqdn(),
"Manually specified fqdn", True)
self._add('server_secure_port', Conf.getDefaultServerSecurePort(),
"Port number the server uses for communication from servers ",
True,None,'\d+')
self._add('client_secure_port', Conf.getDefaultClientSecurePort(),
"Port number the server listens on for communication from clients",
True,None,'\d+')
self._add( 'nodes', Nodes(),
"List of nodes connected to this server", False)
self._add('revoked_nodes',Nodes(),"List of revoked nodes",False)
self._add('node_connect_requests',Nodes(),
"List of nodes requesting to connect to this server",False)
self._add('sent_node_connect_requests',Nodes(),
"List of connect requests sent",False)
self._add('project_file', "projects.xml",
"Projects file name (relative to conf_dir)",
relTo='conf_dir')
self._add('state_save_interval', 240,
"Time in seconds between state saves",
True, validation='\d+')
self._add('import_path', "",
"Colon-separated list of directories to search for imports, in addition to cpc/lib, .copernicus/lib and .copernicus/<hostname>/lib",
True)
self._add('mode','prod',
"The run mode of the server",
True,None,None,['trace','debug','prod'])
self._add('profiling', 'false',
"Profile the server CPU usage using yappi (ver >= 0.82)",
True, None, None, ['false', 'true'])
# run options
self._add('run_dir', None,
"Base directory of all files produced by running projects.",
True)
# log options
self._add('log_dir', "log",
"Directory containing logs",
True,
relTo='conf_dir')
self._add('server_log_file', "server.log",
"The server log file", False,
relTo='log_dir')
self._add('error_log_file', "error.log",
"The error log file", False,
relTo='log_dir')
# heartbeat options
self._add('heartbeat_time', 120,
"Time in seconds between heartbeats",
True, validation='\d+')
self._add('heartbeat_file', "heartbeatlist.xml",
"Heartbeat monitor list", False,
relTo='conf_dir')
# Task exec queue size. If it exceeds this size, the dataflow
# propagation blocks.
self._add('task_queue_size', 1024,
"Dataflow execution task queue size",
True, validation='\d+')
#static configuration
self._add('web_root', 'web',
"The directory where html,js and css files are located")
# assets
self._add('local_assets_dir', "local_assets",
"Directory containing local assets such as command output files",
True,
relTo='conf_dir')
self._add('server_cores', -1,
"Number of cores to use on the server (for OpenMP tasks).",
userSettable=True, validation='\d+')
self._add('num_persistent_connections',5,
"Number of persistent connection to establish for each trusted "
"server",
userSettable=True)
self._add('keep_alive_interval',60,
"Keep alive interval of server connections,value is in minutes"
,userSettable=True)
self._add('reconnect_interval',300,
"Interval between trying to reestablish failed connections ,"
"value is in seconds"
,userSettable=True)
dn=os.path.dirname(sys.argv[0])
self.execBasedir = ''
if dn != "":
self.execBasedir=os.path.abspath(dn)
self._add('exec_base_dir', self.execBasedir,
'executable base directory')
# make child processes inherit our path
if os.environ.has_key('PYTHONPATH'):
os.environ['PYTHONPATH'] += ":%s"%self.execBasedir
else:
os.environ['PYTHONPATH'] = self.execBasedir