def __init__(self): """ c'tor Just setting defaults """ self.workDir = '' # Working directory where the files are going to be stored self.jsonFile = 'pilot.json' # default filename of the pilot json file # domain name of the web server(s) used to upload the pilot json file and the pilot scripts self.pilotFileServer = '' # pilot sync parameters self.pilotRepo = 'https://github.com/DIRACGrid/Pilot.git' # repository of the pilot self.pilotVORepo = '' # repository of the VO that can contain a pilot extension # 'pilotLocalRepo' = 'pilotLocalRepo' # local repository to be created # 'pilotVOLocalRepo' = 'pilotVOLocalRepo' # local VO repository to be created self.pilotSetup = gConfig.getValue('/DIRAC/Setup', '') self.projectDir = '' # where the find the pilot scripts in the VO pilot repository self.pilotScriptPath = 'Pilot' # where the find the pilot scripts in the pilot repository self.pilotVOScriptPath = '' self.pilotVersion = '' self.pilotVOVersion = '' self.pilotRepoBranch = 'master' self.pilotVORepoBranch = 'master' self.certAndKeyLocation = getHostCertificateAndKeyLocation() self.casLocation = getCAsLocation() self._checksumDict = {} # If this is set to True, we will attempt to upload the files # to all the servers in the list. Obviously, it will only work # for the DIRAC web servers (see _upload) self.uploadToWebApp = True self.log = gLogger.getSubLogger(__name__)
def __init__(self): """ c'tor Just setting defaults """ self.jsonFile = 'pilot.json' # default filename of the pilot json file # domain name of the web server used to upload the pilot json file and the pilot scripts self.pilotFileServer = '' # pilot sync parameters self.pilotRepo = 'https://github.com/DIRACGrid/Pilot.git' # repository of the pilot self.pilotVORepo = '' # repository of the VO that can contain a pilot extension # 'pilotLocalRepo' = 'pilotLocalRepo' # local repository to be created # 'pilotVOLocalRepo' = 'pilotVOLocalRepo' # local VO repository to be created self.pilotSetup = gConfig.getValue('/DIRAC/Setup', '') self.projectDir = '' # where the find the pilot scripts in the VO pilot repository self.pilotScriptPath = 'Pilot' # where the find the pilot scripts in the pilot repository self.pilotVOScriptPath = '' self.pilotVersion = '' self.pilotVOVersion = '' self.certAndKeyLocation = getHostCertificateAndKeyLocation() self.casLocation = getCAsLocation() self.log = gLogger.getSubLogger(__name__)
def __init__(self, adminUrls=None): '''initialise''' adminUrls = adminUrls or {} self.__soapClients = {} locs = getHostCertificateAndKeyLocation() if not locs: raise RuntimeError("Cannot find the host cert and key files") hostCert, hostKey = locs gLogger.info("using host cert: %s" % hostCert) gLogger.info("using host key: %s" % hostKey) result = gConfig.getSections('/Registry/VOMS/URLs') if not result['OK']: raise Exception(result['Message']) self.__vos = result['Value'] for vo in self.__vos: result = gConfig.getOptionsDict('/Registry/VOMS/URLs/%s' % vo) if not result['OK']: gLogger.error(result['Message']) continue url_dict = result['Value'] if 'VOMSAdmin' not in url_dict: gLogger.error("Skipping setting up VOMSService for VO: %s " "as no VOMSAdmin option in config" % vo) continue retries = 3 while retries: try: admin = adminUrls.get(vo, url_dict['VOMSAdmin']) httpstransport = HTTPSClientCertTransport(hostCert, hostKey, getCAsLocation()) adminClient = Client(admin + '?wsdl', transport=httpstransport) adminClient.set_options(headers={"X-VOMS-CSRF-GUARD": "1"}) compatClient = Client(os.path.join(os.path.dirname(admin), 'VOMSCompatibility?wsdl'), transport=HTTPSClientCertTransport(hostCert, hostKey, getCAsLocation())) compatClient.set_options(headers={"X-VOMS-CSRF-GUARD": "1"}) self.__soapClients[vo] = {'Admin': adminClient, 'Compat': compatClient} break except Exception: gLogger.warn("Failed to connect suds client to VOMSAdmin or VOMSCompatibility URL, retrying...") retries -= 1 else: gLogger.error("Maximum number of retries reached. Skipping " "setting up VOMSService for VO: %s" % vo) gLogger.error(traceback.format_exc()) if not self.__soapClients: raise RuntimeError("Couldn't setup ANY SOAP clients")
def initialize(self): """Initial settings""" self.workingDirectory = self.am_getOption("WorkDirectory") self.saveDir = self.am_getOption("SaveDirectory", self.saveDir) self.uploadLocations = self.am_getOption("UploadLocations", self.uploadLocations) includeMasterCS = self.am_getOption("IncludeMasterCS", self.includeMasterCS) if isinstance(includeMasterCS, str) and includeMasterCS.lower() in ["n", "no", "false"]: self.includeMasterCS = False self.certAndKeyLocation = getHostCertificateAndKeyLocation() self.casLocation = getCAsLocation() return S_OK()
def initialize(self): """ Initial settings """ self.workingDirectory = self.am_getOption('WorkDirectory') self.saveDir = self.am_getOption('SaveDirectory', self.saveDir) self.uploadLocations = self.am_getOption('UploadLocations', self.uploadLocations) includeMasterCS = self.am_getOption('IncludeMasterCS', self.includeMasterCS) if isinstance(includeMasterCS, six.string_types) and includeMasterCS.lower() in [ 'n', 'no', 'false' ]: self.includeMasterCS = False self.certAndKeyLocation = getHostCertificateAndKeyLocation() self.casLocation = getCAsLocation() return S_OK()
def __init__(self): """ c'tor Just setting defaults """ self.jsonFile = 'pilot.json' # default filename of the pilot json file # domain name of the web server used to upload the pilot json file and the pilot scripts self.pilotFileServer = '' # pilot sync parameters self.pilotRepo = 'https://github.com/DIRACGrid/Pilot.git' # repository of the pilot self.pilotVORepo = '' # repository of the VO that can contain a pilot extension # 'pilotLocalRepo' = 'pilotLocalRepo' # local repository to be created # 'pilotVOLocalRepo' = 'pilotVOLocalRepo' # local VO repository to be created self.pilotSetup = gConfig.getValue('/DIRAC/Setup', '') self.projectDir = '' # where the find the pilot scripts in the VO pilot repository self.pilotScriptPath = 'Pilot' # where the find the pilot scripts in the pilot repository self.pilotVOScriptPath = '' self.pilotVersion = '' self.pilotVOVersion = '' self.certAndKeyLocation = getHostCertificateAndKeyLocation()
def __forceServiceUpdate(url, fromMaster): """ Force updating configuration on a given service :param str url: service URL :param bool fromMaster: flag to force updating from the master CS :return: S_OK/S_ERROR """ gLogger.info('Updating service configuration on', url) if url.startswith('dip'): rpc = RPCClient(url) result = rpc.refreshConfiguration(fromMaster) elif url.startswith('http'): hostCertTuple = getHostCertificateAndKeyLocation() resultRequest = requests.get(url, headers={'X-RefreshConfiguration': "True"}, cert=hostCertTuple, verify=False) result = S_OK() if resultRequest.status_code != 200: result = S_ERROR("Status code returned %d" % resultRequest.status_code) result['URL'] = url return result
def __readHostInfo(): """ Get host current loads, memory, etc """ result = dict() # Memory info re_parser = re.compile(r'^(?P<key>\S*):\s*(?P<value>\d*)\s*kB') for line in open('/proc/meminfo'): match = re_parser.match(line) if not match: continue key, value = match.groups(['key', 'value']) result[key] = int(value) for mtype in ['Mem', 'Swap']: memory = int(result.get(mtype + 'Total')) mfree = int(result.get(mtype + 'Free')) if memory > 0: percentage = float(memory - mfree) / float(memory) * 100. else: percentage = 0 name = 'Memory' if mtype == "Swap": name = 'Swap' result[name] = '%.1f%%/%.1fMB' % (percentage, memory / 1024.) # Loads l1, l5, l15 = (str(lx) for lx in os.getloadavg()) result['Load1'] = l1 result['Load5'] = l5 result['Load15'] = l15 result['Load'] = '/'.join([l1, l5, l15]) # CPU info with open('/proc/cpuinfo', 'r') as fd: lines = fd.readlines() processors = 0 physCores = {} for line in lines: if line.strip(): parameter, value = line.split(':') parameter = parameter.strip() value = value.strip() if parameter.startswith('processor'): processors += 1 if parameter.startswith('physical id'): physCores[value] = parameter if parameter.startswith('model name'): result['CPUModel'] = value if parameter.startswith('cpu MHz'): result['CPUClock'] = value result['Cores'] = processors result['PhysicalCores'] = len(physCores) # Disk occupancy summary = '' _status, output = commands.getstatusoutput('df') lines = output.split('\n') for i in xrange(len(lines)): if lines[i].startswith('/dev'): fields = lines[i].split() if len(fields) == 1: fields += lines[i + 1].split() _disk = fields[0].replace('/dev/sd', '') partition = fields[5] occupancy = fields[4] summary += ",%s:%s" % (partition, occupancy) result['DiskOccupancy'] = summary[1:] result['RootDiskSpace'] = Os.getDiskSpace(rootPath) # Open files puser = getpass.getuser() _status, output = commands.getstatusoutput('lsof') pipes = 0 files = 0 sockets = 0 lines = output.split('\n') for line in lines: fType = line.split()[4] user = line.split()[2] if user == puser: if fType in ['REG']: files += 1 elif fType in ['unix', 'IPv4']: sockets += 1 elif fType in ['FIFO']: pipes += 1 result['OpenSockets'] = sockets result['OpenFiles'] = files result['OpenPipes'] = pipes infoResult = gComponentInstaller.getInfo() if infoResult['OK']: result.update(infoResult['Value']) # the infoResult value is {"Extensions":{'a1':'v1',a2:'v2'}; we convert to a string result.update({ "Extensions": ";".join([ "%s:%s" % (key, value) for (key, value) in infoResult["Value"].get('Extensions').iteritems() ]) }) # Host certificate properties certFile, _keyFile = getHostCertificateAndKeyLocation() chain = X509Chain() chain.loadChainFromFile(certFile) resultCert = chain.getCredentials() if resultCert['OK']: result['SecondsLeft'] = resultCert['Value']['secondsLeft'] result['CertificateValidity'] = str( timedelta(seconds=resultCert['Value']['secondsLeft'])) result['CertificateDN'] = resultCert['Value']['subject'] result['HostProperties'] = resultCert['Value']['groupProperties'] result['CertificateIssuer'] = resultCert['Value']['issuer'] # Host uptime result['Uptime'] = str( timedelta(seconds=(time.time() - psutil.boot_time()))) return S_OK(result)
def export_getHostInfo(self): """ Get host current loads, memory, etc """ result = dict() # Memory info re_parser = re.compile(r'^(?P<key>\S*):\s*(?P<value>\d*)\s*kB' ) for line in open('/proc/meminfo'): match = re_parser.match(line) if not match: continue key, value = match.groups(['key', 'value']) result[key] = int(value) for mtype in ['Mem','Swap']: memory = int(result.get(mtype+'Total')) mfree = int(result.get(mtype+'Free')) if memory > 0: percentage = float(memory-mfree)/float(memory)*100. else: percentage = 0 name = 'Memory' if mtype == "Swap": name = 'Swap' result[name] = '%.1f%%/%.1fMB' % (percentage,memory/1024.) # Loads line = open('/proc/loadavg').read() l1,l5,l15,d1,d2 = line.split() result['Load1'] = l1 result['Load5'] = l5 result['Load15'] = l15 result['Load'] = '/'.join([l1,l5,l15]) # CPU info lines = open( '/proc/cpuinfo', 'r' ).readlines() processors = 0 physCores = {} for line in lines: if line.strip(): parameter, value = line.split(':') parameter = parameter.strip() value = value.strip() if parameter.startswith('processor'): processors += 1 if parameter.startswith('physical id'): physCores[value] = parameter if parameter.startswith('model name'): result['CPUModel'] = value if parameter.startswith('cpu MHz'): result['CPUClock'] = value result['Cores'] = processors result['PhysicalCores'] = len(physCores) # Disk occupancy summary = '' status,output = commands.getstatusoutput('df') lines = output.split('\n') for i in range( len( lines ) ): if lines[i].startswith('/dev'): fields = lines[i].split() if len( fields ) == 1: fields += lines[i+1].split() disk = fields[0].replace('/dev/sd','') partition = fields[5] occupancy = fields[4] summary += ",%s:%s" % (partition,occupancy) result['DiskOccupancy'] = summary[1:] result['RootDiskSpace'] = Os.getDiskSpace( DIRAC.rootPath ) # Open files puser= getpass.getuser() status,output = commands.getstatusoutput('lsof') pipes = 0 files = 0 sockets = 0 lines = output.split('\n') for line in lines: fType = line.split()[4] user = line.split()[2] if user == puser: if fType in ['REG']: files += 1 elif fType in ['unix','IPv4']: sockets += 1 elif fType in ['FIFO']: pipes += 1 result['OpenSockets'] = sockets result['OpenFiles'] = files result['OpenPipes'] = pipes infoResult = InstallTools.getInfo( getCSExtensions() ) if infoResult['OK']: result.update( infoResult['Value'] ) # Host certificate properties certFile,keyFile = getHostCertificateAndKeyLocation() chain = X509Chain() chain.loadChainFromFile( certFile ) resultCert = chain.getCredentials() if resultCert['OK']: result['SecondsLeft'] = resultCert['Value']['secondsLeft'] result['CertificateValidity'] = str( timedelta( seconds = resultCert['Value']['secondsLeft'] ) ) result['CertificateDN'] = resultCert['Value']['subject'] result['HostProperties'] = ','.join( resultCert['Value']['groupProperties'] ) result['CertificateIssuer'] = resultCert['Value']['issuer'] # Host uptime try: upFile = open('/proc/uptime', 'r') uptime_seconds = float(upFile.readline().split()[0]) upFile.close() result['Uptime'] = str(timedelta(seconds = uptime_seconds)) except: pass return S_OK(result)
def __readHostInfo(): """Get host current loads, memory, etc""" result = dict() # Memory info re_parser = re.compile(r"^(?P<key>\S*):\s*(?P<value>\d*)\s*kB") for line in open("/proc/meminfo"): match = re_parser.match(line) if not match: continue key, value = match.groups(["key", "value"]) result[key] = int(value) for mtype in ["Mem", "Swap"]: memory = int(result.get(mtype + "Total")) mfree = int(result.get(mtype + "Free")) if memory > 0: percentage = float(memory - mfree) / float(memory) * 100.0 else: percentage = 0 name = "Memory" if mtype == "Swap": name = "Swap" result[name] = "%.1f%%/%.1fMB" % (percentage, memory / 1024.0) # Loads l1, l5, l15 = (str(lx) for lx in os.getloadavg()) result["Load1"] = l1 result["Load5"] = l5 result["Load15"] = l15 result["Load"] = "/".join([l1, l5, l15]) # CPU info with open("/proc/cpuinfo", "r") as fd: lines = fd.readlines() processors = 0 physCores = {} for line in lines: if line.strip(): parameter, value = line.split(":") parameter = parameter.strip() value = value.strip() if parameter.startswith("processor"): processors += 1 if parameter.startswith("physical id"): physCores[value] = parameter if parameter.startswith("model name"): result["CPUModel"] = value if parameter.startswith("cpu MHz"): result["CPUClock"] = value result["Cores"] = processors result["PhysicalCores"] = len(physCores) # Disk occupancy summary = "" _status, output = commands.getstatusoutput("df") lines = output.split("\n") for i in range(len(lines)): if lines[i].startswith("/dev"): fields = lines[i].split() if len(fields) == 1: fields += lines[i + 1].split() partition = fields[5] occupancy = fields[4] summary += ",%s:%s" % (partition, occupancy) result["DiskOccupancy"] = summary[1:] result["RootDiskSpace"] = Os.getDiskSpace(rootPath) # Open files puser = getpass.getuser() _status, output = commands.getstatusoutput("lsof") pipes = 0 files = 0 sockets = 0 lines = output.split("\n") for line in lines: fType = line.split()[4] user = line.split()[2] if user == puser: if fType in ["REG"]: files += 1 elif fType in ["unix", "IPv4"]: sockets += 1 elif fType in ["FIFO"]: pipes += 1 result["OpenSockets"] = sockets result["OpenFiles"] = files result["OpenPipes"] = pipes infoResult = gComponentInstaller.getInfo() if infoResult["OK"]: result.update(infoResult["Value"]) # the infoResult value is {"Extensions":{'a1':'v1',a2:'v2'}; we convert to a string result.update({ "Extensions": ";".join([ "%s:%s" % (key, value) for (key, value ) in infoResult["Value"].get("Extensions").items() ]) }) # Host certificate properties certFile, _keyFile = getHostCertificateAndKeyLocation() chain = X509Chain() chain.loadChainFromFile(certFile) resultCert = chain.getCredentials() if resultCert["OK"]: result["SecondsLeft"] = resultCert["Value"]["secondsLeft"] result["CertificateValidity"] = str( timedelta(seconds=resultCert["Value"]["secondsLeft"])) result["CertificateDN"] = resultCert["Value"]["subject"] result["CertificateIssuer"] = resultCert["Value"]["issuer"] # Host uptime result["Uptime"] = str( timedelta(seconds=(time.time() - psutil.boot_time()))) return S_OK(result)
def export_getHostInfo(self): """ Get host current loads, memory, etc """ result = dict() # Memory info re_parser = re.compile(r"^(?P<key>\S*):\s*(?P<value>\d*)\s*kB") for line in open("/proc/meminfo"): match = re_parser.match(line) if not match: continue key, value = match.groups(["key", "value"]) result[key] = int(value) for mtype in ["Mem", "Swap"]: memory = int(result.get(mtype + "Total")) mfree = int(result.get(mtype + "Free")) if memory > 0: percentage = float(memory - mfree) / float(memory) * 100.0 else: percentage = 0 name = "Memory" if mtype == "Swap": name = "Swap" result[name] = "%.1f%%/%.1fMB" % (percentage, memory / 1024.0) # Loads line = open("/proc/loadavg").read() l1, l5, l15, d1, d2 = line.split() result["Load1"] = l1 result["Load5"] = l5 result["Load15"] = l15 result["Load"] = "/".join([l1, l5, l15]) # CPU info lines = open("/proc/cpuinfo", "r").readlines() processors = 0 physCores = {} for line in lines: if line.strip(): parameter, value = line.split(":") parameter = parameter.strip() value = value.strip() if parameter.startswith("processor"): processors += 1 if parameter.startswith("physical id"): physCores[value] = parameter if parameter.startswith("model name"): result["CPUModel"] = value if parameter.startswith("cpu MHz"): result["CPUClock"] = value result["Cores"] = processors result["PhysicalCores"] = len(physCores) # Disk occupancy summary = "" status, output = commands.getstatusoutput("df") lines = output.split("\n") for i in range(len(lines)): if lines[i].startswith("/dev"): fields = lines[i].split() if len(fields) == 1: fields += lines[i + 1].split() disk = fields[0].replace("/dev/sd", "") partition = fields[5] occupancy = fields[4] summary += ",%s:%s" % (partition, occupancy) result["DiskOccupancy"] = summary[1:] result["RootDiskSpace"] = Os.getDiskSpace(DIRAC.rootPath) # Open files puser = getpass.getuser() status, output = commands.getstatusoutput("lsof") pipes = 0 files = 0 sockets = 0 lines = output.split("\n") for line in lines: fType = line.split()[4] user = line.split()[2] if user == puser: if fType in ["REG"]: files += 1 elif fType in ["unix", "IPv4"]: sockets += 1 elif fType in ["FIFO"]: pipes += 1 result["OpenSockets"] = sockets result["OpenFiles"] = files result["OpenPipes"] = pipes infoResult = InstallTools.getInfo(getCSExtensions()) if infoResult["OK"]: result.update(infoResult["Value"]) # Host certificate properties certFile, keyFile = getHostCertificateAndKeyLocation() chain = X509Chain() chain.loadChainFromFile(certFile) resultCert = chain.getCredentials() if resultCert["OK"]: result["SecondsLeft"] = resultCert["Value"]["secondsLeft"] result["CertificateValidity"] = str(timedelta(seconds=resultCert["Value"]["secondsLeft"])) result["CertificateDN"] = resultCert["Value"]["subject"] result["HostProperties"] = ",".join(resultCert["Value"]["groupProperties"]) result["CertificateIssuer"] = resultCert["Value"]["issuer"] # Host uptime try: upFile = open("/proc/uptime", "r") uptime_seconds = float(upFile.readline().split()[0]) upFile.close() result["Uptime"] = str(timedelta(seconds=uptime_seconds)) except: pass return S_OK(result)
def export_getHostInfo(self): """ Get host current loads, memory, etc """ result = dict() # Memory info re_parser = re.compile(r'^(?P<key>\S*):\s*(?P<value>\d*)\s*kB') for line in open('/proc/meminfo'): match = re_parser.match(line) if not match: continue key, value = match.groups(['key', 'value']) result[key] = int(value) for mtype in ['Mem', 'Swap']: memory = int(result.get(mtype + 'Total')) mfree = int(result.get(mtype + 'Free')) if memory > 0: percentage = float(memory - mfree) / float(memory) * 100. else: percentage = 0 name = 'Memory' if mtype == "Swap": name = 'Swap' result[name] = '%.1f%%/%.1fMB' % (percentage, memory / 1024.) # Loads line = open('/proc/loadavg').read() l1, l5, l15, d1, d2 = line.split() result['Load1'] = l1 result['Load5'] = l5 result['Load15'] = l15 result['Load'] = '/'.join([l1, l5, l15]) # CPU info lines = open('/proc/cpuinfo', 'r').readlines() processors = 0 physCores = {} for line in lines: if line.strip(): parameter, value = line.split(':') parameter = parameter.strip() value = value.strip() if parameter.startswith('processor'): processors += 1 if parameter.startswith('physical id'): physCores[value] = parameter if parameter.startswith('model name'): result['CPUModel'] = value if parameter.startswith('cpu MHz'): result['CPUClock'] = value result['Cores'] = processors result['PhysicalCores'] = len(physCores) # Disk occupancy summary = '' status, output = commands.getstatusoutput('df') lines = output.split('\n') for i in range(len(lines)): if lines[i].startswith('/dev'): fields = lines[i].split() if len(fields) == 1: fields += lines[i + 1].split() disk = fields[0].replace('/dev/sd', '') partition = fields[5] occupancy = fields[4] summary += ",%s:%s" % (partition, occupancy) result['DiskOccupancy'] = summary[1:] result['RootDiskSpace'] = Os.getDiskSpace(DIRAC.rootPath) # Open files puser = getpass.getuser() status, output = commands.getstatusoutput('lsof') pipes = 0 files = 0 sockets = 0 lines = output.split('\n') for line in lines: fType = line.split()[4] user = line.split()[2] if user == puser: if fType in ['REG']: files += 1 elif fType in ['unix', 'IPv4']: sockets += 1 elif fType in ['FIFO']: pipes += 1 result['OpenSockets'] = sockets result['OpenFiles'] = files result['OpenPipes'] = pipes infoResult = InstallTools.getInfo(getCSExtensions()) if infoResult['OK']: result.update(infoResult['Value']) # Host certificate properties certFile, keyFile = getHostCertificateAndKeyLocation() chain = X509Chain() chain.loadChainFromFile(certFile) resultCert = chain.getCredentials() if resultCert['OK']: result['SecondsLeft'] = resultCert['Value']['secondsLeft'] result['CertificateValidity'] = str( timedelta(seconds=resultCert['Value']['secondsLeft'])) result['CertificateDN'] = resultCert['Value']['subject'] result['HostProperties'] = ','.join( resultCert['Value']['groupProperties']) result['CertificateIssuer'] = resultCert['Value']['issuer'] # Host uptime try: upFile = open('/proc/uptime', 'r') uptime_seconds = float(upFile.readline().split()[0]) upFile.close() result['Uptime'] = str(timedelta(seconds=uptime_seconds)) except: pass return S_OK(result)