def delete(args, switchDict):
"""
Given the switches, request a query 'delete' on the ResourceStatusDB
that deletes from <element><tableType> all rows that match the parameters given.
"""
rssClient = ResourceStatusClient.ResourceStatusClient()
result = {
'output': None,
'successful': None,
'message': None,
'match': None
}
output = rssClient.deleteStatusElement(
element=args[1].title(),
tableType=args[2].title(),
name=switchDict['name'],
statusType=switchDict['statusType'],
status=switchDict['status'],
elementType=switchDict['elementType'],
reason=switchDict['reason'],
tokenOwner=switchDict['tokenOwner'])
if 'Value' in output:
result['match'] = int(output['Value'] if output['Value'] else 0)
result['successful'] = output['OK']
result['message'] = output['Message'] if 'Message' in output else None
return result
def select(args, switchDict):
"""
Given the switches, request a query 'select' on the ResourceStatusDB
that gets from <element><tableType> all rows that match the parameters given.
"""
rssClient = ResourceStatusClient.ResourceStatusClient()
meta = {'columns': ['name', 'statusType', 'status', 'elementType', 'reason',
'dateEffective', 'lastCheckTime', 'tokenOwner', 'tokenExpiration']}
result = {'output': None, 'successful': None, 'message': None, 'match': None}
output = rssClient.selectStatusElement(element=args[1].title(),
tableType=args[2].title(),
name=switchDict['name'],
statusType=switchDict['statusType'],
status=switchDict['status'],
elementType=switchDict['elementType'],
lastCheckTime=switchDict['lastCheckTime'],
tokenOwner=switchDict['tokenOwner'],
meta=meta)
result['output'] = [dict(zip(output['Columns'], e)) for e in output['Value']]
result['output'] = filterReason(result['output'], switchDict['reason'])
result['match'] = len(result['output'])
result['successful'] = output['OK']
result['message'] = output['Message'] if 'Message' in output else None
return result
def add( args, switchDict ):
'''
Given the switches, request a query 'addOrModify' on the ResourceStatusDB
that inserts or updates-if-duplicated from <element><tableType> and also adds
a log if flag is active.
'''
rssClient = ResourceStatusClient.ResourceStatusClient()
result = { 'output': None, 'successful': None, 'message': None, 'match': None }
output = rssClient.addOrModifyStatusElement( element = args[1].title(),
tableType = args[2].title(),
name = switchDict[ 'name' ],
statusType = switchDict[ 'statusType' ],
status = switchDict[ 'status' ],
elementType = switchDict[ 'elementType' ],
reason = switchDict[ 'reason' ],
#dateEffective = switchDict[ 'dateEffective' ],
#lastCheckTime = switchDict[ 'lastCheckTime' ],
tokenOwner = getToken( 'owner' ),
tokenExpiration = getToken( 'expiration' )
)
if output.get('Value'):
result['match'] = int( output['Value'] if output['Value'] else 0 )
result['successful'] = output['OK']
result['message'] = output['Message'] if 'Message' in output else None
return result
def getElements():
'''
Given the switches, gets a list of elements with their respective statustype
and status attributes.
'''
rssClient = ResourceStatusClient.ResourceStatusClient()
meta = {'columns': []}
for key in ('Name', 'StatusType', 'Status', 'ElementType', 'TokenOwner'):
#Transforms from upper lower case to lower upper case
if switchDict[key[0].lower() + key[1:]] is None:
meta['columns'].append(key)
elements = rssClient.selectStatusElement(
switchDict['element'],
'Status',
name=switchDict['name'],
statusType=switchDict['statusType'],
status=switchDict['status'],
elementType=switchDict['elementType'],
tokenOwner=switchDict['tokenOwner'],
meta=meta)
return elements
def getElements():
"""
Given the switches, gets a list of elements with their respective statustype
and status attributes.
"""
rssClient = ResourceStatusClient.ResourceStatusClient()
meta = {"columns": []}
for key in ("Name", "StatusType", "Status", "ElementType", "TokenOwner"):
# Transforms from upper lower case to lower upper case
if switchDict[key[0].lower() + key[1:]] is None:
meta["columns"].append(key)
elements = rssClient.selectStatusElement(
switchDict["element"],
"Status",
name=switchDict["name"].split(",") if switchDict["name"] else None,
statusType=switchDict["statusType"].split(",")
if switchDict["statusType"] else None,
status=switchDict["status"].split(",")
if switchDict["status"] else None,
elementType=switchDict["elementType"].split(",")
if switchDict["elementType"] else None,
tokenOwner=switchDict["tokenOwner"].split(",")
if switchDict["tokenOwner"] else None,
meta=meta,
)
return elements
def initSites():
'''
Initializes Sites statuses taking their values from the "SiteMask" table of "JobDB" database.
'''
rssClient = ResourceStatusClient.ResourceStatusClient()
sites = WMSAdministratorClient().getAllSiteMaskStatus()
if not sites['OK']:
subLogger.error(sites['Message'])
DIRACExit(1)
for site, elements in sites['Value'].iteritems():
result = rssClient.addOrModifyStatusElement(
"Site",
"Status",
name=site,
statusType='all',
status=elements[0],
elementType=site.split('.')[0],
tokenOwner='rs_svc',
reason='dirac-rss-sync')
if not result['OK']:
subLogger.error(result['Message'])
DIRACExit(1)
return S_OK()
def modify(args, switchDict):
"""
Given the switches, request a query 'modify' on the ResourceStatusDB
that updates from <element><tableType> and also adds a log if flag is active.
"""
rssClient = ResourceStatusClient.ResourceStatusClient()
result = {'output': None, 'successful': None, 'message': None, 'match': None}
output = rssClient.modifyStatusElement(element=args[1].title(),
tableType=args[2].title(),
name=switchDict['name'],
statusType=switchDict['statusType'],
status=switchDict['status'],
elementType=switchDict['elementType'],
reason=switchDict['reason'],
tokenOwner=getToken('owner'),
tokenExpiration=getToken('expiration')
)
if output.get('Value'):
result['match'] = int(output['Value'] if output['Value'] else 0)
result['successful'] = output['OK']
result['message'] = output['Message'] if 'Message' in output else None
return result
def modify(args, switchDict):
"""
Given the switches, request a query 'modify' on the ResourceStatusDB
that updates from <element><tableType> and also adds a log if flag is active.
"""
rssClient = ResourceStatusClient.ResourceStatusClient()
result = {
"output": None,
"successful": None,
"message": None,
"match": None
}
output = rssClient.modifyStatusElement(
element=args[1].title(),
tableType=args[2].title(),
name=switchDict["name"],
statusType=switchDict["statusType"],
status=switchDict["status"],
elementType=switchDict["elementType"],
reason=switchDict["reason"],
tokenOwner=getToken("owner"),
tokenExpiration=getToken("expiration"),
vO=switchDict["VO"],
)
if output.get("Value"):
result["match"] = int(output["Value"] if output["Value"] else 0)
result["successful"] = output["OK"]
result["message"] = output["Message"] if "Message" in output else None
return result
def delete(args, switchDict):
"""
Given the switches, request a query 'delete' on the ResourceStatusDB
that deletes from <element><tableType> all rows that match the parameters given.
"""
rssClient = ResourceStatusClient.ResourceStatusClient()
result = {
"output": None,
"successful": None,
"message": None,
"match": None
}
output = rssClient.deleteStatusElement(
element=args[1].title(),
tableType=args[2].title(),
name=switchDict["name"],
statusType=switchDict["statusType"],
status=switchDict["status"],
elementType=switchDict["elementType"],
reason=switchDict["reason"],
tokenOwner=switchDict["tokenOwner"],
vO=switchDict["VO"],
)
if "Value" in output:
result["match"] = int(output["Value"] if output["Value"] else 0)
result["successful"] = output["OK"]
result["message"] = output["Message"] if "Message" in output else None
return result
def initSites():
"""
Initializes Sites statuses taking their values from the "SiteMask" table of "JobDB" database.
"""
from DIRAC.WorkloadManagementSystem.Client.WMSAdministratorClient import WMSAdministratorClient
from DIRAC.ResourceStatusSystem.Client import ResourceStatusClient
rssClient = ResourceStatusClient.ResourceStatusClient()
sites = WMSAdministratorClient().getAllSiteMaskStatus()
if not sites["OK"]:
subLogger.error(sites["Message"])
DIRACExit(1)
for site, elements in sites["Value"].items():
result = rssClient.addOrModifyStatusElement(
"Site",
"Status",
name=site,
statusType="all",
status=elements[0],
elementType=site.split(".")[0],
tokenOwner="rs_svc",
reason="dirac-rss-sync",
)
if not result["OK"]:
subLogger.error(result["Message"])
DIRACExit(1)
return S_OK()
def setStatus(switchDict, tokenOwner):
"""
Function that gets the user token, sets the validity for it. Gets the elements
in the database for a given name and statusType(s). Then updates the status
of all them adding a reason and the token.
"""
rssClient = ResourceStatusClient.ResourceStatusClient()
elements = rssClient.selectStatusElement(
switchDict["element"],
"Status",
name=switchDict["name"],
statusType=switchDict["statusType"],
vO=switchDict["VO"],
meta={"columns": ["Status", "StatusType"]},
)
if not elements["OK"]:
return elements
elements = elements["Value"]
if not elements:
subLogger.warn("Nothing found for %s, %s, %s %s" %
(switchDict["element"], switchDict["name"],
switchDict["VO"], switchDict["statusType"]))
return S_OK()
tomorrow = datetime.utcnow().replace(microsecond=0) + timedelta(days=1)
for status, statusType in elements:
subLogger.debug("%s %s" % (status, statusType))
if switchDict["status"] == status:
subLogger.notice("Status for %s (%s) is already %s. Ignoring.." %
(switchDict["name"], statusType, status))
continue
subLogger.debug(
"About to set status %s -> %s for %s, statusType: %s, VO: %s, reason: %s"
% (status, switchDict["status"], switchDict["name"], statusType,
switchDict["VO"], switchDict["reason"]))
result = rssClient.modifyStatusElement(
switchDict["element"],
"Status",
name=switchDict["name"],
statusType=statusType,
status=switchDict["status"],
reason=switchDict["reason"],
vO=switchDict["VO"],
tokenOwner=tokenOwner,
tokenExpiration=tomorrow,
)
if not result["OK"]:
return result
return S_OK()
def __init__(self, rStatus=None, rManagement=None):
# Warm up local CS
CSHelpers.warmUp()
if rStatus is None:
self.rStatus = ResourceStatusClient.ResourceStatusClient()
if rManagement is None:
self.rManagement = ResourceManagementClient()
self.rssConfig = RssConfiguration()
def select(args, switchDict):
"""
Given the switches, request a query 'select' on the ResourceStatusDB
that gets from <element><tableType> all rows that match the parameters given.
"""
rssClient = ResourceStatusClient.ResourceStatusClient()
meta = {
"columns": [
"name",
"statusType",
"status",
"elementType",
"reason",
"dateEffective",
"lastCheckTime",
"tokenOwner",
"tokenExpiration",
"vO",
]
}
result = {
"output": None,
"successful": None,
"message": None,
"match": None
}
output = rssClient.selectStatusElement(
element=args[1].title(),
tableType=args[2].title(),
name=switchDict["name"],
statusType=switchDict["statusType"],
status=switchDict["status"],
elementType=switchDict["elementType"],
lastCheckTime=switchDict["lastCheckTime"],
tokenOwner=switchDict["tokenOwner"],
vO=switchDict["VO"],
meta=meta,
)
result["output"] = [
dict(zip(output["Columns"], e)) for e in output["Value"]
]
result["output"] = filterReason(result["output"], switchDict["reason"])
result["match"] = len(result["output"])
result["successful"] = output["OK"]
result["message"] = output["Message"] if "Message" in output else None
return result
def setStatus(switchDict, tokenOwner):
'''
Function that gets the user token, sets the validity for it. Gets the elements
in the database for a given name and statusType(s). Then updates the status
of all them adding a reason and the token.
'''
rssClient = ResourceStatusClient.ResourceStatusClient()
elements = rssClient.selectStatusElement(
switchDict['element'],
'Status',
name=switchDict['name'],
statusType=switchDict['statusType'],
meta={'columns': ['Status', 'StatusType']})
if not elements['OK']:
return elements
elements = elements['Value']
if not elements:
subLogger.warn('Nothing found for %s, %s, %s' %
(switchDict['element'], switchDict['name'],
switchDict['statusType']))
return S_OK()
tomorrow = datetime.utcnow().replace(microsecond=0) + timedelta(days=1)
for status, statusType in elements:
subLogger.debug('%s %s' % (status, statusType))
if switchDict['status'] == status:
subLogger.notice('Status for %s (%s) is already %s. Ignoring..' %
(switchDict['name'], statusType, status))
continue
result = rssClient.modifyStatusElement(switchDict['element'],
'Status',
name=switchDict['name'],
statusType=statusType,
status=switchDict['status'],
reason=switchDict['reason'],
tokenOwner=tokenOwner,
tokenExpiration=tomorrow)
if not result['OK']:
return result
return S_OK()
def __init__(self):
"""
Constructor.
examples:
>>> s = Synchronizer()
"""
self.log = gLogger.getSubLogger(self.__class__.__name__)
self.operations = Operations()
self.resources = Resources()
self.rStatus = ResourceStatusClient.ResourceStatusClient()
self.rssConfig = RssConfiguration()
self.diracAdmin = DiracAdmin()
def __init__(self,
rStatus=None,
rManagement=None,
defaultStatus="Unknown"):
# Warm up local CS
CSHelpers.warmUp()
if rStatus is None:
self.rStatus = ResourceStatusClient.ResourceStatusClient()
if rManagement is None:
self.rManagement = ResourceManagementClient()
self.defaultStatus = defaultStatus
self.rssConfig = RssConfiguration()
self.tokenOwner = "rs_svc"
result = getProxyInfo()
if result['OK']:
self.tokenOwner = result['Value']['username']
def initSites():
'''
Initializes Sites statuses taking their values from the "SiteMask" table of "JobDB" database.
'''
rssClient = ResourceStatusClient.ResourceStatusClient()
sites = jobDB.getAllSiteMaskStatus()
if not sites['OK']:
subLogger.error(sites['Message'])
DIRACExit(1)
for site, elements in sites['Value'].iteritems():
table = {'table': 'SiteStatus'}
parameters = {
'status': elements[0],
'reason': 'Synchronized',
'name': site,
'dateEffective': elements[1],
'tokenExpiration': Datetime,
'elementType': 'Site',
'statusType': 'all',
'lastCheckTime': None,
'tokenOwner': elements[2],
'meta': table
}
result = rssClient.addIfNotThereStatusElement("Site", "Status",
**parameters)
if not result['OK']:
subLogger.error(result['Message'])
DIRACExit(1)
return S_OK()
def setUp(self):
import DIRAC.ResourceStatusSystem.Client.ResourceStatusClient as mockedModule
_serviceIn = ResourceStatusDB.ResourceStatusDB()
self.client = mockedModule.ResourceStatusClient(serviceIn=_serviceIn)
def initSEs():
'''
Initializes SEs statuses taking their values from the CS.
'''
# WarmUp local copy
CSHelpers.warmUp()
subLogger.info('Initializing SEs')
rssClient = ResourceStatusClient.ResourceStatusClient()
statuses = StateMachine.RSSMachine(None).getStates()
statusTypes = RssConfiguration.RssConfiguration().getConfigStatusType(
'StorageElement')
reason = 'dirac-rss-sync'
subLogger.debug(statuses)
subLogger.debug(statusTypes)
for se in DMSHelpers().getStorageElements():
subLogger.debug(se)
opts = gConfig.getOptionsDict('/Resources/StorageElements/%s' % se)
if not opts['OK']:
subLogger.warn(opts['Message'])
continue
opts = opts['Value']
subLogger.debug(opts)
# We copy the list into a new object to remove items INSIDE the loop !
statusTypesList = statusTypes[:]
for statusType, status in opts.iteritems():
# Sanity check...
if statusType not in statusTypesList:
continue
# Transforms statuses to RSS terms
if status in ('NotAllowed', 'InActive'):
status = 'Banned'
if status not in statuses:
subLogger.error('%s not a valid status for %s - %s' %
(status, se, statusType))
continue
# We remove from the backtracking
statusTypesList.remove(statusType)
subLogger.debug([se, statusType, status, reason])
result = rssClient.addOrModifyStatusElement(
'Resource',
'Status',
name=se,
statusType=statusType,
status=status,
elementType='StorageElement',
tokenOwner='rs_svc',
reason=reason)
if not result['OK']:
subLogger.error('Failed to modify')
subLogger.error(result['Message'])
continue
# Backtracking: statusTypes not present on CS
for statusType in statusTypesList:
result = rssClient.addOrModifyStatusElement(
'Resource',
'Status',
name=se,
statusType=statusType,
status=DEFAULT_STATUS,
elementType='StorageElement',
tokenOwner='rs_svc',
reason=reason)
if not result['OK']:
subLogger.error('Error in backtracking for %s,%s,%s' %
(se, statusType, status))
subLogger.error(result['Message'])
return S_OK()
def initSEs():
"""
Initializes SEs statuses taking their values from the CS.
"""
from DIRAC import gConfig
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers
from DIRAC.ResourceStatusSystem.Utilities import CSHelpers, RssConfiguration
from DIRAC.ResourceStatusSystem.PolicySystem import StateMachine
from DIRAC.ResourceStatusSystem.Client import ResourceStatusClient
# WarmUp local copy
CSHelpers.warmUp()
subLogger.info("Initializing SEs")
rssClient = ResourceStatusClient.ResourceStatusClient()
statuses = StateMachine.RSSMachine(None).getStates()
statusTypes = RssConfiguration.RssConfiguration().getConfigStatusType(
"StorageElement")
reason = "dirac-rss-sync"
subLogger.debug(statuses)
subLogger.debug(statusTypes)
for se in DMSHelpers().getStorageElements():
subLogger.debug(se)
opts = gConfig.getOptionsDict("/Resources/StorageElements/%s" % se)
if not opts["OK"]:
subLogger.warn(opts["Message"])
continue
opts = opts["Value"]
subLogger.debug(opts)
# We copy the list into a new object to remove items INSIDE the loop !
statusTypesList = statusTypes[:]
for statusType, status in opts.items():
# Sanity check...
if statusType not in statusTypesList:
continue
# Transforms statuses to RSS terms
if status in ("NotAllowed", "InActive"):
status = "Banned"
if status not in statuses:
subLogger.error("%s not a valid status for %s - %s" %
(status, se, statusType))
continue
# We remove from the backtracking
statusTypesList.remove(statusType)
subLogger.debug([se, statusType, status, reason])
result = rssClient.addOrModifyStatusElement(
"Resource",
"Status",
name=se,
statusType=statusType,
status=status,
elementType="StorageElement",
tokenOwner="rs_svc",
reason=reason,
)
if not result["OK"]:
subLogger.error("Failed to modify")
subLogger.error(result["Message"])
continue
# Backtracking: statusTypes not present on CS
for statusType in statusTypesList:
result = rssClient.addOrModifyStatusElement(
"Resource",
"Status",
name=se,
statusType=statusType,
status=DEFAULT_STATUS,
elementType="StorageElement",
tokenOwner="rs_svc",
reason=reason,
)
if not result["OK"]:
subLogger.error("Error in backtracking for %s,%s,%s" %
(se, statusType, status))
subLogger.error(result["Message"])
return S_OK()