def _check_fields(self):
"""
Add extra checks to those made in every record.
"""
# First, call the parent's version.
Record._check_fields(self)
# Extract the relevant information from the user fqan.
# Keep the fqan itself as other methods in the class use it.
if self._record_content["FQAN"] not in ("None", None):
role, group, vo = parse_fqan(self._record_content["FQAN"])
# We can't / don't put NULL in the database, so we use 'None'
if role is None:
role = "None"
if group is None:
group = "None"
if vo is None:
vo = "None"
self._record_content["VORole"] = role
self._record_content["VOGroup"] = group
# Confusing terminology from the CAR
self._record_content["VO"] = vo
# Check the ScalingFactor.
slt = self._record_content["ServiceLevelType"]
sl = self._record_content["ServiceLevel"]
(slt, sl) = self._check_factor(slt, sl)
self._record_content["ServiceLevelType"] = slt
self._record_content["ServiceLevel"] = sl
# Check the values of StartTime and EndTime
self._check_start_end_times()
def __init__(self):
'''Provide the necessary lists containing message information.'''
Record.__init__(self)
# Fields which are required by the message format.
self._mandatory_fields = ["VMUUID", "SiteName"]
# This list allows us to specify the order of lines when we construct records.
self._msg_fields = [
"VMUUID", "SiteName", "MachineName", "LocalUserId", "LocalGroupId",
"GlobalUserName", "FQAN", "Status", "StartTime", "EndTime",
"SuspendDuration", "WallDuration", "CpuDuration", "CpuCount",
"NetworkType", "NetworkInbound", "NetworkOutbound", "Memory",
"Disk", "StorageRecordId", "ImageId", "CloudType"
]
# This list specifies the information that goes in the database.
self._db_fields = self._msg_fields[:7] + ['VO', 'VOGroup', 'VORole'
] + self._msg_fields[7:]
self._all_fields = self._db_fields
self._ignored_fields = ["UpdateTime"]
# Fields which will have an integer stored in them
self._int_fields = [
"SuspendDuration", "WallDuration", "CpuDuration", "CpuCount",
"NetworkInbound", "NetworkOutbound", "Memory", "Disk"
]
self._datetime_fields = ["StartTime", "EndTime"]
def __init__(self):
'''Provide the necessary lists containing message information.'''
Record.__init__(self)
# Fields which are required by the message format.
self._mandatory_fields = ['SiteName', 'Month', 'Year', 'NumberOfVMs']
# This list allows us to specify the order of lines when we construct records.
self._msg_fields = ['SiteName', 'CloudComputeService', 'Month', 'Year',
'GlobalUserName', 'VO', 'VOGroup', 'VORole',
'Status', 'CloudType', 'ImageId',
'EarliestStartTime', 'LatestStartTime',
'WallDuration', 'CpuDuration', 'CpuCount',
'NetworkInbound', 'NetworkOutbound',
'Memory', 'Disk',
'BenchmarkType', 'Benchmark', 'NumberOfVMs']
# This list specifies the information that goes in the database.
self._db_fields = self._msg_fields
self._all_fields = self._db_fields
self._ignored_fields = ['UpdateTime']
# Fields which will have an integer stored in them
self._int_fields = ['Month', 'Year', 'WallDuration', 'CpuDuration',
'CpuCount', 'NetworkInbound', 'NetworkOutbound',
'Memory', 'Disk', 'NumberOfVMs']
self._float_fields = ['Benchmark']
self._datetime_fields = ['EarliestStartTime', 'LatestStartTime']
def __init__(self):
'''Provide the necessary lists containing message information.'''
Record.__init__(self)
# Fields which are required by the message format.
self._mandatory_fields = ['SiteName', 'Month', 'Year', 'NumberOfVMs']
# This list allows us to specify the order of lines when we construct records.
self._msg_fields = [
'SiteName', 'Month', 'Year', 'GlobalUserName', 'VO', 'VOGroup',
'VORole', 'Status', 'CloudType', 'ImageId', 'EarliestStartTime',
'LatestStartTime', 'WallDuration', 'CpuDuration', 'NetworkInbound',
'NetworkOutbound', 'Memory', 'Disk', 'NumberOfVMs'
]
# This list specifies the information that goes in the database.
self._db_fields = self._msg_fields
self._all_fields = self._db_fields
self._ignored_fields = ['UpdateTime']
# Fields which will have an integer stored in them
self._int_fields = [
'Month', 'Year', 'WallDuration', 'CpuDuration', 'NetworkInbound',
'NetworkOutbound', 'Memory', 'Disk', 'NumberOfVMs'
]
self._datetime_fields = ['EarliestStartTime', 'LatestStartTime']
def __init__(self):
'''Populate fields required to load the message.'''
Record.__init__(self)
# Fields which are required by the message format.
self._mandatory_fields = ["Site", "Month", "Year", "WallDuration",
"CpuDuration", "NumberOfJobs"]
# This list allows us to specify the order of lines when we construct
# records.
self._msg_fields = ["Site", "Month", "Year", "GlobalUserName", "VO",
"VOGroup", "VORole", "SubmitHost", "InfrastructureType", "ServiceLevelType",
"ServiceLevel", "NodeCount", "Processors", "EarliestEndTime", "LatestEndTime",
"WallDuration", "CpuDuration", "NumberOfJobs"]
# Fields which will have an integer stored in them
self._int_fields = ["Month", "Year", "NodeCount", "Processors",
"WallDuration", "CpuDuration", "NumberOfJobs"]
self._datetime_fields = ["EarliestEndTime", "LatestEndTime"]
self._ignored_fields = ["UpdateTime"]
# This list specifies the information that goes in the database.
self._db_fields = self._msg_fields
# All allowed fields.
self._all_fields = self._msg_fields
def _check_fields(self):
'''
Add extra checks to those made in every record.
'''
# First, call the parent's version.
Record._check_fields(self)
# Extract the relevant information from the user fqan.
# Keep the fqan itself as other methods in the class use it.
if self._record_content['FQAN'] not in ('None', None):
role, group, vo = parse_fqan(self._record_content['FQAN'])
# We can't / don't put NULL in the database, so we use 'None'
if role == None:
role = 'None'
if group == None:
group = 'None'
if vo == None:
vo = 'None'
self._record_content['VORole'] = role
self._record_content['VOGroup'] = group
# Confusing terminology from the CAR
self._record_content['VO'] = vo
# Check the ScalingFactor.
slt = self._record_content['ServiceLevelType']
sl = self._record_content['ServiceLevel']
(slt, sl) = self._check_factor(slt, sl)
self._record_content['ServiceLevelType'] = slt
self._record_content['ServiceLevel'] = sl
# Check the values of StartTime and EndTime
self._check_start_end_times()
def __init__(self):
'''Provide the necessary lists containing message information.'''
Record.__init__(self)
# Fields which are required by the message format.
self._mandatory_fields = ["VMUUID", "SiteName"]
# This list allows us to specify the order of lines when we construct records.
self._msg_fields = ["RecordCreateTime", "VMUUID", "SiteName", "CloudComputeService", "MachineName",
"LocalUserId", "LocalGroupId", "GlobalUserName", "FQAN",
"Status", "StartTime", "EndTime", "SuspendDuration",
"WallDuration", "CpuDuration", "CpuCount",
"NetworkType", "NetworkInbound", "NetworkOutbound", "PublicIPCount",
"Memory", "Disk", "BenchmarkType", "Benchmark",
"StorageRecordId", "ImageId", "CloudType"]
# This list specifies the information that goes in the database.
self._db_fields = self._msg_fields[:9] + ['VO', 'VOGroup', 'VORole'] + self._msg_fields[9:]
self._all_fields = self._db_fields
self._ignored_fields = ["UpdateTime"]
# Fields which will have an integer stored in them
self._int_fields = [ "SuspendDuration", "WallDuration", "CpuDuration", "CpuCount",
"NetworkInbound", "NetworkOutbound", "PublicIPCount", "Memory", "Disk"]
self._float_fields = ['Benchmark']
self._datetime_fields = ["RecordCreateTime", "StartTime", "EndTime"]
def __init__(self):
'''Populate fields required to load the message.'''
Record.__init__(self)
# Fields which are required by the message format.
self._mandatory_fields = ["Site", "Month", "Year", "WallDuration",
"CpuDuration", "NormalisedWallDuration",
"NormalisedCpuDuration", "NumberOfJobs"]
# This list allows us to specify the order of lines when we construct
# records. We use the field "Infrastructure" rather than the previously
# used "InfrastructureType"
self._msg_fields = ["Site", "Month", "Year", "GlobalUserName", "VO",
"VOGroup", "VORole", "SubmitHost", "Infrastructure",
"NodeCount", "Processors", "EarliestEndTime",
"LatestEndTime", "WallDuration", "CpuDuration",
"NormalisedWallDuration", "NormalisedCpuDuration",
"NumberOfJobs"]
# Fields which will have an integer stored in them
self._int_fields = ["Month", "Year", "NodeCount", "Processors",
"WallDuration", "CpuDuration",
"NormalisedWallDuration", "NormalisedCpuDuration",
"NumberOfJobs"]
self._datetime_fields = ["EarliestEndTime", "LatestEndTime"]
self._ignored_fields = ["UpdateTime"]
# This list specifies the information that goes in the database.
self._db_fields = self._msg_fields
# All allowed fields.
self._all_fields = self._msg_fields
def test_set_field(self):
record = Record()
self.assertRaises(InvalidRecordException, record.set_field, 'Test',
'value')
record._db_fields = ['Test']
record.set_field('Test', 'value')
self.assertEqual(record._record_content['Test'], 'value')
def test_set_field(self):
record = Record()
self.assertRaises(InvalidRecordException,
record.set_field, 'Test', 'value')
record._db_fields = ['Test']
record.set_field('Test', 'value')
self.assertEqual(record._record_content['Test'], 'value')
def test_set_all(self):
record = Record()
self.assertRaises(InvalidRecordException, record.set_all,
{'Test': 'value'})
record._db_fields = ['Test']
record.set_all({'Test': 'value'})
self.assertEqual(record._record_content['Test'], 'value')
def test_set_all(self):
record = Record()
self.assertRaises(InvalidRecordException,
record.set_all, {'Test':'value'})
record._db_fields = ['Test']
record.set_all({'Test':'value'})
self.assertEqual(record._record_content['Test'], 'value')
def __init__(self):
'''
Initializer for GroupAttributeRecord
'''
Record.__init__(self)
self._db_fields = self.DB_FIELDS
self._mandatory_fields = self.MANDATORY_FIELDS
self._all_fields = self._db_fields
def __init__(self):
'''
Initializes ProcessedRecord
'''
Record.__init__(self)
self._db_fields = ["HostName", "FileName", "Hash", "StopLine", "Parsed"]
self._int_fields = ["StopLine", "Parsed"]
self._all_fields = self._db_fields
def __init__(self):
'''
Initializes ProcessedRecord
'''
Record.__init__(self)
self._db_fields = [
"HostName", "FileName", "Hash", "StopLine", "Parsed"
]
self._int_fields = ["StopLine", "Parsed"]
self._all_fields = self._db_fields
def __init__(self):
'''Provide the necessary lists containing message information.'''
Record.__init__(self)
# Fields which are required by the message format.
self._mandatory_fields = [
"Site", "LocalJobId", "WallDuration", "CpuDuration", "StartTime",
"EndTime"
]
# This list allows us to specify the order of lines when we construct records.
self._msg_fields = [
"Site", "SubmitHost", "MachineName", "Queue", "LocalJobId",
"LocalUserId", "GlobalUserName", "FQAN", "VO", "VOGroup", "VORole",
"WallDuration", "CpuDuration", "Processors", "NodeCount",
"StartTime", "EndTime", "InfrastructureDescription",
"InfrastructureType", "MemoryReal", "MemoryVirtual",
"ServiceLevelType", "ServiceLevel"
]
# This list specifies the information that goes in the database.
self._db_fields = [
"Site", "SubmitHost", "MachineName", "Queue", "LocalJobId",
"LocalUserId", "GlobalUserName", "FQAN", "VO", "VOGroup", "VORole",
"WallDuration", "CpuDuration", "Processors", "NodeCount",
"StartTime", "EndTime", "InfrastructureDescription",
"InfrastructureType", "MemoryReal", "MemoryVirtual",
"ServiceLevelType", "ServiceLevel"
]
# Not used in the message format, but required by the database.
#self._fqan_fields = ["VO", "VOGroup", "VORole"]
# Fields which are accepted but currently ignored.
self._ignored_fields = ["SubmitHostType", "UpdateTime"]
self._all_fields = self._msg_fields + self._ignored_fields
# Fields which will have an integer stored in them
self._int_fields = [
"WallDuration", "CpuDuration", "Processors", "NodeCount",
"MemoryReal", "MemoryVirtual"
]
self._float_fields = ["ServiceLevel"]
self._datetime_fields = ["StartTime", "EndTime"]
# Acceptable values for the ServiceLevelType field, not case-sensitive
self._valid_slts = ["si2k", "hepspec"]
def __init__(self):
Record.__init__(self)
self._mandatory_fields = []
self._db_fields = ["Site", "JobName", "LocalUserID", "LocalUserGroup",
"WallDuration", "CpuDuration", "StartTime", "StopTime", "Infrastructure",
"MachineName", "Queue", "MemoryReal", "MemoryVirtual", "Processors", "NodeCount"]
self._all_fields = self._db_fields
self._int_fields = ["WallDuration", "CpuDuration", "MemoryReal",
"MemoryVirtual", "Processors", "NodeCount"]
self._datetime_fields = ["StartTime", "StopTime"]
def get_db_tuple(self, source=None):
"""
Return record contents as tuple ignoring the 'source' keyword argument.
The source (DN of the sender) isn't used in this record type currently.
"""
return Record.get_db_tuple(self)
def __init__(self):
'''Provide the necessary lists containing message information.'''
Record.__init__(self)
# Fields which are required by the message format.
self._mandatory_fields = StorageRecord.MANDATORY_FIELDS
# This list specifies the information that goes in the database.
self._db_fields = StorageRecord.DB_FIELDS
# Fields which are accepted but currently ignored.
self._ignored_fields = []
self._all_fields = self._db_fields
self._datetime_fields = ["CreateTime", "StartTime", "EndTime"]
# Fields which will have an integer stored in them
self._int_fields = ["FileCount", "ResourceCapacityUsed", "LogicalCapacityUsed", "ResourceCapacityAllocated"]
def __init__(self):
'''Provide the necessary lists containing message information.'''
Record.__init__(self)
# Fields which are required by the message format.
self._mandatory_fields = ["Site", "LocalJobId",
"WallDuration", "CpuDuration",
"StartTime", "EndTime"]
# This list allows us to specify the order of lines when we construct records.
self._msg_fields = ["Site", "SubmitHost", "MachineName", "Queue", "LocalJobId", "LocalUserId",
"GlobalUserName", "FQAN", "VO", "VOGroup", "VORole", "WallDuration",
"CpuDuration", "Processors", "NodeCount", "StartTime", "EndTime", "InfrastructureDescription", "InfrastructureType",
"MemoryReal", "MemoryVirtual", "ServiceLevelType",
"ServiceLevel"]
# This list specifies the information that goes in the database.
self._db_fields = ["Site", "SubmitHost", "MachineName", "Queue", "LocalJobId", "LocalUserId",
"GlobalUserName", "FQAN", "VO",
"VOGroup", "VORole", "WallDuration", "CpuDuration", "Processors",
"NodeCount", "StartTime", "EndTime", "InfrastructureDescription", "InfrastructureType", "MemoryReal",
"MemoryVirtual", "ServiceLevelType", "ServiceLevel"]
# Not used in the message format, but required by the database.
# self._fqan_fields = ["VO", "VOGroup", "VORole"]
# Fields which are accepted but currently ignored.
self._ignored_fields = ["SubmitHostType", "UpdateTime"]
self._all_fields = self._msg_fields + self._ignored_fields
# Fields which will have an integer stored in them
self._int_fields = ["WallDuration", "CpuDuration", "Processors",
"NodeCount", "MemoryReal",
"MemoryVirtual"]
self._float_fields = ["ServiceLevel"]
self._datetime_fields = ["StartTime", "EndTime"]
# Acceptable values for the ServiceLevelType field, not case-sensitive
self._valid_slts = ["si2k", "hepspec"]
def __init__(self):
'''Populate fields required to load the message.'''
Record.__init__(self)
# Fields which are required by the message format.
self._mandatory_fields = ["Site", "NumberOfJobs", "Month", "Year"]
# This list allows us to specify the order of lines when we construct
# records.
self._msg_fields = ['Site', 'SubmitHost', 'NumberOfJobs', 'Month', 'Year']
# Fields which will have an integer stored in them
self._int_fields = ["NumberOfJobs", "Month", "Year"]
# This list specifies the information that goes in the database.
self._db_fields = self._msg_fields
# All allowed fields.
self._all_fields = self._msg_fields
def __init__(self):
'''Provide the necessary lists containing message information.'''
Record.__init__(self)
# Fields which are required by the message format.
self._mandatory_fields = StorageRecord.MANDATORY_FIELDS
# This list specifies the information that goes in the database.
self._db_fields = StorageRecord.DB_FIELDS
# Fields which are accepted but currently ignored.
self._ignored_fields = []
self._all_fields = self._db_fields
self._datetime_fields = ["CreateTime", "StartTime", "EndTime"]
# Fields which will have an integer stored in them
self._int_fields = [
"FileCount", "ResourceCapacityUsed", "LogicalCapacityUsed",
"ResourceCapacityAllocated"
]
def _check_fields(self):
'''
Add extra checks to those made in every record.
'''
# First, call the parent's version.
Record._check_fields(self)
# Extract the relevant information from the user fqan.
# Keep the fqan itself as other methods in the class use it.
role, group, vo = parse_fqan(self._record_content['FQAN'])
# We can't / don't put NULL in the database, so we use 'None'
if role == None:
role = 'None'
if group == None:
group = 'None'
if vo == None:
vo = 'None'
self._record_content['VORole'] = role
self._record_content['VOGroup'] = group
self._record_content['VO'] = vo
def _check_fields(self):
'''
Add extra checks to those made in every record.
'''
# First, call the parent's version.
Record._check_fields(self)
# Extract the relevant information from the user fqan.
# Keep the fqan itself as other methods in the class use it.
role, group, vo = parse_fqan(self._record_content['FQAN'])
# We can't / don't put NULL in the database, so we use 'None'
if role == None:
role = 'None'
if group == None:
group = 'None'
if vo == None:
vo = 'None'
self._record_content['VORole'] = role
self._record_content['VOGroup'] = group
self._record_content['VO'] = vo
def __init__(self):
'''Populate fields required to load the message.'''
Record.__init__(self)
# Fields which are required by the message format.
self._mandatory_fields = ["Site", "NumberOfJobs", "Month", "Year"]
# This list allows us to specify the order of lines when we construct
# records.
self._msg_fields = [
'Site', 'SubmitHost', 'NumberOfJobs', 'Month', 'Year'
]
# Fields which will have an integer stored in them
self._int_fields = ["NumberOfJobs", "Month", "Year"]
# This list specifies the information that goes in the database.
self._db_fields = self._msg_fields
# All allowed fields.
self._all_fields = self._msg_fields
def _check_fields(self):
'''
Add extra checks to those made in every record.
'''
# First, call the parent's version.
Record._check_fields(self)
# Extract the relevant information from the user fqan.
# Keep the fqan itself as other methods in the class use it.
role, group, vo = parse_fqan(self._record_content['FQAN'])
# We can't / don't put NULL in the database, so we use 'None'
if role is None:
role = 'None'
if group is None:
group = 'None'
if vo is None:
vo = 'None'
if self._record_content['Benchmark'] is None:
# If Benchmark is not present in the original record the
# parent Record class level type checking will set it to
# None. We can't pass None as a Benchmark as the field is
# NOT NULL in the database, so we set it to something
# meaningful. In this case the float 0.0.
self._record_content['Benchmark'] = 0.0
self._record_content['VORole'] = role
self._record_content['VOGroup'] = group
self._record_content['VO'] = vo
# If the message was missing a CpuCount, assume it used
# zero Cpus, to prevent a NULL being written into the column
# in the CloudRecords tables.
# Doing so would be a problem despite the CloudRecords
# table allowing it because the CloudSummaries table
# doesn't allow it, creating a problem at summariser time.
if self._record_content['CpuCount'] is None:
self._record_content['CpuCount'] = 0
def get_db_tuple(self, source):
'''
Last item in list is not usable for us.
'''
return Record.get_db_tuple(self, source)[:-1]
def _check_fields(self):
'''
Add extra checks to the ones in the parent class.
'''
# Call the parent's checks first.
Record._check_fields(self)
# shorthand
rc = self._record_content
month_start = None
month_end = None
# For storing the month and year for the subsequent month.
next_month_year = None
next_month = None
try:
# A bit convoluted for finding the first second in the next month.
if (int(rc['Month']) == 12):
next_month_year = int(rc['Year']) + 1
next_month = 1
else:
next_month_year = int(rc['Year'])
next_month = int(rc['Month']) + 1
month_start = datetime(int(rc['Year']), int(rc['Month']), 1)
month_end = datetime(next_month_year, next_month, 1)
except KeyError:
raise InvalidRecordException("Invalid values for month and/or year.")
except TypeError:
raise InvalidRecordException("Invalid values for month and/or year.")
except ValueError:
raise InvalidRecordException("Invalid values for month and/or year.")
# Check that the EarliestEndTime and LatestEndTime fall within the right
# month, and that EET < LET.
try:
earliest_end = rc['EarliestEndTime']
latest_end = rc['LatestEndTime']
if not (month_start <= earliest_end <= month_end):
raise InvalidRecordException("EarliestEndTime is not within stated month.")
if not (month_start <= latest_end <= month_end):
raise InvalidRecordException("LatestEndTime is not within stated month.")
if earliest_end > latest_end:
raise InvalidRecordException("LatestEndTime is earlier than EarliestEndTime.")
except TypeError:
# These two fields are not compulsory.
pass
# Check that the month isn't in the future
now = datetime.now()
if month_start > now:
raise InvalidRecordException("Month specified in record is in the future.")
if not 1 <= self._record_content['Month'] <= 12:
raise InvalidRecordException("Month value is out of range")
if self._record_content['WallDuration'] < 0:
raise InvalidRecordException("Negative WallDuration")
if self._record_content['CpuDuration'] < 0:
raise InvalidRecordException("Negative WallDuration")
def test_get_field(self):
record = Record()
record._db_fields = ['Test']
record._record_content['Test'] = 'value'
self.assertEqual(record.get_field('Test'), 'value')