def _ClassifyFree(self, free, pool_name):
# Ensure free machines are unique.
free = setlib.make_dict(free)
free = free.keys()
free.sort()
inter = setlib.intersect(free, self._used_hosts)
if inter:
prodlib.log('Ignoring used machines: %s' % string.join(inter))
free = setlib.diff(free, inter)
# Preload hardware data.
self._mach_mgr.MachineList(free, load_hardware=1)
# Group free machines by their defining characteristics. This
# allows us to examine a smaller set of free machines as
# candidates for a replacement.
freedict = {}
# Prune out free without hardware information.
for host in free:
mach = self._mach_mgr.Machine(host)
if not mach or not mach.hardware(): continue
freedict.setdefault(mach.ClassString(), []).append(mach.name())
prodlib.log('Found %d free machs from %s.\n' % (len(free), pool_name))
return freedict
def _ClassifyFree(self, free, pool_name):
# Ensure free machines are unique.
free = setlib.make_dict(free)
free = free.keys()
free.sort()
inter = setlib.intersect(free, self._used_hosts)
if inter:
prodlib.log('Ignoring used machines: %s' % string.join(inter))
free = setlib.diff(free, inter)
# Preload hardware data.
self._mach_mgr.MachineList(free, load_hardware=1)
# Group free machines by their defining characteristics. This
# allows us to examine a smaller set of free machines as
# candidates for a replacement.
freedict = {}
# Prune out free without hardware information.
for host in free:
mach = self._mach_mgr.Machine(host)
if not mach or not mach.hardware(): continue
freedict.setdefault(mach.ClassString(), []).append(mach.name())
prodlib.log('Found %d free machs from %s.\n' % (len(free), pool_name))
return freedict
def _AllocateHostFromFreePool(self,
srv_mgr,
server,
pool,
free_dict,
force=0,
exclude=None):
"""
Allocate a machine for the passed in server from specific pool.
"""
cnstr_mgr = srv_mgr.constraint_mgr()
# Find currently used compatible machines.
used_hosts = {}
if self._used:
used_hosts = cnstr_mgr.Constraint('sharing').CompatibleHosts(
srv_mgr, server)
# Find free machine set from the free_dict - we get one of each class
# of machines and save the others in its class. We know that we can
# rank members of the same class with the same score.
free = {}
for (machclass, hosts) in free_dict.items():
if not hosts: continue
free[hosts[0]] = hosts
prodlib.log(' Allocating server for %s (used=%d, free=%d, pool=%s)' % \
(server, len(used_hosts), len(free), pool))
hosts = used_hosts.keys() + free.keys()
# Exclude optional excludes.
if self._exclude: hosts = setlib.diff(hosts, self._exclude)
# Exclude locally specified excludes.
if exclude: hosts = setlib.diff(hosts, exclude)
random.shuffle(hosts)
# Save the original host.
orig_host = server.host()
results = []
failed = []
# Assign weights and prune out ones that don't fit.
for host in hosts:
# Replace the server's host with the candidate host and verify.
if self._verbose: prodlib.log(' ranking candidate: %s' % host)
srv_mgr.ReplaceServer(server, host)
servers = [server] + used_hosts.get(host, [])
ver_results = cnstr_mgr.VerifyServer(srv_mgr,
server,
servers=servers,
force=force)
if self._verbose:
for res in ver_results:
if res.error(): status = 'fail'
else: status = 'ok'
prodlib.log(' %s: %s' % (status, res))
if ver_results[-1].error():
failed.append(host)
else:
# Compute total weight assigned to machine.
weight = 0.0
for res in ver_results:
weight = weight + res.weight()
if self._verbose:
prodlib.log(' weight: %.2f' % weight)
# Append results for machine. We augment the hosts with
# free machines of the same class since these should receive
# the same score.
if free.has_key(host):
for free_host in free[host]:
results.append((free_host, weight))
else:
results.append((host, weight))
# Sort the results by highest weight to find the best candidate.
results.sort(lambda x, y: -cmp(x[1], y[1]))
for (host, weight) in results:
prodlib.log(' Trying %s (%.2f)' % (host, weight))
# Set server to new host.
srv_mgr.ReplaceServer(server, host)
if not used_hosts.has_key(host):
# Remove the machine from the free list if necessary.
key = self._mach_mgr.Machine(host).ClassString()
hosts = free_dict[key]
hosts.remove(host)
if hosts == []: del (free_dict[key])
# Return the server with its newly allocated host.
prodlib.log(' Allocated %s (%.2f)' % (server, weight))
return server
# Failed so replace old host.
srv_mgr.ReplaceServer(server, orig_host)
prodlib.log(' Unable to allocate server.')
return None
def _AllocateHostFromFreePool(self, srv_mgr, server, pool,
free_dict, force=0, exclude=None):
"""
Allocate a machine for the passed in server from specific pool.
"""
cnstr_mgr = srv_mgr.constraint_mgr()
# Find currently used compatible machines.
used_hosts = {}
if self._used:
used_hosts = cnstr_mgr.Constraint('sharing').CompatibleHosts(srv_mgr,
server)
# Find free machine set from the free_dict - we get one of each class
# of machines and save the others in its class. We know that we can
# rank members of the same class with the same score.
free = {}
for (machclass, hosts) in free_dict.items():
if not hosts: continue
free[hosts[0]] = hosts
prodlib.log(' Allocating server for %s (used=%d, free=%d, pool=%s)' % \
(server, len(used_hosts), len(free), pool))
hosts = used_hosts.keys() + free.keys()
# Exclude optional excludes.
if self._exclude: hosts = setlib.diff(hosts, self._exclude)
# Exclude locally specified excludes.
if exclude: hosts = setlib.diff(hosts, exclude)
random.shuffle(hosts)
# Save the original host.
orig_host = server.host()
results = []
failed = []
# Assign weights and prune out ones that don't fit.
for host in hosts:
# Replace the server's host with the candidate host and verify.
if self._verbose: prodlib.log(' ranking candidate: %s' % host)
srv_mgr.ReplaceServer(server, host)
servers = [server] + used_hosts.get(host, [])
ver_results = cnstr_mgr.VerifyServer(srv_mgr, server,
servers=servers,
force=force)
if self._verbose:
for res in ver_results:
if res.error(): status = 'fail'
else: status = 'ok'
prodlib.log(' %s: %s' % (status, res))
if ver_results[-1].error():
failed.append(host)
else:
# Compute total weight assigned to machine.
weight = 0.0
for res in ver_results:
weight = weight + res.weight()
if self._verbose:
prodlib.log(' weight: %.2f' % weight)
# Append results for machine. We augment the hosts with
# free machines of the same class since these should receive
# the same score.
if free.has_key(host):
for free_host in free[host]:
results.append((free_host, weight))
else:
results.append((host, weight))
# Sort the results by highest weight to find the best candidate.
results.sort(lambda x,y: -cmp(x[1], y[1]))
for (host, weight) in results:
prodlib.log(' Trying %s (%.2f)' % (host, weight))
# Set server to new host.
srv_mgr.ReplaceServer(server, host)
if not used_hosts.has_key(host):
# Remove the machine from the free list if necessary.
key = self._mach_mgr.Machine(host).ClassString()
hosts = free_dict[key]
hosts.remove(host)
if hosts == []: del(free_dict[key])
# Return the server with its newly allocated host.
prodlib.log(' Allocated %s (%.2f)' % (server, weight))
return server
# Failed so replace old host.
srv_mgr.ReplaceServer(server, orig_host)
prodlib.log(' Unable to allocate server.')
return None
def LoadConfigData(config_file,
do_includes=0, seen_files=[], config_load_dir=None,
base_scope=None):
# We copy base scope (which we're going to exec over)
# since we're going to delete some protected vars that shouldn't
# be shared.
if base_scope is None: base_scope = {}
base_scope = copy.deepcopy(base_scope)
original_base_scope = copy.deepcopy(base_scope)
# Delete vars from the base_scope which should not be propagated.
# INCLUDES and CURRENT_CONFIG are deleted, since we don't want to
# be affected by earlier peoples includes when we actively process
# includes below. Servers defined in one file aren't made
# available either - at some point only TOP level files will have
# servers (except for glue files).
protected_vars = [ 'INCLUDES', 'CURRENT_CONFIG', 'SERVERS', ]
for var in protected_vars:
if base_scope.has_key(var): del base_scope[var]
# The files that we load
loaded_files = []
# Load the raw data over base_scope. This allows variables from
# earlier includes to be available and referenced in later includes.
base_data = config_namespace.ConfigNameSpace(base_scope)
if config_file:
try:
base_data.ExecFile(config_file, config_dir=config_load_dir)
if type(config_file) == types.StringType:
loaded_files.append(GetConfigFilePath(config_file,
config_dir=config_load_dir))
except IOError:
raise IOError('unable to load config %s' % config_file)
shared_ports = []
orig_servers = base_data.namespace.get('SERVERS', {})
# Find out directory of current config file.
if type(config_file) == types.StringType:
(config_dir, config_name) = os.path.split(config_file)
else:
config_name = '<>UNNAMED<>'
# We only allow certain variables in servers.* files.
allowed_vars = AllowedServerConfigVars() + \
GetPreloadedScope().keys()
if config_name[:len('servers.')] == 'servers.':
my_vars = setlib.diff(base_data.namespace.keys(),
original_base_scope.keys())
if setlib.diff(my_vars, allowed_vars):
raise RuntimeError('Server file %s has vars outside allowed set: %s.'
' bad vars: %s'
% (config_file, AllowedServerConfigVars(),
setlib.diff(my_vars, allowed_vars)))
# Clone seen_files to silence pycheck
seen_files = seen_files + [config_name]
update_data = config_namespace.ConfigNameSpace(base_data.namespace)
# INCLUDES is a list of files to include. For backwards compatibility
# support this as a dictionary as well. However, INCLUDES as a dictionary
# is DEPRECATED.
include_specs = []
if do_includes:
includes = base_data.namespace.get('INCLUDES', {})
if type(includes) == types.ListType:
tmp = {}
for source in includes:
include_specs.append((source, {'ownservers' : 1, 'include_vars' : 1}))
tmp[source] = { 'ownservers' : 1, 'include_vars' : 1 }
# For now convert it in the actual data to a dictionary.
# This is necessary if there are multiple levels of includes
# and we try to merge different types. Once we are no longer
# using the dictionary type include, then get rid of this
# and a lot of this other crap can be removed.
base_data.namespace['INCLUDES'] = tmp
else:
include_specs = includes.items()
# Extend includes with the active config file.
active_config = base_data.namespace.get('CURRENT_CONFIG')
if active_config:
include_specs.append((active_config,
{'ownservers' : 1, 'include_vars' : 1}))
for (source, options) in include_specs:
if source in seen_files:
raise RuntimeError("Detected loop while loading %s: %s already in %s" %
(config_name, source, string.join(seen_files)))
# Test if this is a service name.
if type(source) == types.TupleType:
colo = source[0]
source = source[1]
from google3.enterprise.legacy.production.babysitter import masterconfig
factory = masterconfig.Factory(colo, config_dir = config_load_dir)
source = factory.GetConfigFiles(source)[0]
# Load the included data and remove servers.
(include_data, _, lf) = LoadConfigData(source,
do_includes=do_includes,
seen_files=seen_files,
config_load_dir=config_load_dir,
base_scope=update_data.namespace)
# Update the loaded files list
for f in lf:
if f not in loaded_files: loaded_files.append(f)
include_servers = include_data.namespace.get('SERVERS', {})
if include_data.namespace.has_key('SERVERS'):
del include_data.namespace['SERVERS']
# Process the merging options.
server_option = options.get('ownservers', 0)
types_option = options.get('wantedtypes', [])
include_vars = options.get('include_vars', 1)
server_types = servertype.CollectTypes(types_option, {})
# Handle server merge.
for (port, servers) in include_servers.items():
typelvl = servertype.GetTypeLevel(port)
if not servertype.WantedType(typelvl, server_types): continue
if orig_servers.has_key(port):
raise RuntimeError("Attempt to overwrite %s servers (port %s) "
"for service %s." % (typelvl, port, source))
elif server_option == 0:
shared_ports.append(port)
orig_servers[port] = copy.deepcopy(servers)
# Handle variable merge.
if include_vars:
# Apply any included segment specific variables to the top level
# if this is a segment. This allows us to define in one file,
# all differing per-segment variables.
if base_data.namespace.has_key('SEGMENT_SET') and \
include_data.namespace.has_key('SEGMENT_SET_VARS'):
segment = base_data.namespace['SEGMENT_SET']
vars = include_data.namespace['SEGMENT_SET_VARS']
if not vars.has_key(segment):
raise RuntimeError('Segment %s data not found in SEGMENT_SET_VARS '
'for %s' % (segment, source))
vars = vars[segment]
# We change the variable values here for purposes of
# regression testing. This makes data.* files return
# standard values.
if segment_data.UseFakeData():
vars = segment_data.MakeFakeSetVars(vars, segment)
tmp_data = config_namespace.ConfigNameSpace(vars)
include_data.MergeNamespace(tmp_data, 1)
update_data.MergeNamespace(include_data, 1)
# Merge included data into the params which contains the defaults.
update_data.MergeNamespace(base_data, 1)
shared_ports.sort()
# Convert the final servers map into lists (if needed) and save it.
update_data.namespace['SERVERS'] = GetServers(orig_servers)
return (update_data, shared_ports, loaded_files)
def LoadConfigData(config_file,
do_includes=0,
seen_files=[],
config_load_dir=None,
base_scope=None):
# We copy base scope (which we're going to exec over)
# since we're going to delete some protected vars that shouldn't
# be shared.
if base_scope is None: base_scope = {}
base_scope = copy.deepcopy(base_scope)
original_base_scope = copy.deepcopy(base_scope)
# Delete vars from the base_scope which should not be propagated.
# INCLUDES and CURRENT_CONFIG are deleted, since we don't want to
# be affected by earlier peoples includes when we actively process
# includes below. Servers defined in one file aren't made
# available either - at some point only TOP level files will have
# servers (except for glue files).
protected_vars = [
'INCLUDES',
'CURRENT_CONFIG',
'SERVERS',
]
for var in protected_vars:
if base_scope.has_key(var): del base_scope[var]
# The files that we load
loaded_files = []
# Load the raw data over base_scope. This allows variables from
# earlier includes to be available and referenced in later includes.
base_data = config_namespace.ConfigNameSpace(base_scope)
if config_file:
try:
base_data.ExecFile(config_file, config_dir=config_load_dir)
if type(config_file) == types.StringType:
loaded_files.append(
GetConfigFilePath(config_file, config_dir=config_load_dir))
except IOError:
raise IOError('unable to load config %s' % config_file)
shared_ports = []
orig_servers = base_data.namespace.get('SERVERS', {})
# Find out directory of current config file.
if type(config_file) == types.StringType:
(config_dir, config_name) = os.path.split(config_file)
else:
config_name = '<>UNNAMED<>'
# We only allow certain variables in servers.* files.
allowed_vars = AllowedServerConfigVars() + \
GetPreloadedScope().keys()
if config_name[:len('servers.')] == 'servers.':
my_vars = setlib.diff(base_data.namespace.keys(),
original_base_scope.keys())
if setlib.diff(my_vars, allowed_vars):
raise RuntimeError(
'Server file %s has vars outside allowed set: %s.'
' bad vars: %s' % (config_file, AllowedServerConfigVars(),
setlib.diff(my_vars, allowed_vars)))
# Clone seen_files to silence pycheck
seen_files = seen_files + [config_name]
update_data = config_namespace.ConfigNameSpace(base_data.namespace)
# INCLUDES is a list of files to include. For backwards compatibility
# support this as a dictionary as well. However, INCLUDES as a dictionary
# is DEPRECATED.
include_specs = []
if do_includes:
includes = base_data.namespace.get('INCLUDES', {})
if type(includes) == types.ListType:
tmp = {}
for source in includes:
include_specs.append((source, {
'ownservers': 1,
'include_vars': 1
}))
tmp[source] = {'ownservers': 1, 'include_vars': 1}
# For now convert it in the actual data to a dictionary.
# This is necessary if there are multiple levels of includes
# and we try to merge different types. Once we are no longer
# using the dictionary type include, then get rid of this
# and a lot of this other crap can be removed.
base_data.namespace['INCLUDES'] = tmp
else:
include_specs = includes.items()
# Extend includes with the active config file.
active_config = base_data.namespace.get('CURRENT_CONFIG')
if active_config:
include_specs.append((active_config, {
'ownservers': 1,
'include_vars': 1
}))
for (source, options) in include_specs:
if source in seen_files:
raise RuntimeError(
"Detected loop while loading %s: %s already in %s" %
(config_name, source, string.join(seen_files)))
# Test if this is a service name.
if type(source) == types.TupleType:
colo = source[0]
source = source[1]
from google3.enterprise.legacy.production.babysitter import masterconfig
factory = masterconfig.Factory(colo, config_dir=config_load_dir)
source = factory.GetConfigFiles(source)[0]
# Load the included data and remove servers.
(include_data, _,
lf) = LoadConfigData(source,
do_includes=do_includes,
seen_files=seen_files,
config_load_dir=config_load_dir,
base_scope=update_data.namespace)
# Update the loaded files list
for f in lf:
if f not in loaded_files: loaded_files.append(f)
include_servers = include_data.namespace.get('SERVERS', {})
if include_data.namespace.has_key('SERVERS'):
del include_data.namespace['SERVERS']
# Process the merging options.
server_option = options.get('ownservers', 0)
types_option = options.get('wantedtypes', [])
include_vars = options.get('include_vars', 1)
server_types = servertype.CollectTypes(types_option, {})
# Handle server merge.
for (port, servers) in include_servers.items():
typelvl = servertype.GetTypeLevel(port)
if not servertype.WantedType(typelvl, server_types): continue
if orig_servers.has_key(port):
raise RuntimeError("Attempt to overwrite %s servers (port %s) "
"for service %s." % (typelvl, port, source))
elif server_option == 0:
shared_ports.append(port)
orig_servers[port] = copy.deepcopy(servers)
# Handle variable merge.
if include_vars:
# Apply any included segment specific variables to the top level
# if this is a segment. This allows us to define in one file,
# all differing per-segment variables.
if base_data.namespace.has_key('SEGMENT_SET') and \
include_data.namespace.has_key('SEGMENT_SET_VARS'):
segment = base_data.namespace['SEGMENT_SET']
vars = include_data.namespace['SEGMENT_SET_VARS']
if not vars.has_key(segment):
raise RuntimeError(
'Segment %s data not found in SEGMENT_SET_VARS '
'for %s' % (segment, source))
vars = vars[segment]
# We change the variable values here for purposes of
# regression testing. This makes data.* files return
# standard values.
if segment_data.UseFakeData():
vars = segment_data.MakeFakeSetVars(vars, segment)
tmp_data = config_namespace.ConfigNameSpace(vars)
include_data.MergeNamespace(tmp_data, 1)
update_data.MergeNamespace(include_data, 1)
# Merge included data into the params which contains the defaults.
update_data.MergeNamespace(base_data, 1)
shared_ports.sort()
# Convert the final servers map into lists (if needed) and save it.
update_data.namespace['SERVERS'] = GetServers(orig_servers)
return (update_data, shared_ports, loaded_files)