def do_format(conf):
random_delay(conf)
if conf.input_definition_file:
computer = parse_computer_definition(conf, conf.input_definition_file)
else:
# no definition file, figure out computer
computer = parse_computer_xml(conf, conf.computer_xml)
computer.instance_root = conf.instance_root
computer.software_root = conf.software_root
conf.logger.info('Updating computer')
address = computer.getAddress(conf.create_tap)
computer.address = address['addr']
computer.netmask = address['netmask']
if conf.output_definition_file:
write_computer_definition(conf, computer)
computer.construct(alter_user=conf.alter_user,
alter_network=conf.alter_network,
create_tap=conf.create_tap)
if getattr(conf, 'certificate_repository_path', None):
mkdir_p(conf.certificate_repository_path, mode=0o700)
# Dumping and sending to the erp5 the current configuration
if not conf.dry_run:
computer.dump(path_to_xml=conf.computer_xml,
path_to_json=conf.computer_json,
logger=conf.logger)
conf.logger.info('Posting information to %r' % conf.master_url)
computer.send(conf)
conf.logger.info('slapos successfully prepared the computer.')
def run(self):
"""
Run the promise
This will first load the promise module (which will update process sys.path)
"""
try:
os.chdir(self.partition_folder)
promise_started = False
if self.uid and self.gid:
dropPrivileges(self.uid, self.gid, logger=self.logger)
mkdir_p(self.cache_folder)
if self.wrap_promise:
promise_instance = WrapPromise(self.argument_dict)
else:
self._createInitFile()
promise_module = self._loadPromiseModule()
promise_instance = promise_module.RunPromise(self.argument_dict)
if not hasattr(promise_instance, 'isAnomalyDetected') or not \
hasattr(promise_instance, 'isTested') or \
(promise_instance.isAnomalyDetected() and self.check_anomaly) or \
(promise_instance.isTested() and not self.check_anomaly):
# if the promise will run, we save execution timestamp
promise_started = True
self.updatePromiseCache(
WrapPromise if self.wrap_promise else promise_module.RunPromise,
promise_instance,
started=promise_started)
promise_instance.run(self.check_anomaly, self.allow_bang)
except Exception:
self.logger.error(traceback.format_exc())
raise
def writeFile(name, folder, date_scope, rows):
folder = os.path.join(folder, date_scope)
f = os.path.join(folder, "dump_%s.csv" % name)
if os.path.exists(f):
# File already exists, no reason to recreate it.
return
mkdir_p(folder, 0o755)
with open(f, "w") as file_io:
csv.writer(file_io).writerows(rows)
def start(self):
"""
run all promises in sequential ways
"""
if self.config.pid_path:
if os.path.exists(self.config.pid_path):
# Check if another run promise is running
with open(self.config.pid_path) as fpid:
try:
pid = int(fpid.read(6))
except ValueError:
pid = None
if pid and os.path.exists("/proc/" + str(pid)):
self.logger.warning("A process is already running with pid " + str(pid))
return []
with open(self.config.pid_path, 'w') as fpid:
fpid.write(str(os.getpid()))
if not self.config.partition_folder:
raise ValueError("Partition folder is not specified")
parameter_dict = {
'promise-timeout': self.config.promise_timeout or promise_timeout,
'promise-folder': os.path.join(self.config.partition_folder, 'etc', 'plugin'),
'legacy-promise-folder': os.path.join(self.config.partition_folder, 'etc', 'promise'),
'partition-folder': self.config.partition_folder,
'master-url': self.config.master_url,
'partition-cert': self.config.partition_cert,
'partition-key': self.config.partition_key,
'partition-id': self.config.partition_id,
'computer-id': self.config.computer_id,
'debug': self.config.debug,
'check-anomaly': self.config.check_anomaly,
'force': self.config.force,
'run-only-promise-list': [x for x in self.config.run_only.split(' ') if x]
}
if self.config.log_folder:
parameter_dict['log-folder'] = self.config.log_folder
else:
parameter_dict['log-folder'] = os.path.join(self.config.partition_folder,
PROMISE_LOG_FOLDER_NAME)
mkdir_p(parameter_dict['log-folder'])
promise_launcher = PromiseLauncher(
config=parameter_dict,
logger=self.logger,
dry_run=self.config.dry_run
)
self.logger.info("Checking promises...")
exit_code = 0
try:
promise_launcher.run()
except PromiseError, e:
# error was already logged
exit_code = 1
def do_configure_client(logger, master_url_web, token, config_path, master_url):
while not token:
token = input('Credential security token: ').strip()
# Check for existence of previous configuration, certificate or key files
# where we expect to create them. If so, ask the use to manually remove them.
if os.path.exists(config_path):
logger.critical('There is a file in %s. '
'Please remove it before creating a new configuration.', config_path)
sys.exit(1)
basedir = os.path.dirname(config_path)
if not os.path.isdir(basedir):
logger.debug('Creating directory %s', basedir)
mkdir_p(basedir, mode=0o700)
cert_path = os.path.join(basedir, 'client.crt')
if os.path.exists(cert_path):
logger.critical('There is a file in %s. '
'Please remove it before creating a new certificate.', cert_path)
sys.exit(1)
key_path = os.path.join(basedir, 'client.key')
if os.path.exists(key_path):
logger.critical('There is a file in %s. '
'Please remove it before creating a new key.', key_path)
sys.exit(1)
# retrieve a template for the configuration file
cfg = fetch_configuration_template()
cfg = re.sub('master_url = .*', 'master_url = %s' % master_url, cfg)
cfg = re.sub('cert_file = .*', 'cert_file = %s' % cert_path, cfg)
cfg = re.sub('key_file = .*', 'key_file = %s' % key_path, cfg)
# retrieve and parse the certicate and key
certificate, key = get_certificate_key_pair(logger, master_url_web, token)
# write everything
with os.fdopen(os.open(config_path, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600), 'w') as fout:
logger.debug('Writing configuration to %s', config_path)
fout.write(cfg)
with os.fdopen(os.open(cert_path, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600), 'w') as fout:
logger.debug('Writing certificate to %s', cert_path)
fout.write(certificate)
with os.fdopen(os.open(key_path, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600), 'w') as fout:
logger.debug('Writing key to %s', key_path)
fout.write(key)
logger.info('SlapOS client configuration written to %s', config_path)
def writeFile(name, folder, date_scope, rows):
if os.path.exists(
os.path.join(folder, "%s/dump_%s.csv" % (date_scope, name))):
# File already exists, no reason to recreate it.
return
mkdir_p(os.path.join(folder, date_scope), 0o755)
file_io = open(os.path.join(folder, "%s/dump_%s.csv" % (date_scope, name)), "w")
csv_output = csv.writer(file_io)
csv_output.writerows(rows)
file_io.close()
def writeFile(name, folder, date_scope, rows):
if os.path.exists(
os.path.join(folder, "%s/dump_%s.csv" % (date_scope, name))):
# File already exists, no reason to recreate it.
return
mkdir_p(os.path.join(folder, date_scope), 0o755)
file_io = open(os.path.join(folder, "%s/dump_%s.csv" % (date_scope, name)),
"w")
csv_output = csv.writer(file_io)
csv_output.writerows(rows)
file_io.close()
def _configureLogger(self):
self.logger = logging.getLogger(self.__name)
for handler in self.logger.handlers:
self.logger.removeHandler(handler)
if self.__log_folder is None:
self.__logger_buffer = six.StringIO()
logger_handler = logging.StreamHandler(self.__logger_buffer)
self.__log_file = None
else:
mkdir_p(self.__log_folder)
self.__log_file = os.path.join(
self.__log_folder,
'%s.log' % self.__title
)
logger_handler = logging.FileHandler(self.__log_file)
self.logger.setLevel(logging.DEBUG if self.__debug else logging.INFO)
logger_handler.setFormatter(
fmt=logging.Formatter("%(asctime)s - %(levelname)s - " +
self.__transaction_id + " - %(message)s",
datefmt="%Y-%m-%d %H:%M:%S")
)
self.logger.addHandler(logger_handler)
def fetch_parameter_dict(self, options, instance_root):
slap = slapos.slap.slap()
slap.initializeConnection(
options['url'],
options.get('key'),
options.get('cert'),
)
computer_partition = slap.registerComputerPartition(
options['computer'],
options['partition'],
)
parameter_dict = computer_partition.getInstanceParameterDict()
options['instance-state'] = computer_partition.getState()
# XXX: those are not partition parameters, strictly speaking.
# Make them available as individual section keys.
for his_key in (
'slap_software_type',
'slap_computer_partition_id',
'slap_computer_id',
'slap_software_release_url',
'slave_instance_list',
'timestamp',
):
try:
value = parameter_dict.pop(his_key)
except KeyError:
pass
else:
options[his_key.replace('_', '-')] = value
ipv4_set = set()
v4_add = ipv4_set.add
ipv6_set = set()
v6_add = ipv6_set.add
tap_set = set()
tap_add = tap_set.add
route_gw_set = set()
route_gw_add = route_gw_set.add
route_mask_set = set()
route_mask_add = route_mask_set.add
route_ipv4_set = set()
route_v4_add = route_ipv4_set.add
route_network_set = set()
route_net_add = route_network_set.add
for tap, ip in parameter_dict.pop('ip_list'):
tap_add(tap)
if valid_ipv4(ip):
v4_add(ip)
elif valid_ipv6(ip):
v6_add(ip)
# XXX: emit warning on unknown address type ?
if 'full_ip_list' in parameter_dict:
for item in parameter_dict.pop('full_ip_list'):
if len(item) == 5:
tap, ip, gw, netmask, network = item
if tap.startswith('route_'):
if valid_ipv4(gw):
route_gw_add(gw)
if valid_ipv4(netmask):
route_mask_add(netmask)
if valid_ipv4(ip):
route_v4_add(ip)
if valid_ipv4(network):
route_net_add(network)
options['ipv4'] = ipv4_set
options['ipv6'] = ipv6_set
# also export single ip values for those recipes that don't support sets.
if ipv4_set:
options['ipv4-random'] = list(ipv4_set)[0].encode('UTF-8')
if ipv6_set:
options['ipv6-random'] = list(ipv6_set)[0].encode('UTF-8')
if route_ipv4_set:
options['tap-ipv4'] = list(route_ipv4_set)[0].encode('UTF-8')
options['tap-network-information-dict'] = dict(
ipv4=route_ipv4_set,
netmask=route_mask_set,
gateway=route_gw_set,
network=route_network_set)
else:
options['tap-network-information-dict'] = {}
if route_gw_set:
options['tap-gateway'] = list(route_gw_set)[0].encode('UTF-8')
if route_mask_set:
options['tap-netmask'] = list(route_mask_set)[0].encode('UTF-8')
if route_network_set:
options['tap-network'] = list(route_network_set)[0].encode('UTF-8')
storage_home = options.get('storage-home')
storage_dict = {}
if storage_home and os.path.exists(storage_home) and \
os.path.isdir(storage_home):
for filename in os.listdir(storage_home):
storage_path = os.path.join(
storage_home, filename,
options['slap-computer-partition-id'])
if os.path.exists(storage_path) and os.path.isdir(
storage_path):
storage_link = os.path.join(instance_root, 'DATA',
filename)
mkdir_p(os.path.join(instance_root, 'DATA'))
if not os.path.lexists(storage_link):
os.symlink(storage_path, storage_link)
storage_dict[filename] = storage_link
options['storage-dict'] = storage_dict
options['tap'] = tap_set
return self._expandParameterDict(options, parameter_dict)
def do_collect(conf):
"""
Main function
The idea here is to poll system every so many seconds
For each poll, we get a list of Snapshots, holding informations about
processes. We iterate over that list to store datas on a per user basis:
Each user object is a dict, indexed on timestamp. We add every snapshot
matching the user so that we get informations for each users
"""
try:
collected_date, collected_time = _get_time()
user_dict = get_user_list(conf)
try:
for snapshot in current_state(user_dict):
if snapshot:
user_dict[snapshot.username].append(snapshot)
except (KeyboardInterrupt, SystemExit, NoSuchProcess):
raise
log_directory = "%s/var/data-log" % conf.get("slapos", "instance_root")
mkdir_p(log_directory, 0o755)
consumption_report_directory = "%s/var/consumption-report" % \
conf.get("slapos", "instance_root")
mkdir_p(consumption_report_directory, 0o755)
xml_report_directory = "%s/var/xml_report/%s" % \
(conf.get("slapos", "instance_root"),
conf.get("slapos", "computer_id"))
mkdir_p(xml_report_directory, 0o755)
if stat.S_IMODE(os.stat(log_directory).st_mode) != 0o755:
os.chmod(log_directory, 0o755)
database = Database(log_directory)
if conf.has_option("slapformat", "computer_model_id"):
computer_model_id = conf.get("slapformat",
"computer_model_id")
else:
computer_model_id = "no_model"
uptime = _get_uptime()
if conf.has_option("slapformat", "heating_sensor_id"):
heating_sensor_id = conf.get("slapformat",
"heating_sensor_id")
database.connect()
test_heating = uptime is not None and \
uptime > datetime.timedelta(seconds=86400) and \
database.getLastHeatingTestTime() > uptime
database.close()
else:
heating_sensor_id = "no_sensor"
test_heating = False
computer = Computer(ComputerSnapshot(model_id=computer_model_id,
sensor_id = heating_sensor_id,
test_heating=test_heating))
computer.save(database, collected_date, collected_time)
for user in user_dict.values():
user.save(database, collected_date, collected_time)
SystemCSVReporterDumper(database).dump(log_directory)
RawCSVDumper(database).dump(log_directory)
consumption_report = ConsumptionReport(
computer_id=conf.get("slapos", "computer_id"),
user_list=get_user_list(conf),
database=database,
location=consumption_report_directory)
base = datetime.datetime.today()
for x in range(1, 3):
report_file = consumption_report.buildXMLReport(
(base - datetime.timedelta(days=x)).strftime("%Y-%m-%d"))
if report_file is not None:
shutil.copy(report_file, xml_report_directory)
compressLogFolder(log_directory)
# Drop older entries already reported
database.garbageCollect()
except AccessDenied:
print("You HAVE TO execute this script with root permission.")
def processComputerPartition(self, computer_partition):
"""
Process a Computer Partition, depending on its state
"""
computer_partition_id = computer_partition.getId()
# Sanity checks before processing
# Those values should not be None or empty string or any falsy value
if not computer_partition_id:
raise ValueError('Computer Partition id is empty.')
# Check if we defined explicit list of partitions to process.
# If so, if current partition not in this list, skip.
if len(self.computer_partition_filter_list) > 0 and \
(computer_partition_id not in self.computer_partition_filter_list):
return
self.logger.debug('Check if %s requires processing...' % computer_partition_id)
instance_path = os.path.join(self.instance_root, computer_partition_id)
# Try to get partition timestamp (last modification date)
timestamp_path = os.path.join(
instance_path,
COMPUTER_PARTITION_TIMESTAMP_FILENAME
)
parameter_dict = computer_partition.getInstanceParameterDict()
if 'timestamp' in parameter_dict:
timestamp = parameter_dict['timestamp']
else:
timestamp = None
try:
software_url = computer_partition.getSoftwareRelease().getURI()
except NotFoundError:
# Problem with instance: SR URI not set.
# Try to process it anyway, it may need to be deleted.
software_url = None
try:
software_path = os.path.join(self.software_root, md5digest(software_url))
except TypeError:
# Problem with instance: SR URI not set.
# Try to process it anyway, it may need to be deleted.
software_path = None
periodicity = self.maximum_periodicity
if software_path:
periodicity_path = os.path.join(software_path, 'periodicity')
if os.path.exists(periodicity_path):
try:
periodicity = int(open(periodicity_path).read())
except ValueError:
os.remove(periodicity_path)
self.logger.exception('')
# Check if timestamp from server is more recent than local one.
# If not: it's not worth processing this partition (nothing has
# changed).
if (computer_partition_id not in self.computer_partition_filter_list and
not self.develop and os.path.exists(timestamp_path)):
old_timestamp = open(timestamp_path).read()
last_runtime = int(os.path.getmtime(timestamp_path))
if timestamp:
try:
if periodicity == 0:
os.remove(timestamp_path)
elif int(timestamp) <= int(old_timestamp):
# Check periodicity, i.e if periodicity is one day, partition
# should be processed at least every day.
if int(time.time()) <= (last_runtime + periodicity) or periodicity < 0:
self.logger.debug('Partition already up-to-date, skipping.')
return
else:
# Periodicity forced processing this partition. Removing
# the timestamp file in case it fails.
os.remove(timestamp_path)
except ValueError:
os.remove(timestamp_path)
self.logger.exception('')
# Include Partition Logging
log_folder_path = "%s/.slapgrid/log" % instance_path
mkdir_p(log_folder_path)
partition_file_handler = logging.FileHandler(
filename="%s/instance.log" % (log_folder_path)
)
stat_info = os.stat(instance_path)
chownDirectory("%s/.slapgrid" % instance_path,
uid=stat_info.st_uid,
gid=stat_info.st_gid)
formatter = logging.Formatter(
'[%(asctime)s] %(levelname)-8s %(name)s %(message)s')
partition_file_handler.setFormatter(formatter)
self.logger.addHandler(partition_file_handler)
try:
self.logger.info('Processing Computer Partition %s.' % computer_partition_id)
self.logger.info(' Software URL: %s' % software_url)
self.logger.info(' Software path: %s' % software_path)
self.logger.info(' Instance path: %s' % instance_path)
filter_dict = getattr(computer_partition, '_filter_dict', None)
if filter_dict:
retention_delay = filter_dict.get('retention_delay', '0')
else:
retention_delay = '0'
local_partition = Partition(
software_path=software_path,
instance_path=instance_path,
supervisord_partition_configuration_path=os.path.join(
_getSupervisordConfigurationDirectory(self.instance_root), '%s.conf' %
computer_partition_id),
supervisord_socket=self.supervisord_socket,
computer_partition=computer_partition,
computer_id=self.computer_id,
partition_id=computer_partition_id,
server_url=self.master_url,
software_release_url=software_url,
certificate_repository_path=self.certificate_repository_path,
buildout=self.buildout,
logger=self.logger,
retention_delay=retention_delay,
instance_min_free_space=self.instance_min_free_space,
instance_storage_home=self.instance_storage_home,
ipv4_global_network=self.ipv4_global_network,
)
computer_partition_state = computer_partition.getState()
# XXX this line breaks 37 tests
# self.logger.info(' Instance type: %s' % computer_partition.getType())
self.logger.info(' Instance status: %s' % computer_partition_state)
if computer_partition_state == COMPUTER_PARTITION_STARTED_STATE:
local_partition.install()
computer_partition.available()
local_partition.start()
self._checkPromises(computer_partition)
computer_partition.started()
elif computer_partition_state == COMPUTER_PARTITION_STOPPED_STATE:
try:
# We want to process the partition, even if stopped, because it should
# propagate the state to children if any.
local_partition.install()
computer_partition.available()
finally:
# Instance has to be stopped even if buildout/reporting is wrong.
local_partition.stop()
computer_partition.stopped()
elif computer_partition_state == COMPUTER_PARTITION_DESTROYED_STATE:
local_partition.stop()
try:
computer_partition.stopped()
except (SystemExit, KeyboardInterrupt):
computer_partition.error(traceback.format_exc(), logger=self.logger)
raise
except Exception:
pass
else:
error_string = "Computer Partition %r has unsupported state: %s" % \
(computer_partition_id, computer_partition_state)
computer_partition.error(error_string, logger=self.logger)
raise NotImplementedError(error_string)
finally:
self.logger.removeHandler(partition_file_handler)
# If partition has been successfully processed, write timestamp
if timestamp:
open(timestamp_path, 'w').write(timestamp)
def fetch_parameter_dict(self, options, instance_root):
"""Gather parameters about current computer and partition.
Use two sources of truth
1. SlapOS Master - for external computer/partition information
2. format.Partition.resource_file - for partition specific details
"""
slap = slapos.slap.slap()
slap.initializeConnection(
options['url'],
options.get('key'),
options.get('cert'),
)
computer_partition = slap.registerComputerPartition(
options['computer'],
options['partition'],
)
parameter_dict = computer_partition.getInstanceParameterDict()
options['instance-state'] = computer_partition.getState()
# XXX: those are not partition parameters, strictly speaking.
# Make them available as individual section keys.
for his_key in (
'slap_software_type',
'slap_computer_partition_id',
'slap_computer_id',
'slap_software_release_url',
'slave_instance_list',
'timestamp',
):
try:
value = parameter_dict.pop(his_key)
except KeyError:
pass
else:
options[his_key.replace('_', '-')] = value
# Get Instance and root instance title or return UNKNOWN if not set
options['instance-title'] = parameter_dict.pop('instance_title',
'UNKNOWN Instance')
options['root-instance-title'] = parameter_dict.pop(
'root_instance_title', 'UNKNOWN')
options['instance-guid'] = computer_partition.getInstanceGuid()
ipv4_set = set()
v4_add = ipv4_set.add
ipv6_set = set()
v6_add = ipv6_set.add
tap_set = set()
tap_add = tap_set.add
route_gw_set = set()
route_gw_add = route_gw_set.add
route_mask_set = set()
route_mask_add = route_mask_set.add
route_ipv4_set = set()
route_v4_add = route_ipv4_set.add
route_network_set = set()
route_net_add = route_network_set.add
for tap, ip in parameter_dict.pop('ip_list'):
tap_add(tap)
if valid_ipv4(ip):
v4_add(ip)
elif valid_ipv6(ip):
v6_add(ip)
# XXX: emit warning on unknown address type ?
if 'full_ip_list' in parameter_dict:
for item in parameter_dict.pop('full_ip_list'):
if len(item) == 5:
tap, ip, gw, netmask, network = item
if tap.startswith('route_'):
if valid_ipv4(gw):
route_gw_add(gw)
if valid_ipv4(netmask):
route_mask_add(netmask)
if valid_ipv4(ip):
route_v4_add(ip)
if valid_ipv4(network):
route_net_add(network)
options['ipv4'] = ipv4_set
options['ipv6'] = ipv6_set
# also export single ip values for those recipes that don't support sets.
if ipv4_set:
options['ipv4-random'] = min(ipv4_set)
if ipv6_set:
options['ipv6-random'] = min(ipv6_set)
storage_home = options.get('storage-home')
storage_dict = {}
if storage_home and os.path.exists(storage_home) and \
os.path.isdir(storage_home):
for filename in os.listdir(storage_home):
storage_path = os.path.join(
storage_home, filename,
options['slap-computer-partition-id'])
if os.path.exists(storage_path) and os.path.isdir(
storage_path):
storage_link = os.path.join(instance_root, 'DATA',
filename)
mkdir_p(os.path.join(instance_root, 'DATA'))
if not os.path.lexists(storage_link):
os.symlink(storage_path, storage_link)
storage_dict[filename] = storage_link
options['storage-dict'] = storage_dict
# The external information transfered from Slap Master has been processed
# so we extend with information gathered from partition resource file
if hasattr(slapformat.Partition, "resource_file"):
resource_home = instance_root
while not os.path.exists(
os.path.join(resource_home,
slapformat.Partition.resource_file)):
resource_home = os.path.normpath(
os.path.join(resource_home, '..'))
if resource_home == "/":
break
else:
# no break happened - let's add partition resources into options
logger.debug("Using partition resource file {}".format(
os.path.join(resource_home,
slapformat.Partition.resource_file)))
with open(
os.path.join(
resource_home,
slapformat.Partition.resource_file)) as fi:
partition_params = json.load(fi)
# be very careful with overriding master's information
for key, value in flatten_dict(partition_params).items():
if key not in options:
if six.PY2 and isinstance(value, unicode):
value = value.encode('UTF-8')
options[key] = value
# print out augmented options to see what we are passing
logger.debug(str(options))
return self._expandParameterDict(options, parameter_dict)
def writeFile(self, name, folder, date_scope, rows): mkdir_p(os.path.join(folder, date_scope), 0o755) file_io = open(os.path.join(folder, "%s/dump_%s.csv" % (date_scope, name)), "w") csv_output = csv.writer(file_io) csv_output.writerows(rows) file_io.close()
def fetch_parameter_dict(self, options, instance_root):
slap = slapos.slap.slap()
slap.initializeConnection(
options['url'],
options.get('key'),
options.get('cert'),
)
computer_partition = slap.registerComputerPartition(
options['computer'],
options['partition'],
)
parameter_dict = computer_partition.getInstanceParameterDict()
options['instance-state'] = computer_partition.getState()
# XXX: those are not partition parameters, strictly speaking.
# Make them available as individual section keys.
for his_key in (
'slap_software_type',
'slap_computer_partition_id',
'slap_computer_id',
'slap_software_release_url',
'slave_instance_list',
'timestamp',
):
try:
value = parameter_dict.pop(his_key)
except KeyError:
pass
else:
options[his_key.replace('_', '-')] = value
ipv4_set = set()
v4_add = ipv4_set.add
ipv6_set = set()
v6_add = ipv6_set.add
tap_set = set()
tap_add = tap_set.add
route_gw_set = set()
route_gw_add = route_gw_set.add
route_mask_set = set()
route_mask_add = route_mask_set.add
route_ipv4_set = set()
route_v4_add = route_ipv4_set.add
route_network_set = set()
route_net_add = route_network_set.add
for tap, ip in parameter_dict.pop('ip_list'):
tap_add(tap)
if valid_ipv4(ip):
v4_add(ip)
elif valid_ipv6(ip):
v6_add(ip)
# XXX: emit warning on unknown address type ?
if 'full_ip_list' in parameter_dict:
for item in parameter_dict.pop('full_ip_list'):
if len(item) == 5:
tap, ip, gw, netmask, network = item
if tap.startswith('route_'):
if valid_ipv4(gw):
route_gw_add(gw)
if valid_ipv4(netmask):
route_mask_add(netmask)
if valid_ipv4(ip):
route_v4_add(ip)
if valid_ipv4(network):
route_net_add(network)
options['ipv4'] = ipv4_set
options['ipv6'] = ipv6_set
# also export single ip values for those recipes that don't support sets.
if ipv4_set:
options['ipv4-random'] = list(ipv4_set)[0].encode('UTF-8')
if ipv6_set:
options['ipv6-random'] = list(ipv6_set)[0].encode('UTF-8')
if route_ipv4_set:
options['tap-ipv4'] = list(route_ipv4_set)[0].encode('UTF-8')
options['tap-network-information-dict'] = dict(ipv4=route_ipv4_set,
netmask=route_mask_set,
gateway=route_gw_set,
network=route_network_set)
else:
options['tap-network-information-dict'] = {}
if route_gw_set:
options['tap-gateway'] = list(route_gw_set)[0].encode('UTF-8')
if route_mask_set:
options['tap-netmask'] = list(route_mask_set)[0].encode('UTF-8')
if route_network_set:
options['tap-network'] = list(route_network_set)[0].encode('UTF-8')
storage_home = options.get('storage-home')
storage_dict = {}
if storage_home and os.path.exists(storage_home) and \
os.path.isdir(storage_home):
for filename in os.listdir(storage_home):
storage_path = os.path.join(storage_home, filename,
options['slap-computer-partition-id'])
if os.path.exists(storage_path) and os.path.isdir(storage_path):
storage_link = os.path.join(instance_root, 'DATA', filename)
mkdir_p(os.path.join(instance_root, 'DATA'))
if not os.path.lexists(storage_link):
os.symlink(storage_path, storage_link)
storage_dict[filename] = storage_link
options['storage-dict'] = storage_dict
options['tap'] = tap_set
return self._expandParameterDict(options, parameter_dict)
def fetch_parameter_dict(self, options, instance_root):
"""Gather parameters about current computer and partition.
Use two sources of truth
1. SlapOS Master - for external computer/partition information
2. format.Partition.resource_file - for partition specific details
"""
slap = slapos.slap.slap()
slap.initializeConnection(
options['url'],
options.get('key'),
options.get('cert'),
)
computer_partition = slap.registerComputerPartition(
options['computer'],
options['partition'],
)
parameter_dict = computer_partition.getInstanceParameterDict()
options['instance-state'] = computer_partition.getState()
# XXX: those are not partition parameters, strictly speaking.
# Make them available as individual section keys.
for his_key in (
'slap_software_type',
'slap_computer_partition_id',
'slap_computer_id',
'slap_software_release_url',
'slave_instance_list',
'timestamp',
):
try:
value = parameter_dict.pop(his_key)
except KeyError:
pass
else:
options[his_key.replace('_', '-')] = value
# Get Instance and root instance title or return UNKNOWN if not set
options['instance-title'] = parameter_dict.pop('instance_title',
'UNKNOWN Instance').encode('UTF-8')
options['root-instance-title'] = parameter_dict.pop('root_instance_title',
'UNKNOWN').encode('UTF-8')
options['instance-guid'] = computer_partition.getInstanceGuid() \
.encode('UTF-8')
ipv4_set = set()
v4_add = ipv4_set.add
ipv6_set = set()
v6_add = ipv6_set.add
tap_set = set()
tap_add = tap_set.add
route_gw_set = set()
route_gw_add = route_gw_set.add
route_mask_set = set()
route_mask_add = route_mask_set.add
route_ipv4_set = set()
route_v4_add = route_ipv4_set.add
route_network_set = set()
route_net_add = route_network_set.add
for tap, ip in parameter_dict.pop('ip_list'):
tap_add(tap)
if valid_ipv4(ip):
v4_add(ip)
elif valid_ipv6(ip):
v6_add(ip)
# XXX: emit warning on unknown address type ?
if 'full_ip_list' in parameter_dict:
for item in parameter_dict.pop('full_ip_list'):
if len(item) == 5:
tap, ip, gw, netmask, network = item
if tap.startswith('route_'):
if valid_ipv4(gw):
route_gw_add(gw)
if valid_ipv4(netmask):
route_mask_add(netmask)
if valid_ipv4(ip):
route_v4_add(ip)
if valid_ipv4(network):
route_net_add(network)
options['ipv4'] = ipv4_set
options['ipv6'] = ipv6_set
# also export single ip values for those recipes that don't support sets.
if ipv4_set:
options['ipv4-random'] = list(ipv4_set)[0].encode('UTF-8')
if ipv6_set:
options['ipv6-random'] = list(ipv6_set)[0].encode('UTF-8')
storage_home = options.get('storage-home')
storage_dict = {}
if storage_home and os.path.exists(storage_home) and \
os.path.isdir(storage_home):
for filename in os.listdir(storage_home):
storage_path = os.path.join(storage_home, filename,
options['slap-computer-partition-id'])
if os.path.exists(storage_path) and os.path.isdir(storage_path):
storage_link = os.path.join(instance_root, 'DATA', filename)
mkdir_p(os.path.join(instance_root, 'DATA'))
if not os.path.lexists(storage_link):
os.symlink(storage_path, storage_link)
storage_dict[filename] = storage_link
options['storage-dict'] = storage_dict
# The external information transfered from Slap Master has been processed
# so we extend with information gathered from partition resource file
if hasattr(slapformat.Partition, "resource_file"):
resource_home = instance_root
while not os.path.exists(os.path.join(resource_home, slapformat.Partition.resource_file)):
resource_home = os.path.normpath(os.path.join(resource_home, '..'))
if resource_home == "/":
break
else:
# no break happened - let's add partition resources into options
logger.debug("Using partition resource file {}".format(
os.path.join(resource_home, slapformat.Partition.resource_file)))
with open(os.path.join(resource_home, slapformat.Partition.resource_file)) as fi:
partition_params = json.load(fi)
# be very careful with overriding master's information
for key, value in flatten_dict(partition_params).items():
if key not in options:
if isinstance(value, unicode):
value = value.encode('UTF-8')
options[key] = value
# print out augmented options to see what we are passing
logger.debug(str(options))
return self._expandParameterDict(options, parameter_dict)
def __init__(self, config=None, logger=None, dry_run=False):
"""
Promise launcher will run promises
@param config_file: A file containing configurations
@param dry_run: Only run all promises without save the result
@param logger: Set the logger to use, if None a logger will be configured
to console.
@param config: A configuration dict to use. Values send here will
overwrite configs from `config_file`. Expected values in config are:
promise-timeout
Maximum promise execution time before timeout. Default: 20
partition-folder
Base path of the partition
promise-folder
Promises folder, all promises scripts will be imported from that folder
legacy-promise-folder
Legacy promises folder, where to find bash, shell and standard promises
log-folder
Folder where promises will write logs. Can be None
check-anomaly
Ask to check anomaly instead of test. Default: False
debug
Configure loggin in debug mode. Default: True
master-url
SlapOS Master service URL
partition-cert
Computer Partition Certificate file
partition-key
Computer Partition key file
partition-id
Computer Partition ID, ex: slappart13
computer-id
Computer ID, ex: COMP-1234
uid
User UID
gid
User GID
debug
If True, show Promise consumption and execution time information, etc
run-only-promise-list
A list of promise from plugins directory that will be executed
force
Set to True if force run promises without check their periodicity
"""
self.dry_run = dry_run
self.__config = {
'promise-timeout': 20,
'promise-folder': None,
'legacy-promise-folder': None,
'log-folder': None,
'partition-folder': None,
'debug': False,
'uid': None,
'gid': None,
'master-url': None,
'partition-cert': None,
'partition-key': None,
'partition-id': None,
'computer-id': None,
'check-anomaly': False,
'force': False,
'run-only-promise-list': None
}
if config is not None:
self.__config.update(config)
for key, value in self.__config.items():
setattr(self, key.replace('-', '_'), value or None)
if self.promise_folder is None:
raise ValueError("Promise folder is missing in configuration!")
if self.partition_folder is None:
raise ValueError("Partition folder is missing in configuration!")
if logger is None:
self.logger = logging.getLogger(__name__)
self.logger.setLevel(logging.DEBUG if self.debug else logging.INFO)
if len(self.logger.handlers) == 0 or \
not isinstance(self.logger.handlers[0], logging.StreamHandler):
handler = logging.StreamHandler()
handler.setFormatter(
logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
)
self.logger.addHandler(handler)
else:
self.logger = logger
self.queue_result = MQueue()
self.bang_called = False
self._skipped_amount = 0
self.promise_output_dir = os.path.join(
self.partition_folder,
PROMISE_RESULT_FOLDER_NAME
)
if not os.path.exists(self.promise_output_dir):
mkdir_p(self.promise_output_dir)
self._updateFolderOwner()
