def _configure_db_confs(self):
LOG.info("Fixing up %s mysql configs.", colorizer.quote(self.distro.name))
my_cnf = ini_parser.RewritableConfigParser(fns=[DBInstaller.MYSQL_CONF])
my_cnf.remove_option('mysqld', 'skip-grant-tables')
my_cnf.set('mysqld', 'default-storage-engine', 'InnoDB')
my_cnf.set('mysqld', 'bind-address', '0.0.0.0')
sh.write_file_and_backup(DBInstaller.MYSQL_CONF, my_cnf.stringify())
def _clean_pip_requires(self):
# Fixup these files if they exist (sometimes they have 'junk' in them)
req_fns = []
for fn in self.requires_files:
if not sh.isfile(fn):
continue
req_fns.append(fn)
if req_fns:
utils.log_iterable(req_fns, logger=LOG,
header="Adjusting %s pip 'requires' files" % (len(req_fns)))
for fn in req_fns:
new_lines = []
for line in sh.load_file(fn).splitlines():
s_line = line.strip()
if len(s_line) == 0:
continue
elif s_line.startswith("#"):
new_lines.append(s_line)
elif not self._filter_pip_requires_line(s_line):
new_lines.append(("# %s" % (s_line)))
else:
new_lines.append(s_line)
contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(), "\n".join(new_lines))
sh.write_file_and_backup(fn, contents)
return len(req_fns)
def replace_forced_requirements(fn, forced_by_key):
old_lines = sh.load_file(fn).splitlines()
new_lines = []
alterations = []
for line in old_lines:
try:
source_req = pip_helper.extract_requirement(line)
except (ValueError, TypeError):
pass
else:
if source_req:
validate_requirement(fn, source_req)
try:
replace_req = forced_by_key[source_req.key]
except KeyError:
pass
else:
replace_req = str(replace_req)
source_req = str(source_req)
if replace_req != source_req:
line = replace_req
alterations.append("%s => %s"
% (colorizer.quote(source_req),
colorizer.quote(replace_req)))
new_lines.append(line)
if alterations:
contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(), "\n".join(new_lines))
sh.write_file_and_backup(fn, contents)
utils.log_iterable(alterations,
logger=LOG,
header="Replaced %s requirements in %s"
% (len(alterations), fn),
color=None)
return len(alterations)
def _clean_pip_requires(self):
# Fixup these files if they exist (sometimes they have 'junk' in them)
req_fns = []
for fn in self.requires_files:
if not sh.isfile(fn):
continue
req_fns.append(fn)
if req_fns:
utils.log_iterable(req_fns, logger=LOG,
header="Adjusting %s pip 'requires' files" % (len(req_fns)))
for fn in req_fns:
new_lines = []
for line in sh.load_file(fn).splitlines():
s_line = line.strip()
if len(s_line) == 0:
continue
elif s_line.startswith("#"):
new_lines.append(s_line)
elif not self._filter_pip_requires_line(fn, s_line):
new_lines.append(("# %s" % (s_line)))
else:
new_lines.append(s_line)
contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(), "\n".join(new_lines))
sh.write_file_and_backup(fn, contents)
return len(req_fns)
def _configure_db_confs(self):
LOG.info("Fixing up %s mysql configs.", colorizer.quote(self.distro.name))
new_lines = []
for line in sh.load_file(DBInstaller.MYSQL_CONF).splitlines():
if line.startswith('skip-grant-tables'):
new_lines.append('#' + line)
elif line.startswith('bind-address'):
new_lines.append('#' + line)
new_lines.append('bind-address = 0.0.0.0')
else:
new_lines.append(line)
sh.write_file_and_backup(DBInstaller.MYSQL_CONF, utils.joinlinesep(*new_lines))
def _configure_db_confs(self):
LOG.info("Fixing up %s mysql configs.", colorizer.quote(self.distro.name))
new_lines = []
for line in sh.load_file(DBInstaller.MYSQL_CONF).splitlines():
if line.startswith('skip-grant-tables'):
new_lines.append('#' + line)
elif line.startswith('bind-address'):
new_lines.append('#' + line)
new_lines.append('bind-address = 0.0.0.0')
else:
new_lines.append(line)
with sh.Rooted(True):
sh.write_file_and_backup(DBInstaller.MYSQL_CONF, utils.joinlinesep(*new_lines))
def _clean_pip_requires(self):
# Fixup these files if they exist, sometimes they have 'junk' in them
# that anvil will install instead of pip or setup.py and we don't want
# the setup.py file to attempt to install said dependencies since it
# typically picks locations that either are not what we desire or if
# said file contains editables, it may even pick external source directories
# which is what anvil is setting up as well...
req_fns = [f for f in self.requires_files if sh.isfile(f)]
if req_fns:
utils.log_iterable(req_fns, logger=LOG, header="Adjusting %s pip 'requires' files" % (len(req_fns)))
for fn in req_fns:
old_lines = sh.load_file(fn).splitlines()
new_lines = self._filter_pip_requires(fn, old_lines)
contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(), "\n".join(new_lines))
sh.write_file_and_backup(fn, contents)
return len(req_fns)
def _config_fix_httpd(self):
LOG.info("Fixing up: %s", colorizer.quote(HorizonInstaller.HTTPD_CONF))
(user, group) = self._get_apache_user_group()
new_lines = []
for line in sh.load_file(HorizonInstaller.HTTPD_CONF).splitlines():
# Directives in the configuration files are case-insensitive,
# but arguments to directives are often case sensitive...
# NOTE(harlowja): we aren't handling multi-line fixups...
if re.match(r"^\s*User\s+(.*)$", line, re.I):
line = "User %s" % (user)
if re.match(r"^\s*Group\s+(.*)$", line, re.I):
line = "Group %s" % (group)
if re.match(r"^\s*Listen\s+(.*)$", line, re.I):
line = "Listen 0.0.0.0:80"
new_lines.append(line)
sh.write_file_and_backup(HorizonInstaller.HTTPD_CONF, utils.joinlinesep(*new_lines))
def _config_fix_httpd(self):
LOG.info("Fixing up: %s", colorizer.quote(HorizonInstaller.HTTPD_CONF))
(user, group) = self._get_apache_user_group()
new_lines = []
for line in sh.load_file(HorizonInstaller.HTTPD_CONF).splitlines():
# Directives in the configuration files are case-insensitive,
# but arguments to directives are often case sensitive...
# NOTE(harlowja): we aren't handling multi-line fixups...
if re.match("^\s*User\s+(.*)$", line, re.I):
line = "User %s" % (user)
if re.match("^\s*Group\s+(.*)$", line, re.I):
line = "Group %s" % (group)
if re.match("^\s*Listen\s+(.*)$", line, re.I):
line = "Listen 0.0.0.0:80"
new_lines.append(line)
with sh.Rooted(True):
sh.write_file_and_backup(HorizonInstaller.HTTPD_CONF, utils.joinlinesep(*new_lines))
def _clean_pip_requires(self, requires_files):
# Fixup incompatible dependencies
if not (requires_files and self.forced_packages):
return
utils.log_iterable(sorted(requires_files),
logger=LOG,
header="Adjusting %s pip 'requires' files" % (len(requires_files)))
forced_by_key = dict((pkg.key, pkg) for pkg in self.forced_packages)
for fn in requires_files:
old_lines = sh.load_file(fn).splitlines()
new_lines = []
for line in old_lines:
try:
req = pip_helper.extract_requirement(line)
new_lines.append(str(forced_by_key[req.key]))
except Exception:
# we don't force the package or it has a bad format
new_lines.append(line)
contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(), "\n".join(new_lines))
sh.write_file_and_backup(fn, contents)
def _clean_pip_requires(self):
# Fixup these files if they exist, sometimes they have 'junk' in them
# that anvil will install instead of pip or setup.py and we don't want
# the setup.py file to attempt to install said dependencies since it
# typically picks locations that either are not what we desire or if
# said file contains editables, it may even pick external source directories
# which is what anvil is setting up as well...
req_fns = [f for f in self.requires_files if sh.isfile(f)]
if req_fns:
utils.log_iterable(req_fns,
logger=LOG,
header="Adjusting %s pip 'requires' files" %
(len(req_fns)))
for fn in req_fns:
old_lines = sh.load_file(fn).splitlines()
new_lines = self._filter_pip_requires(fn, old_lines)
contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(),
"\n".join(new_lines))
sh.write_file_and_backup(fn, contents)
return len(req_fns)
def _clean_pip_requires(self, requires_files):
# Fixup incompatible dependencies
if not (requires_files and self.forced_packages):
return
utils.log_iterable(sorted(requires_files),
logger=LOG,
header="Adjusting %s pip 'requires' files" %
(len(requires_files)))
forced_by_key = dict((pkg.key, pkg) for pkg in self.forced_packages)
for fn in requires_files:
old_lines = sh.load_file(fn).splitlines()
new_lines = []
for line in old_lines:
try:
req = pip_helper.extract_requirement(line)
new_lines.append(str(forced_by_key[req.key]))
except Exception:
# we don't force the package or it has a bad format
new_lines.append(line)
contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(),
"\n".join(new_lines))
sh.write_file_and_backup(fn, contents)
def replace_forced_requirements(fn, forced_by_key):
old_lines = sh.load_file(fn).splitlines()
new_lines = []
alterations = []
for line in old_lines:
try:
source_req = pip_helper.extract_requirement(line)
except (ValueError, TypeError):
pass
else:
if source_req:
validate_requirement(fn, source_req)
try:
replace_req = forced_by_key[source_req.key]
except KeyError:
pass
else:
replace_req = str(replace_req)
source_req = str(source_req)
if replace_req != source_req:
line = replace_req
alterations.append(
"%s => %s" %
(colorizer.quote(source_req),
colorizer.quote(replace_req)))
new_lines.append(line)
if alterations:
contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(),
"\n".join(new_lines))
sh.write_file_and_backup(fn, contents)
utils.log_iterable(alterations,
logger=LOG,
header="Replaced %s requirements in %s" %
(len(alterations), fn),
color=None)
return len(alterations)
def _scan_pip_requires(self, requires_files):
def validate_requirement(filename, source_req):
install_egg = None
for egg_info in self._python_eggs:
if egg_info['name'] == source_req.key:
install_egg = egg_info
break
if not install_egg:
return
# Ensure what we are about to install/create will actually work
# with the desired version. If it is not compatible then we should
# abort and someone should update the tag/branch in the origin
# file (or fix it via some other mechanism).
if install_egg['version'] not in source_req:
msg = ("Can not satisfy '%s' with '%s', version"
" conflict found in %s")
raise exc.DependencyException(msg % (source_req,
install_egg['req'],
filename))
if not requires_files:
return
utils.log_iterable(sorted(requires_files),
logger=LOG,
header="Scanning %s pip 'requires' files" % (len(requires_files)))
forced_by_key = dict((pkg.key, pkg) for pkg in self.forced_packages)
mutations = 0
for fn in sorted(requires_files):
old_lines = sh.load_file(fn).splitlines()
new_lines = []
alterations = []
for line in old_lines:
try:
source_req = pip_helper.extract_requirement(line)
except (ValueError, TypeError):
pass
else:
if source_req:
validate_requirement(fn, source_req)
try:
replace_req = forced_by_key[source_req.key]
except KeyError:
pass
else:
replace_req = str(replace_req)
source_req = str(source_req)
if replace_req != source_req:
line = replace_req
alterations.append("%s => %s"
% (colorizer.quote(source_req),
colorizer.quote(replace_req)))
new_lines.append(line)
if alterations:
contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(), "\n".join(new_lines))
sh.write_file_and_backup(fn, contents)
mutations += len(alterations)
utils.log_iterable(alterations,
logger=LOG,
header="Replaced %s requirements in %s"
% (len(alterations), fn),
color=None)
# NOTE(imelnikov): after updating requirement lists we should re-fetch
# data from them again, so we drop pip helper caches here.
if mutations > 0:
pip_helper.drop_caches()
def _scan_pip_requires(self, requires_files):
def validate_requirement(filename, source_req):
install_egg = None
for egg_info in self._python_eggs:
if egg_info['name'] == source_req.key:
install_egg = egg_info
break
if not install_egg:
return
# Ensure what we are about to install/create will actually work
# with the desired version. If it is not compatible then we should
# abort and someone should update the tag/branch in the origin
# file (or fix it via some other mechanism).
if install_egg['version'] not in source_req:
msg = ("Can not satisfy '%s' with '%s', version"
" conflict found in %s")
raise exc.DependencyException(
msg % (source_req, install_egg['req'], filename))
if not requires_files:
return
utils.log_iterable(sorted(requires_files),
logger=LOG,
header="Scanning %s pip 'requires' files" %
(len(requires_files)))
forced_by_key = dict((pkg.key, pkg) for pkg in self.forced_packages)
mutations = 0
for fn in sorted(requires_files):
old_lines = sh.load_file(fn).splitlines()
new_lines = []
alterations = []
for line in old_lines:
try:
source_req = pip_helper.extract_requirement(line)
except (ValueError, TypeError):
pass
else:
if source_req:
validate_requirement(fn, source_req)
try:
replace_req = forced_by_key[source_req.key]
except KeyError:
pass
else:
replace_req = str(replace_req)
source_req = str(source_req)
if replace_req != source_req:
line = replace_req
alterations.append(
"%s => %s" %
(colorizer.quote(source_req),
colorizer.quote(replace_req)))
new_lines.append(line)
if alterations:
contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(),
"\n".join(new_lines))
sh.write_file_and_backup(fn, contents)
mutations += len(alterations)
utils.log_iterable(alterations,
logger=LOG,
header="Replaced %s requirements in %s" %
(len(alterations), fn),
color=None)
# NOTE(imelnikov): after updating requirement lists we should re-fetch
# data from them again, so we drop pip helper caches here.
if mutations > 0:
pip_helper.drop_caches()
