def _build_from_spec(self, instance, spec_filename, patches=None):
pkg_dir = instance.get_option('app_dir')
if sh.isfile(sh.joinpths(pkg_dir, "setup.py")):
self._write_python_tarball(instance, pkg_dir, ENSURE_NOT_MISSING)
else:
self._write_git_tarball(instance, pkg_dir, spec_filename)
self._copy_sources(instance)
if patches:
self._copy_patches(patches)
cmdline = [self.specprint_executable]
cmdline.extend(['-f', spec_filename])
spec_details = json.loads(sh.execute(cmdline)[0])
rpm_requires = []
for k in ('requires', 'requirenevrs'):
try:
rpm_requires.extend(spec_details['headers'][k])
except (KeyError, TypeError):
pass
if rpm_requires:
buff = six.StringIO()
buff.write("# %s\n" % instance.name)
if rpm_requires:
for req in rpm_requires:
buff.write("%s\n" % req)
buff.write("\n")
sh.append_file(self.rpm_build_requires_filename, buff.getvalue())
self._copy_startup_scripts(instance, spec_details)
cmdline = [
self.rpmbuild_executable,
"-bs",
"--define", "_topdir %s" % self.rpmbuild_dir,
spec_filename,
]
out_filename = sh.joinpths(self.log_dir, "rpmbuild-%s.log" % instance.name)
sh.execute_save_output(cmdline, out_filename)
def mark(self, phasename):
contents = dict()
contents['name'] = phasename
contents['when'] = date.rcf8222date()
yield phasename
LOG.debug("Marking the completion of phase %r in file %r", phasename, self.fn)
lines = [json.dumps(contents), '']
sh.append_file(self.fn, utils.joinlinesep(*lines))
def _record_srpm_files(self, files):
if not files:
return
buf = six.StringIO()
for f in files:
buf.write(f)
buf.write("\n")
if sh.isfile(self.generated_srpms_filename):
sh.append_file(self.generated_srpms_filename, "\n" + buf.getvalue())
else:
sh.write_file(self.generated_srpms_filename, buf.getvalue())
def update(self, fn):
current_vars = RcReader().extract(fn)
possible_vars = dict()
possible_vars.update(self._get_general_envs())
possible_vars.update(self._get_ec2_envs())
possible_vars.update(self._get_password_envs())
possible_vars.update(self._get_os_envs())
possible_vars.update(self._get_euca_envs())
possible_vars.update(self._get_nova_envs())
possible_vars.update(self._get_misc_envs())
new_vars = dict()
updated_vars = dict()
for (key, value) in possible_vars.items():
if value is not None:
if key in current_vars and (current_vars.get(key) != value):
updated_vars[key] = value
elif key not in current_vars:
new_vars[key] = value
if new_vars or updated_vars:
lines = list()
lines.append("")
lines.append('# Updated on %s' % (date.rcf8222date()))
lines.append("")
if new_vars:
lines.append('# New stuff')
lines.extend(self._make_dict_export(new_vars))
lines.append("")
if updated_vars:
lines.append('# Updated stuff')
lines.extend(self._make_dict_export(updated_vars))
lines.append("")
append_contents = utils.joinlinesep(*lines)
sh.append_file(fn, append_contents)
return len(new_vars) + len(updated_vars)
else:
return 0
def trace(self, cmd, action=None):
if action is None:
action = ''
if cmd is not None:
sh.append_file(self.trace_fn, "%s - %s\n" % (cmd, action))
def _build_dependencies(self):
(pips_downloaded, package_files) = self.download_dependencies()
# Analyze what was downloaded and eject things that were downloaded
# by pip as a dependency of a download but which we do not want to
# build or can satisfy by other means
no_pips = [pkg_resources.Requirement.parse(name).key
for name in self.python_names]
no_pips.extend(self.ignore_pips)
yum_map = self._get_known_yum_packages()
pips_keys = set([p.key for p in pips_downloaded])
package_reqs = []
for filename in package_files:
package_details = pip_helper.get_archive_details(filename)
package_reqs.append((filename, package_details['req']))
def _filter_package_files():
yum_provided = []
req_names = [req.key for (filename, req) in package_reqs]
package_rpm_names = self.py2rpm_helper.names_to_rpm_names(req_names)
filtered_files = []
for filename, req in package_reqs:
rpm_name = package_rpm_names[req.key]
if req.key in no_pips:
LOG.info(("Dependency %s was downloaded additionally "
"but it is disallowed."), colorizer.quote(req))
continue
if req.key in pips_keys:
filtered_files.append(filename)
continue
# See if pip tried to download it but we already can satisfy
# it via yum and avoid building it in the first place...
rpm_info = self._find_yum_match(yum_map, req, rpm_name)
if not rpm_info:
filtered_files.append(filename)
else:
yum_provided.append((req, rpm_info))
LOG.info(("Dependency %s was downloaded additionally "
"but it can be satisfied by %s from repository "
"%s instead."), colorizer.quote(req),
colorizer.quote(rpm_name),
colorizer.quote(rpm_info['repo']))
return (filtered_files, yum_provided)
LOG.info("Filtering %s downloaded files.", len(package_files))
filtered_package_files, yum_provided = _filter_package_files()
if yum_provided:
yum_buff = six.StringIO()
for (req, rpm_info) in yum_provided:
dep_info = {
'requirement': str(req),
'rpm': rpm_info,
}
yum_buff.write(json.dumps(dep_info))
yum_buff.write("\n")
sh.append_file(self.yum_satisfies_filename, yum_buff.getvalue())
if not filtered_package_files:
LOG.info("No SRPM package dependencies to build.")
return
for filename in package_files:
if filename not in filtered_package_files:
sh.unlink(filename)
build_requires = six.StringIO()
for (filename, req) in package_reqs:
if filename in filtered_package_files:
build_requires.write("%s # %s\n" % (req, sh.basename(filename)))
sh.write_file(self.build_requires_filename, build_requires.getvalue())
# Now build them into SRPM rpm files.
package_files = sorted(filtered_package_files)
self.py2rpm_helper.build_all_srpms(package_files=package_files,
tracewriter=self.tracewriter,
jobs=self.jobs)
def trace(self, cmd, action=None):
if action is None:
action = ''
if cmd is not None:
sh.append_file(self.trace_fn, "%s - %s\n" % (cmd, action))
def trace(self, cmd, action=None):
if action is None:
action = date.rcf8222date()
if cmd is not None:
sh.append_file(self.trace_fn, TRACE_FMT % (cmd, action))
def _build_dependencies(self):
(pips_downloaded, package_files) = self.download_dependencies()
# Analyze what was downloaded and eject things that were downloaded
# by pip as a dependency of a download but which we do not want to
# build or can satisfy by other means
no_pips = [
pkg_resources.Requirement.parse(name).key
for name in self.python_names
]
no_pips.extend(self.ignore_pips)
yum_map = self._get_known_yum_packages()
pips_keys = set([p.key for p in pips_downloaded])
package_reqs = []
for filename in package_files:
package_details = pip_helper.get_archive_details(filename)
package_reqs.append((filename, package_details['req']))
def _filter_package_files():
yum_provided = []
req_names = [req.key for (filename, req) in package_reqs]
package_rpm_names = self.py2rpm_helper.names_to_rpm_names(
req_names)
filtered_files = []
for filename, req in package_reqs:
rpm_name = package_rpm_names[req.key]
if req.key in no_pips:
LOG.info(("Dependency %s was downloaded additionally "
"but it is disallowed."), colorizer.quote(req))
continue
if req.key in pips_keys:
filtered_files.append(filename)
continue
# See if pip tried to download it but we already can satisfy
# it via yum and avoid building it in the first place...
rpm_info = self._find_yum_match(yum_map, req, rpm_name)
if not rpm_info:
filtered_files.append(filename)
else:
yum_provided.append((req, rpm_info))
LOG.info(("Dependency %s was downloaded additionally "
"but it can be satisfied by %s from repository "
"%s instead."), colorizer.quote(req),
colorizer.quote(rpm_name),
colorizer.quote(rpm_info['repo']))
return (filtered_files, yum_provided)
LOG.info("Filtering %s downloaded files.", len(package_files))
filtered_package_files, yum_provided = _filter_package_files()
if yum_provided:
yum_buff = six.StringIO()
for (req, rpm_info) in yum_provided:
dep_info = {
'requirement': str(req),
'rpm': rpm_info,
}
yum_buff.write(json.dumps(dep_info))
yum_buff.write("\n")
sh.append_file(self.yum_satisfies_filename, yum_buff.getvalue())
if not filtered_package_files:
LOG.info("No SRPM package dependencies to build.")
return
for filename in package_files:
if filename not in filtered_package_files:
sh.unlink(filename)
build_requires = six.StringIO()
for (filename, req) in package_reqs:
if filename in filtered_package_files:
build_requires.write("%s # %s\n" %
(req, sh.basename(filename)))
sh.write_file(self.build_requires_filename, build_requires.getvalue())
# Now build them into SRPM rpm files.
package_files = sorted(filtered_package_files)
self.py2rpm_helper.build_all_srpms(package_files=package_files,
tracewriter=self.tracewriter,
jobs=self.jobs)