def _setup_vol_groups(self): LOG.info("Attempting to setup volume groups for nova volume management.") mp = dict() backing_file = self.cfg.getdefaulted('nova', 'volume_backing_file', sh.joinpths(self.installer.get_option('app_dir'), 'nova-volumes-backing-file')) vol_group = self.cfg.getdefaulted('nova', 'volume_group', 'nova-volumes') backing_file_size = utils.to_bytes(self.cfg.getdefaulted('nova', 'volume_backing_file_size', '2052M')) mp['VOLUME_GROUP'] = vol_group mp['VOLUME_BACKING_FILE'] = backing_file mp['VOLUME_BACKING_FILE_SIZE'] = backing_file_size try: utils.execute_template(*VG_CHECK_CMD, params=mp) LOG.warn("Volume group already exists: %r" % (vol_group)) except exceptions.ProcessExecutionError as err: # Check that the error from VG_CHECK is an expected error if err.exit_code != 5: raise LOG.info("Need to create volume group: %r" % (vol_group)) sh.touch_file(backing_file, die_if_there=False, file_size=backing_file_size) vg_dev_result = utils.execute_template(*VG_DEV_CMD, params=mp) if vg_dev_result and vg_dev_result[0]: LOG.debug("VG dev result: %s" % (vg_dev_result)) # Strip the newlines out of the stdout (which is in the first # element of the first (and only) tuple in the response (sysout, _) = vg_dev_result[0] mp['DEV'] = sysout.replace('\n', '') utils.execute_template(*VG_CREATE_CMD, params=mp) # One way or another, we should have the volume group, Now check the # logical volumes self._process_lvs(mp) # Finish off by restarting tgt, and ignore any errors cmdrestart = self.distro.get_command('iscsi', 'restart', quiet=True) if cmdrestart: sh.execute(*cmdrestart, run_as_root=True, check_exit_code=False)
def download_dependencies(self): """Download dependencies from `$deps_dir/download-requires`.""" # NOTE(aababilov): do not drop download_dir - it can be reused sh.mkdirslist(self.download_dir, tracewriter=self.tracewriter) pips_to_download = self._filter_download_requires() sh.write_file(self.download_requires_filename, "\n".join([str(req) for req in pips_to_download])) if not pips_to_download: return ([], []) # NOTE(aababilov): user could have changed persona, so, # check that all requirements are downloaded if (sh.isfile(self.downloaded_flag_file) and self._requirements_satisfied(pips_to_download, self.download_dir)): LOG.info("All python dependencies have been already downloaded") else: def try_download(attempt): LOG.info("Downloading %s dependencies with pip (attempt %s)...", len(pips_to_download), attempt) output_filename = sh.joinpths(self.log_dir, "pip-download-attempt-%s.log" % (attempt)) pip_helper.download_dependencies(self.download_dir, pips_to_download, output_filename) utils.retry(self.MAX_PIP_DOWNLOAD_ATTEMPTS, self.PIP_DOWNLOAD_DELAY, try_download) # NOTE(harlowja): Mark that we completed downloading successfully sh.touch_file(self.downloaded_flag_file, die_if_there=False, quiet=True, tracewriter=self.tracewriter) pips_downloaded = [pip_helper.extract_requirement(p) for p in pips_to_download] what_downloaded = self._examine_download_dir(pips_downloaded, self.download_dir) return (pips_downloaded, what_downloaded)
def _start(self): if self.started: return else: trace_dirs = sh.mkdirslist(sh.dirname(self.trace_fn)) sh.touch_file(self.trace_fn, die_if_there=self.break_if_there) self.started = True self.dirs_made(*trace_dirs)
def download_dependencies(self): """Download dependencies from `$deps_dir/download-requires`. """ # NOTE(aababilov): do not drop download_dir - it can be reused sh.mkdirslist(self.download_dir, tracewriter=self.tracewriter) download_requires_filename = sh.joinpths(self.deps_dir, "download-requires") raw_pips_to_download = self.filter_download_requires() sh.write_file(download_requires_filename, "\n".join(str(req) for req in raw_pips_to_download)) if not raw_pips_to_download: return ([], []) # NOTE(aababilov): user could have changed persona, so, # check that all requirements are downloaded if (sh.isfile(self.downloaded_flag_file) and self._requirements_satisfied(raw_pips_to_download, self.download_dir)): LOG.info("All python dependencies have been already downloaded") else: pip_dir = sh.joinpths(self.deps_dir, "pip") pip_download_dir = sh.joinpths(pip_dir, "download") pip_build_dir = sh.joinpths(pip_dir, "build") # NOTE(aababilov): do not clean the cache, it is always useful pip_cache_dir = sh.joinpths(self.deps_dir, "pip-cache") pip_failures = [] for attempt in xrange(self.MAX_PIP_DOWNLOAD_ATTEMPTS): # NOTE(aababilov): pip has issues with already downloaded files sh.deldir(pip_dir) sh.mkdirslist(pip_download_dir, tracewriter=self.tracewriter) header = "Downloading %s python dependencies (attempt %s)" header = header % (len(raw_pips_to_download), attempt) utils.log_iterable(sorted(raw_pips_to_download), logger=LOG, header=header) failed = False try: self._try_download_dependencies(attempt, raw_pips_to_download, pip_download_dir, pip_cache_dir, pip_build_dir) pip_failures = [] except exc.ProcessExecutionError as e: LOG.exception("Failed downloading python dependencies") pip_failures.append(e) failed = True if not failed: break for filename in sh.listdir(pip_download_dir, files_only=True): sh.move(filename, self.download_dir, force=True) sh.deldir(pip_dir) if pip_failures: raise pip_failures[-1] sh.touch_file(self.downloaded_flag_file, die_if_there=False, quiet=True, tracewriter=self.tracewriter) pips_downloaded = [pip_helper.extract_requirement(p) for p in raw_pips_to_download] self._examine_download_dir(pips_downloaded, self.download_dir) what_downloaded = sh.listdir(self.download_dir, files_only=True) return (pips_downloaded, what_downloaded)
def _setup_logs(self, clear=False): log_fns = [self.access_log, self.error_log] utils.log_iterable(log_fns, logger=LOG, header="Adjusting %s log files" % (len(log_fns))) for fn in log_fns: if clear: sh.unlink(fn, True) sh.touch_file(fn, die_if_there=False, tracewriter=self.tracewriter) sh.chmod(fn, 0666) return len(log_fns)
def _setup_logs(self, clear): log_fns = [self.access_log, self.error_log] utils.log_iterable(log_fns, logger=LOG, header="Adjusting %s log files" % (len(log_fns))) for fn in log_fns: with sh.Rooted(True): if clear: sh.unlink(fn, True) sh.mkdirslist(sh.dirname(fn)) sh.touch_file(fn, die_if_there=False) sh.chmod(fn, 0666) self.tracewriter.file_touched(fn) return len(log_fns)
def download_dependencies(self): """Download dependencies from `$deps_dir/download-requires`.""" # NOTE(aababilov): do not drop download_dir - it can be reused sh.mkdirslist(self.download_dir, tracewriter=self.tracewriter) pips_to_download = self._filter_download_requires() sh.write_file(self.download_requires_filename, "\n".join([str(req) for req in pips_to_download])) if not pips_to_download: return ([], []) # NOTE(aababilov): user could have changed persona, so, # check that all requirements are downloaded if (sh.isfile(self.downloaded_flag_file) and self._requirements_satisfied(pips_to_download, self.download_dir)): LOG.info("All python dependencies have been already downloaded") else: pip_failures = [] for attempt in xrange(self.MAX_PIP_DOWNLOAD_ATTEMPTS): # NOTE(aababilov): pip has issues with already downloaded files for filename in sh.listdir(self.download_dir, files_only=True): sh.unlink(filename) header = "Downloading %s python dependencies (attempt %s)" header = header % (len(pips_to_download), attempt + 1) utils.log_iterable(sorted(pips_to_download), logger=LOG, header=header) failed = False try: self._try_download_dependencies(attempt + 1, pips_to_download, self.download_dir) pip_failures = [] except exc.ProcessExecutionError as e: LOG.exception("Failed downloading python dependencies") pip_failures.append(e) failed = True if not failed: break if pip_failures: raise pip_failures[-1] # NOTE(harlowja): Mark that we completed downloading successfully sh.touch_file(self.downloaded_flag_file, die_if_there=False, quiet=True, tracewriter=self.tracewriter) pips_downloaded = [ pip_helper.extract_requirement(p) for p in pips_to_download ] self._examine_download_dir(pips_downloaded, self.download_dir) return (pips_downloaded, sh.listdir(self.download_dir, files_only=True))
def download_dependencies(self): """Download dependencies from `$deps_dir/download-requires`.""" # NOTE(aababilov): do not drop download_dir - it can be reused sh.mkdirslist(self.download_dir, tracewriter=self.tracewriter) pips_to_download = self._filter_download_requires() sh.write_file(self.download_requires_filename, "\n".join([str(req) for req in pips_to_download])) if not pips_to_download: return ([], []) # NOTE(aababilov): user could have changed persona, so, # check that all requirements are downloaded if (sh.isfile(self.downloaded_flag_file) and self._requirements_satisfied(pips_to_download, self.download_dir)): LOG.info("All python dependencies have been already downloaded") else: pip_failures = [] for attempt in xrange(self.MAX_PIP_DOWNLOAD_ATTEMPTS): # NOTE(aababilov): pip has issues with already downloaded files for filename in sh.listdir(self.download_dir, files_only=True): sh.unlink(filename) header = "Downloading %s python dependencies (attempt %s)" header = header % (len(pips_to_download), attempt + 1) utils.log_iterable(sorted(pips_to_download), logger=LOG, header=header) failed = False try: self._try_download_dependencies(attempt + 1, pips_to_download, self.download_dir) pip_failures = [] except exc.ProcessExecutionError as e: LOG.exception("Failed downloading python dependencies") pip_failures.append(e) failed = True if not failed: break if pip_failures: raise pip_failures[-1] # NOTE(harlowja): Mark that we completed downloading successfully sh.touch_file(self.downloaded_flag_file, die_if_there=False, quiet=True, tracewriter=self.tracewriter) pips_downloaded = [pip_helper.extract_requirement(p) for p in pips_to_download] self._examine_download_dir(pips_downloaded, self.download_dir) return (pips_downloaded, sh.listdir(self.download_dir, files_only=True))
def download_dependencies(self): """Download dependencies from `$deps_dir/download-requires`.""" # NOTE(aababilov): do not drop download_dir - it can be reused sh.mkdirslist(self.download_dir, tracewriter=self.tracewriter) pips_to_download = self._filter_download_requires() sh.write_file(self.download_requires_filename, "\n".join([str(req) for req in pips_to_download])) if not pips_to_download: return ([], []) # NOTE(aababilov): user could have changed persona, so, # check that all requirements are downloaded if (sh.isfile(self.downloaded_flag_file) and self._requirements_satisfied(pips_to_download, self.download_dir)): LOG.info("All python dependencies have been already downloaded") else: def try_download(attempt): LOG.info( "Downloading %s dependencies with pip (attempt %s)...", len(pips_to_download), attempt) output_filename = sh.joinpths( self.log_dir, "pip-download-attempt-%s.log" % (attempt)) pip_helper.download_dependencies(self.download_dir, pips_to_download, output_filename) utils.retry(self.MAX_PIP_DOWNLOAD_ATTEMPTS, self.PIP_DOWNLOAD_DELAY, try_download) # NOTE(harlowja): Mark that we completed downloading successfully sh.touch_file(self.downloaded_flag_file, die_if_there=False, quiet=True, tracewriter=self.tracewriter) pips_downloaded = [ pip_helper.extract_requirement(p) for p in pips_to_download ] what_downloaded = self._examine_download_dir(pips_downloaded, self.download_dir) return (pips_downloaded, what_downloaded)