def _check_clean_environment(self): """ Check the environment is clean (mainly no G4 variables).""" if "ROOTSYS" in os.environ: self._logger.error("System environment variables for root already set, these cannot be set before running snoing.") raise snoing_exceptions.SystemException("System environment variables for root already set, these cannot be set before running snoing.", os.environ["ROOTSYS"]) for env in os.environ.iterkeys(): inenv = env.find('G4') if inenv!=-1: self._logger.error("System environment variables for geant4 already set, these cannot be set before running snoing.") raise snoing_exceptions.SystemException("System environment variables for geant4 already set, these cannot be set before running snoing.", env)
def download_file(self, url, username=None, password=None, token=None, file_name=None, retries=0): """ Download the file at url, using either username+password or token authentication if supplied and needed. The optional file_name parameter will save the url to a file named file_name. """ # Firstly build the request header url_request = urllib2.Request(url) if username is not None: # HTTP authentication supplied b64string = base64.encodestring( '%s:%s' % (username, password)).replace('\n', '') url_request.add_header("Authorization", "Basic %s" % b64string) elif token is not None: url_request.add_header("Authorization", "token %s" % token) if file_name is None: file_name = url.split('/')[-1] file_path = os.path.join(self.get_cache_path(), file_name) local_file = open(file_path, 'wb') try: self._logger.command("wget " + url) remote_file = urllib2.urlopen(url_request) download_size = int( remote_file.info().getheaders("Content-Length")[0]) local_file.write(remote_file.read()) local_file.close() remote_file.close() except urllib2.URLError, e: # Server not available self.remove(file_path) raise snoing_exceptions.SystemException( "Download error (URLError)", url)
def execute_command(self, command, args=[], cwd=None, env={}, verbose=False): """ Execute the command with args, extra environment env in the path cwd.""" if cwd is None: cwd = self.get_install_path() # Firstly setup the environment local_env = os.environ.copy() for key in env.iterkeys(): self._append_environment(key, env[key], local_env) # Now open and run the shell_command shell_command = [command] + args try: process = subprocess.Popen(args=shell_command, env=local_env, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) except exceptions.OSError,e: raise snoing_exceptions.SystemException("Command failed: %s"%(command), e)
def __init__(self, logger, cache_path, install_path, install_mode=None, arguments={}): """ Initialise with a logger for output and a prefered cache and install path. The install_mode is optional, None is no install mode required. The arguments are extra arguments applied to all configure script calls (package specific). """ self._logger = logger self._check_clean_environment() self._cache_path = self.build_path(cache_path) self._install_path = self.build_path(install_path) self._arguments = arguments # Check the system type, only concerned about mac or linux if os.uname()[0] == "Darwin": self._os_type = System.Mac # Setup the mac environment, first find fink or ports and set a special mac environment fink_path = self.find_library("fink") ports_path = self.find_library("port") mac_dir = None if fink_path is not None and ports_path is not None: # Both fink and ports exist self._logger.info("Both Fink and Ports exist") fink = raw_input("Use fink (f) or ports (p)?") if fink == 'f' or fink == 'F': mac_dir = fink_path.strip().replace("/bin/fink", "") else: mac_dir = ports_path.strip().replace("/bin/port", "") elif fink_path is not None: mac_dir = fink_path.strip().replace("/bin/fink", "") elif ports_path is not None: mac_dir = ports_path.strip().replace("/bin/port", "") if mac_dir is not None: # A special mac path exists, add to environment self._append_environment("PATH", os.path.join(mac_dir, "bin")) self._append_environment("LIBRARY_PATH", os.path.join(mac_dir, "lib")) self._append_environment("CPLUS_INCLUDE_PATH", os.path.join(mac_dir, "include")) # Check if XCode in 10.7 installs X11 to /usr/X11 if os.path.exists("/usr/X11"): self._append_environment("PATH", "/usr/X11/bin") self._append_environment("LIBRARY_PATH", "/usr/X11/lib") self._append_environment("CPLUS_INCLUDE_PATH", "/usr/X11/include") # Check if frameworks is used if os.path.exists("/System/Library/Frameworks"): self._append_environment("CPLUS_INCLUDE_PATH", "/System/Library/Frameworks") else: # So much easier for Linux systems.... self._os_type = System.Linux # Check for g++ now if self.find_library("g++") is None: raise snoing_exceptions.SystemException( "No g++", "g++ not found on this system.") # Check the install mode status of the install_path settings_path = os.path.join(self._install_path, "snoing.pkl") self._install_mode = self._deserialise(settings_path) if isinstance(self._install_mode, dict): if self._install_mode['Graphical'] == 1: self._install_mode = installmode.Graphical elif self._install_mode['Grid'] == 1: self._install_mode = installmode.Grid else: self._install_mode = installmode.Normal if self._install_mode is not None: # Settings exist for install path if self._install_mode is not install_mode: # Existing settings do not match raise snoing_exceptions.InstallModeException( "Install mode mismatch.", self._install_mode, install_mode) else: self._serialise(settings_path, install_mode) self._install_mode = install_mode # All good if we get here self._logger.set_install_path( os.path.join(self.get_install_path(), "snoing.log")) self._logger.info("System ready.") self._logger.info("Caching to " + self._cache_path) self._logger.info("Installing to " + self._install_path) self._logger.info("System is " + ' '.join(os.uname()))
class System(object): """ System object, holds information about the install folder and allows commands to be executed. """ Mac, Linux = range(2) def __init__(self, logger, cache_path, install_path, install_mode=None, arguments={}): """ Initialise with a logger for output and a prefered cache and install path. The install_mode is optional, None is no install mode required. The arguments are extra arguments applied to all configure script calls (package specific). """ self._logger = logger self._check_clean_environment() self._cache_path = self.build_path(cache_path) self._install_path = self.build_path(install_path) self._arguments = arguments # Check the system type, only concerned about mac or linux if os.uname()[0] == "Darwin": self._os_type = System.Mac # Setup the mac environment, first find fink or ports and set a special mac environment fink_path = self.find_library("fink") ports_path = self.find_library("port") mac_dir = None if fink_path is not None and ports_path is not None: # Both fink and ports exist self._logger.info("Both Fink and Ports exist") fink = raw_input("Use fink (f) or ports (p)?") if fink == 'f' or fink == 'F': mac_dir = fink_path.strip().replace("/bin/fink", "") else: mac_dir = ports_path.strip().replace("/bin/port", "") elif fink_path is not None: mac_dir = fink_path.strip().replace("/bin/fink", "") elif ports_path is not None: mac_dir = ports_path.strip().replace("/bin/port", "") if mac_dir is not None: # A special mac path exists, add to environment self._append_environment("PATH", os.path.join(mac_dir, "bin")) self._append_environment("LIBRARY_PATH", os.path.join(mac_dir, "lib")) self._append_environment("CPLUS_INCLUDE_PATH", os.path.join(mac_dir, "include")) # Check if XCode in 10.7 installs X11 to /usr/X11 if os.path.exists("/usr/X11"): self._append_environment("PATH", "/usr/X11/bin") self._append_environment("LIBRARY_PATH", "/usr/X11/lib") self._append_environment("CPLUS_INCLUDE_PATH", "/usr/X11/include") # Check if frameworks is used if os.path.exists("/System/Library/Frameworks"): self._append_environment("CPLUS_INCLUDE_PATH", "/System/Library/Frameworks") else: # So much easier for Linux systems.... self._os_type = System.Linux # Check for g++ now if self.find_library("g++") is None: raise snoing_exceptions.SystemException( "No g++", "g++ not found on this system.") # Check the install mode status of the install_path settings_path = os.path.join(self._install_path, "snoing.pkl") self._install_mode = self._deserialise(settings_path) if isinstance(self._install_mode, dict): if self._install_mode['Graphical'] == 1: self._install_mode = installmode.Graphical elif self._install_mode['Grid'] == 1: self._install_mode = installmode.Grid else: self._install_mode = installmode.Normal if self._install_mode is not None: # Settings exist for install path if self._install_mode is not install_mode: # Existing settings do not match raise snoing_exceptions.InstallModeException( "Install mode mismatch.", self._install_mode, install_mode) else: self._serialise(settings_path, install_mode) self._install_mode = install_mode # All good if we get here self._logger.set_install_path( os.path.join(self.get_install_path(), "snoing.log")) self._logger.info("System ready.") self._logger.info("Caching to " + self._cache_path) self._logger.info("Installing to " + self._install_path) self._logger.info("System is " + ' '.join(os.uname())) #################################################################################################### # Functions that are publically useful and available def get_cache_path(self): """ Return the cache path.""" return self._cache_path def get_install_path(self): """ Return the install path.""" return self._install_path def get_install_mode(self): """ Return the system install mode.""" return self._install_mode def get_os_type(self): """ Return the system os type.""" return self._os_type def clean_cache(self): """ Basically delete the cache folder.""" self.remove( self.get_cache_path()) # Remove the folder and everything in it self.build_path(self.get_cache_path()) # Folder must exist after #################################################################################################### # Functions that do stuff to the system def configure_command(self, command='./configure', args=[], cwd=None, env={}, verbose=False, config_type=None): """ Execute a configure command, add the extra arguments.""" if cwd is None: cwd = self.get_install_path() if config_type is not None: if config_type in self._arguments: args.extend(self._arguments[config_type]) else: self._logger.error('config type %s does not exist!' % (config_type)) self.execute_command(command, args, cwd, env, verbose) def execute_command(self, command, args=[], cwd=None, env={}, verbose=False): """ Execute the command with args, extra environment env in the path cwd.""" if cwd is None: cwd = self.get_install_path() # Firstly setup the environment local_env = os.environ.copy() for key in env.iterkeys(): self._append_environment(key, env[key], local_env) # Now open and run the shell_command shell_command = [command] + args try: process = subprocess.Popen(args=shell_command, env=local_env, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) except exceptions.OSError, e: raise snoing_exceptions.SystemException( "Command failed: %s" % (command), e) output = "" error = "" self._logger.command(command + ' ' + ' '.join(args)) if verbose or self._logger.is_verbose(): for line in iter(process.stdout.readline, ""): sys.stdout.write('\n' + line[:-1]) sys.stdout.flush() self._logger.detail(line[:-1]) output += line[:-1] + '\n' process.wait() else: output, error = process.communicate() if output != "": self._logger.detail(output) if error != "": self._logger.detail(error) # After process has finished if process.returncode != 0: raise snoing_exceptions.SystemException( "Command returned %i" % process.returncode, output + error) return output # Very useful for library checking
local_file.write(remote_file.read()) local_file.close() remote_file.close() except urllib2.URLError, e: # Server not available self.remove(file_path) raise snoing_exceptions.SystemException( "Download error (URLError)", url) except: # Catch everything else self.remove(file_path) if retries > 0: self._logger.detail("Download error, retry") download_size = self.download_file(url, username, password, token, file_name, retries - 1) else: raise snoing_exceptions.SystemException("Download error", url) self._logger.detail("Downloaded %i bytes\n" % download_size) return download_size def untar_file(self, file_name, target_path, strip=0): """ Untar file_name to target_path striping the first strip folders.""" self._logger.command("untar " + file_name) if os.path.exists(target_path): shutil.rmtree(target_path) if strip == 0: # Untar directly into target tar_file = tarfile.open( os.path.join(self.get_cache_path(), file_name)) tar_file.__class__ = snoing_tarfile.TarFile tar_file.extractall(target_path) tar_file.close() else: # Must extract to temp target then copy strip directory to real target