def _check_clean_environment(self):
""" Check the environment is clean (mainly no G4 variables)."""
if "ROOTSYS" in os.environ:
self._logger.error("System environment variables for root already set, these cannot be set before running snoing.")
raise snoing_exceptions.SystemException("System environment variables for root already set, these cannot be set before running snoing.", os.environ["ROOTSYS"])
for env in os.environ.iterkeys():
inenv = env.find('G4')
if inenv!=-1:
self._logger.error("System environment variables for geant4 already set, these cannot be set before running snoing.")
raise snoing_exceptions.SystemException("System environment variables for geant4 already set, these cannot be set before running snoing.", env)
def download_file(self,
url,
username=None,
password=None,
token=None,
file_name=None,
retries=0):
""" Download the file at url, using either username+password or token authentication if
supplied and needed. The optional file_name parameter will save the url to a file named
file_name.
"""
# Firstly build the request header
url_request = urllib2.Request(url)
if username is not None: # HTTP authentication supplied
b64string = base64.encodestring(
'%s:%s' % (username, password)).replace('\n', '')
url_request.add_header("Authorization", "Basic %s" % b64string)
elif token is not None:
url_request.add_header("Authorization", "token %s" % token)
if file_name is None:
file_name = url.split('/')[-1]
file_path = os.path.join(self.get_cache_path(), file_name)
local_file = open(file_path, 'wb')
try:
self._logger.command("wget " + url)
remote_file = urllib2.urlopen(url_request)
download_size = int(
remote_file.info().getheaders("Content-Length")[0])
local_file.write(remote_file.read())
local_file.close()
remote_file.close()
except urllib2.URLError, e: # Server not available
self.remove(file_path)
raise snoing_exceptions.SystemException(
"Download error (URLError)", url)
def execute_command(self, command, args=[], cwd=None, env={}, verbose=False):
""" Execute the command with args, extra environment env in the path cwd."""
if cwd is None:
cwd = self.get_install_path()
# Firstly setup the environment
local_env = os.environ.copy()
for key in env.iterkeys():
self._append_environment(key, env[key], local_env)
# Now open and run the shell_command
shell_command = [command] + args
try:
process = subprocess.Popen(args=shell_command, env=local_env, cwd=cwd,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
except exceptions.OSError,e:
raise snoing_exceptions.SystemException("Command failed: %s"%(command),
e)
def __init__(self,
logger,
cache_path,
install_path,
install_mode=None,
arguments={}):
""" Initialise with a logger for output and a prefered cache and install path. The
install_mode is optional, None is no install mode required. The arguments are extra
arguments applied to all configure script calls (package specific).
"""
self._logger = logger
self._check_clean_environment()
self._cache_path = self.build_path(cache_path)
self._install_path = self.build_path(install_path)
self._arguments = arguments
# Check the system type, only concerned about mac or linux
if os.uname()[0] == "Darwin":
self._os_type = System.Mac
# Setup the mac environment, first find fink or ports and set a special mac environment
fink_path = self.find_library("fink")
ports_path = self.find_library("port")
mac_dir = None
if fink_path is not None and ports_path is not None: # Both fink and ports exist
self._logger.info("Both Fink and Ports exist")
fink = raw_input("Use fink (f) or ports (p)?")
if fink == 'f' or fink == 'F':
mac_dir = fink_path.strip().replace("/bin/fink", "")
else:
mac_dir = ports_path.strip().replace("/bin/port", "")
elif fink_path is not None:
mac_dir = fink_path.strip().replace("/bin/fink", "")
elif ports_path is not None:
mac_dir = ports_path.strip().replace("/bin/port", "")
if mac_dir is not None: # A special mac path exists, add to environment
self._append_environment("PATH", os.path.join(mac_dir, "bin"))
self._append_environment("LIBRARY_PATH",
os.path.join(mac_dir, "lib"))
self._append_environment("CPLUS_INCLUDE_PATH",
os.path.join(mac_dir, "include"))
# Check if XCode in 10.7 installs X11 to /usr/X11
if os.path.exists("/usr/X11"):
self._append_environment("PATH", "/usr/X11/bin")
self._append_environment("LIBRARY_PATH", "/usr/X11/lib")
self._append_environment("CPLUS_INCLUDE_PATH",
"/usr/X11/include")
# Check if frameworks is used
if os.path.exists("/System/Library/Frameworks"):
self._append_environment("CPLUS_INCLUDE_PATH",
"/System/Library/Frameworks")
else: # So much easier for Linux systems....
self._os_type = System.Linux
# Check for g++ now
if self.find_library("g++") is None:
raise snoing_exceptions.SystemException(
"No g++", "g++ not found on this system.")
# Check the install mode status of the install_path
settings_path = os.path.join(self._install_path, "snoing.pkl")
self._install_mode = self._deserialise(settings_path)
if isinstance(self._install_mode, dict):
if self._install_mode['Graphical'] == 1:
self._install_mode = installmode.Graphical
elif self._install_mode['Grid'] == 1:
self._install_mode = installmode.Grid
else:
self._install_mode = installmode.Normal
if self._install_mode is not None: # Settings exist for install path
if self._install_mode is not install_mode: # Existing settings do not match
raise snoing_exceptions.InstallModeException(
"Install mode mismatch.", self._install_mode, install_mode)
else:
self._serialise(settings_path, install_mode)
self._install_mode = install_mode
# All good if we get here
self._logger.set_install_path(
os.path.join(self.get_install_path(), "snoing.log"))
self._logger.info("System ready.")
self._logger.info("Caching to " + self._cache_path)
self._logger.info("Installing to " + self._install_path)
self._logger.info("System is " + ' '.join(os.uname()))
class System(object):
""" System object, holds information about the install folder and allows commands to be
executed.
"""
Mac, Linux = range(2)
def __init__(self,
logger,
cache_path,
install_path,
install_mode=None,
arguments={}):
""" Initialise with a logger for output and a prefered cache and install path. The
install_mode is optional, None is no install mode required. The arguments are extra
arguments applied to all configure script calls (package specific).
"""
self._logger = logger
self._check_clean_environment()
self._cache_path = self.build_path(cache_path)
self._install_path = self.build_path(install_path)
self._arguments = arguments
# Check the system type, only concerned about mac or linux
if os.uname()[0] == "Darwin":
self._os_type = System.Mac
# Setup the mac environment, first find fink or ports and set a special mac environment
fink_path = self.find_library("fink")
ports_path = self.find_library("port")
mac_dir = None
if fink_path is not None and ports_path is not None: # Both fink and ports exist
self._logger.info("Both Fink and Ports exist")
fink = raw_input("Use fink (f) or ports (p)?")
if fink == 'f' or fink == 'F':
mac_dir = fink_path.strip().replace("/bin/fink", "")
else:
mac_dir = ports_path.strip().replace("/bin/port", "")
elif fink_path is not None:
mac_dir = fink_path.strip().replace("/bin/fink", "")
elif ports_path is not None:
mac_dir = ports_path.strip().replace("/bin/port", "")
if mac_dir is not None: # A special mac path exists, add to environment
self._append_environment("PATH", os.path.join(mac_dir, "bin"))
self._append_environment("LIBRARY_PATH",
os.path.join(mac_dir, "lib"))
self._append_environment("CPLUS_INCLUDE_PATH",
os.path.join(mac_dir, "include"))
# Check if XCode in 10.7 installs X11 to /usr/X11
if os.path.exists("/usr/X11"):
self._append_environment("PATH", "/usr/X11/bin")
self._append_environment("LIBRARY_PATH", "/usr/X11/lib")
self._append_environment("CPLUS_INCLUDE_PATH",
"/usr/X11/include")
# Check if frameworks is used
if os.path.exists("/System/Library/Frameworks"):
self._append_environment("CPLUS_INCLUDE_PATH",
"/System/Library/Frameworks")
else: # So much easier for Linux systems....
self._os_type = System.Linux
# Check for g++ now
if self.find_library("g++") is None:
raise snoing_exceptions.SystemException(
"No g++", "g++ not found on this system.")
# Check the install mode status of the install_path
settings_path = os.path.join(self._install_path, "snoing.pkl")
self._install_mode = self._deserialise(settings_path)
if isinstance(self._install_mode, dict):
if self._install_mode['Graphical'] == 1:
self._install_mode = installmode.Graphical
elif self._install_mode['Grid'] == 1:
self._install_mode = installmode.Grid
else:
self._install_mode = installmode.Normal
if self._install_mode is not None: # Settings exist for install path
if self._install_mode is not install_mode: # Existing settings do not match
raise snoing_exceptions.InstallModeException(
"Install mode mismatch.", self._install_mode, install_mode)
else:
self._serialise(settings_path, install_mode)
self._install_mode = install_mode
# All good if we get here
self._logger.set_install_path(
os.path.join(self.get_install_path(), "snoing.log"))
self._logger.info("System ready.")
self._logger.info("Caching to " + self._cache_path)
self._logger.info("Installing to " + self._install_path)
self._logger.info("System is " + ' '.join(os.uname()))
####################################################################################################
# Functions that are publically useful and available
def get_cache_path(self):
""" Return the cache path."""
return self._cache_path
def get_install_path(self):
""" Return the install path."""
return self._install_path
def get_install_mode(self):
""" Return the system install mode."""
return self._install_mode
def get_os_type(self):
""" Return the system os type."""
return self._os_type
def clean_cache(self):
""" Basically delete the cache folder."""
self.remove(
self.get_cache_path()) # Remove the folder and everything in it
self.build_path(self.get_cache_path()) # Folder must exist after
####################################################################################################
# Functions that do stuff to the system
def configure_command(self,
command='./configure',
args=[],
cwd=None,
env={},
verbose=False,
config_type=None):
""" Execute a configure command, add the extra arguments."""
if cwd is None:
cwd = self.get_install_path()
if config_type is not None:
if config_type in self._arguments:
args.extend(self._arguments[config_type])
else:
self._logger.error('config type %s does not exist!' %
(config_type))
self.execute_command(command, args, cwd, env, verbose)
def execute_command(self,
command,
args=[],
cwd=None,
env={},
verbose=False):
""" Execute the command with args, extra environment env in the path cwd."""
if cwd is None:
cwd = self.get_install_path()
# Firstly setup the environment
local_env = os.environ.copy()
for key in env.iterkeys():
self._append_environment(key, env[key], local_env)
# Now open and run the shell_command
shell_command = [command] + args
try:
process = subprocess.Popen(args=shell_command,
env=local_env,
cwd=cwd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except exceptions.OSError, e:
raise snoing_exceptions.SystemException(
"Command failed: %s" % (command), e)
output = ""
error = ""
self._logger.command(command + ' ' + ' '.join(args))
if verbose or self._logger.is_verbose():
for line in iter(process.stdout.readline, ""):
sys.stdout.write('\n' + line[:-1])
sys.stdout.flush()
self._logger.detail(line[:-1])
output += line[:-1] + '\n'
process.wait()
else:
output, error = process.communicate()
if output != "":
self._logger.detail(output)
if error != "":
self._logger.detail(error)
# After process has finished
if process.returncode != 0:
raise snoing_exceptions.SystemException(
"Command returned %i" % process.returncode, output + error)
return output # Very useful for library checking
local_file.write(remote_file.read())
local_file.close()
remote_file.close()
except urllib2.URLError, e: # Server not available
self.remove(file_path)
raise snoing_exceptions.SystemException(
"Download error (URLError)", url)
except: # Catch everything else
self.remove(file_path)
if retries > 0:
self._logger.detail("Download error, retry")
download_size = self.download_file(url, username, password,
token, file_name,
retries - 1)
else:
raise snoing_exceptions.SystemException("Download error", url)
self._logger.detail("Downloaded %i bytes\n" % download_size)
return download_size
def untar_file(self, file_name, target_path, strip=0):
""" Untar file_name to target_path striping the first strip folders."""
self._logger.command("untar " + file_name)
if os.path.exists(target_path):
shutil.rmtree(target_path)
if strip == 0: # Untar directly into target
tar_file = tarfile.open(
os.path.join(self.get_cache_path(), file_name))
tar_file.__class__ = snoing_tarfile.TarFile
tar_file.extractall(target_path)
tar_file.close()
else: # Must extract to temp target then copy strip directory to real target