def main():
if len(sys.argv) == 2:
installer = Installer()
installer.install(separate_hook(sys.argv[1]),sys.argv[1])
else:
print 'invailed argments'
quit()
def main():
if len(sys.argv) == 2:
installer = Installer()
installer.install(separate_hook(sys.argv[1]), sys.argv[1])
else:
print 'invailed argments'
quit()
def ites(args):
es = Elasticsearch(args.offline_dir, args.url)
# create a running instance of Elasticsearch if needed
if args.url is None:
es_reset = args.es_full_reset or args.es_reset
if es_reset:
es_dir = os.path.abspath(es_reset)
es.reset(es_dir, args.es_full_reset is not None)
else:
assert(args.root_dir)
root_dir = os.path.abspath(args.root_dir)
if not args.version:
toks = args.driver.split("-")
for i in range(3, len(toks)):
if re.match("\d+\.\d+\.\d+", toks[-i]):
version = "-".join(toks[len(toks) - i : -2])
else:
version = args.version
if not version:
raise Exception("failed to determine Elasticsearch version to test against (params: driver: %s, "
"version: %s)" % (args.driver, args.version))
es.spawn(version, root_dir, args.ephemeral)
elif not es.is_listening():
raise Exception("no running prestaged Elasticsearch instance found.")
else:
print("Using pre-staged Elasticsearch.")
# add test data into it
if args.reindex or not (args.skip_indexing and args.skip_tests):
if args.skip_indexing_tests:
test_mode = TestData.MODE_NODATA
elif args.skip_indexing:
test_mode = TestData.MODE_NOINDEX
elif args.reindex:
test_mode = TestData.MODE_REINDEX
else:
test_mode = TestData.MODE_INDEX
data = TestData(es, test_mode, args.offline_dir)
data.load()
# install the driver
if args.driver:
driver_path = os.path.abspath(args.driver)
installer = Installer(driver_path)
installer.install(args.ephemeral)
# run the tests
if not args.skip_tests:
assert(data is not None)
cluster_name = es.cluster_name()
assert(len(cluster_name))
if args.dsn:
Testing(es, data, cluster_name, args.dsn).perform()
else:
Testing(es, data, cluster_name, "Packing=JSON;Compression=on;").perform()
Testing(es, data, cluster_name, "Packing=CBOR;Compression=off;").perform()
def install():
installer = Installer(package_dir)
installer.install('tool-scons')
installer.install('toolchain-icestorm')
print('Install icestick.rules')
if not isfile(rules_system_path):
print rules_local_path
subprocess.call(['sudo', 'cp', rules_local_path, rules_system_path])
else:
print('Package icestick.rules is already the newest version')
def uninstall():
key = raw_input('Are you sure? [Y/N]: ')
if key == 'y' or key == 'Y':
installer = Installer(package_dir)
installer.uninstall('tool-scons')
installer.uninstall('toolchain-icestorm')
if isfile(rules_system_path):
print('Uninstall package icestick.rules')
subprocess.call(['sudo', 'rm', rules_system_path])
else:
print('Package icestick.rules is not installed')
def ites(args):
es = Elasticsearch(args.offline_dir)
# create a running instance of Elasticsearch if needed
if not args.pre_staged:
if args.es_reset:
es_dir = os.path.abspath(args.es_reset)
es.reset(es_dir)
else:
assert (args.root_dir)
root_dir = os.path.abspath(args.root_dir)
if not args.version:
toks = args.driver.split("-")
for i in range(3, len(toks)):
if re.match("\d+\.\d+\.\d+", toks[-i]):
version = "-".join(toks[len(toks) - i:-2])
else:
version = args.version
if not version:
raise Exception(
"failed to determine Elasticsearch version to test against (params: driver: %s, "
"version: %s)" % (args.driver, args.version))
es.spawn(version, root_dir, args.ephemeral)
elif not es.is_listening(Elasticsearch.AUTH_PASSWORD):
raise Exception("no running prestaged Elasticsearch instance found.")
else:
print("Using pre-staged Elasticsearch.")
# add test data into it
if args.reindex or not (args.skip_indexing and args.skip_tests):
if args.skip_indexing:
test_mode = TestData.MODE_NOINDEX
elif args.reindex:
test_mode = TestData.MODE_REINDEX
else:
test_mode = TestData.MODE_INDEX
data = TestData(test_mode, args.offline_dir)
data.load()
# install the driver
if args.driver:
driver_path = os.path.abspath(args.driver)
installer = Installer(driver_path)
installer.install(args.ephemeral)
# run the tests
if not args.skip_tests:
assert (data is not None)
tests = Testing(data, args.dsn)
tests.perform()
def test___del__(self):
installer = Installer.fromYaml(config, Cache(wd), "linux", "x86_64")
tempdir = Path(installer.temp_dir)
tempdir.mkdir(exist_ok=True)
assert tempdir.exists()
del installer
assert tempdir.exists() == False
def test_do_cbdep(self):
rmtree(wd, ignore_errors=True)
wd.mkdir()
assert config
installer = Installer.fromYaml(config, Cache(wd), "linux", "x86_64")
installer.do_cbdep({"cbdep": "analytics-jars", "version": "7.0.2-6512", "install_dir": str(wd/"test_do_cbdep")})
assert md5(open(wd/"test_do_cbdep"/"analytics-jars-7.0.2-6512"/"cbas-install-7.0.2.jar", "rb").read()).hexdigest() == "3436fda4756c9aed996a6ad2ed9ddb30"
def do_install(self, args):
"""
Install a package based on a descriptor YAML
"""
installdir = args.dir
if installdir is None:
installdir = "install"
installdir = str(pathlib.Path(installdir).resolve())
installer = Installer.fromYaml(self.configpath(args), self.cache,
get_platforms(),
"x86" if args.x32 else get_arches())
installer.set_cache_only(args.cache_only)
installer.set_recache(args.recache)
if args.cache_local_file is not None:
installer.set_from_local_file(args.cache_local_file)
installer.set_cache_only(True)
installer.install(args.package, args.version, args.base_url,
installdir)
if args.output is not None:
logger.debug(f"Copying downloaded file to {args.output}")
shutil.copy2(installer.get_installer_file(), args.output)
def test_do_install_dir(self):
installer = Installer.fromYaml(config, Cache(wd), "linux", "x86_64")
action = {
"install_dir": str(wd / "test_do_install_dir")
}
installer.do_install_dir(action)
assert installer.symbols["INSTALL_DIR"] == str(wd / "test_do_install_dir")
def test_wrong_platform_install(self):
clear_wd()
installer = Installer.fromYaml(config, Cache(wd), wrong_platform_package["platform"], "x86_64")
with pytest.raises(SystemExit) as e:
installer.install(wrong_platform_package["name"], wrong_platform_package["version"], wrong_platform_package.get("base_url", ""), wd/"install")
assert e.type == SystemExit
assert e.value.code == 1
def test_handle_fixed_dir(self):
rmtree(wd, ignore_errors=True)
wd.mkdir()
installer = Installer.fromYaml(config, Cache(wd), "linux", "x86_64")
installer.symbols["foo"] = "bar"
installer.handle_fixed_dir({ "fixed_dir": str(wd/"missing") })
assert installer.handle_fixed_dir({ "fixed_dir": str(wd/"missing") }) == False
(wd/"present").touch()
assert installer.handle_fixed_dir({ "fixed_dir": str(wd/"present") }) == True
def prepare_build(self, binary):
''' Prepare the build for the test run '''
if self.options.install is not None:
self.options.folder = Installer().install(binary, self.options.install)
self.options.binary = self.options.folder + self.app_binary[sys.platform]
else:
folder = os.path.dirname(binary)
self.options.folder = folder if not os.path.isdir(binary) else binary
self.options.binary = binary
def test_set_from_local_file(self):
installer = Installer.fromYaml(config, Cache(wd), "linux", "x86_64")
Path("/tmp/bar").touch()
installer.set_recache(True)
installer.set_from_local_file("/tmp/bar")
assert installer.recache == False
with pytest.raises(SystemExit) as e:
installer.set_from_local_file("/tmp/barbaz")
assert e.type == SystemExit
assert e.value.code == 1
def test_Installer_methods(self):
"Test various methods of the Installer class."
installer = Installer(self.versions_txt, site=self.site)
self.assertEqual(
installer.stack_dir,
'/nfs/farm/g/lsst/u1/software/redhat6-x86_64-64bit-gcc44/DMstack/v12_0'
)
package_name = 'metrology-data-analysis'
self.assertEqual(installer._env_var(package_name),
'METROLOGYDATAANALYSISDIR')
self.assertEqual(installer._eups_config(), 'setup eotest\n')
self.assertEqual(
installer._schema_paths(),
'export LCATR_SCHEMA_PATH=${HARNESSEDJOBSDIR}/schemas:${LCATR_SCHEMA_PATH}\n'
)
self.assertEqual(
installer._jh_config(),
'''export HARNESSEDJOBSDIR=${INST_DIR}/harnessed-jobs-0.3.49-slac
export VIRTUAL_ENV=${INST_DIR}
source ${INST_DIR}/Modules/3.2.10/init/bash
export PATH=${INST_DIR}/bin:${PATH}
export SITENAME=%s
''' % self.site)
with warnings.catch_warnings():
warnings.filterwarnings('ignore')
self.assertEqual(
installer._python_configs(),
'''export DATACATDIR=/afs/slac/u/gl/srs/datacat/dev/0.4/lib
export DATACAT_CONFIG=/nfs/farm/g/lsst/u1/software/datacat/config.cfg
export PYTHONPATH=${OFFLINEJOBSDIR}/python:${METROLOGYDATAANALYSISDIR}/python:${DATACATDIR}:${HARNESSEDJOBSDIR}/python::${PYTHONPATH}
export MPLBACKEND=Agg
''')
self.assertEqual(
installer._package_env_vars(),
'''export OFFLINEJOBSDIR=${INST_DIR}/offline-jobs-0.0.16
export METROLOGYDATAANALYSISDIR=${INST_DIR}/metrology-data-analysis-0.0.10
''')
self.assertEqual(set(installer.package_dirs.keys()),
set(('metrology-data-analysis', 'offline-jobs')))
def test_working_install(self, caplog, package):
caplog.set_level(logging.DEBUG)
clear_wd()
logger.info(f"Testing package '{package}'")
installer = Installer.fromYaml(config, Cache(wd), package["platform"], package.get("arch", plat.get_arches()))
installer.install(package["name"], package["version"], package.get("base_url", ""), wd/"install")
fn = wd / package["hash"][0:2] / package["hash"] / package.get("filename", f"{package['name']}-{package['version']}.tar.gz")
logger.debug(f" Checking for downloaded file '{fn}'")
assert fn.is_file()
install_dir = Path(package.get("install_dir", f"{package['name']}-{package['version']}"))
if not install_dir.is_absolute():
install_dir = wd / "install" / install_dir
logger.debug(f" Checking for install dir '{install_dir}'")
assert install_dir.is_dir()
final_filename = package.get("final_file", None)
if final_filename is not None:
logger.debug(f" Checking for final file '{final_filename}'")
assert (install_dir / final_filename).is_file()
rmtree(install_dir, ignore_errors=True)
def do_install(self, args):
"""
Install a package based on a descriptor YAML
"""
yamlfile = args.config_file
if yamlfile is None:
if getattr(sys, 'frozen', False):
# running in a bundle
mydir = pathlib.Path(sys._MEIPASS)
else:
# running live
mydir = pathlib.Path.home()
yamlfile = str(mydir / "cbdep.config")
installdir = args.dir
if installdir is None:
# QQQ
installdir = "install"
installdir = str(pathlib.Path(installdir).resolve())
installer = Installer.fromYaml(yamlfile, self.cache, args.platform)
installer.install(args.package, args.version, args.x32, installdir)
def main():
options = get_options()
if options.install:
Installer()
return None
local_conf = ConfigurationFolder(conf.CONFIGURATION_DIRECTORY)
remote_conf = ConfigurationFolder(conf.REQUISITORY_LOCATION)
if options.rm is not None:
local_conf.remove_files(options.rm)
remote_conf.remove_files(options.rm)
return 0
elif options.status:
return DiffTool(local_conf, remote_conf).display_diff()
elif options.load:
dest_config = local_conf
src_config = remote_conf
elif options.save:
dest_config = remote_conf
src_config = local_conf
moved_files = src_config.retrieve_files()
dest_config.save_files(moved_files, src_config.directory)
return 0
def test_Installer_methods(self):
"Test various methods of the Installer class."
installer = Installer(self.versions_txt, site=self.site)
self.assertEqual(installer.stack_dir, '/nfs/farm/g/lsst/u1/software/redhat6-x86_64-64bit-gcc44/DMstack/v12_0')
package_name = 'metrology-data-analysis'
self.assertEqual(installer._env_var(package_name),
'METROLOGYDATAANALYSISDIR')
self.assertEqual(installer._eups_config(), 'setup eotest\n')
self.assertEqual(installer._schema_paths(), 'export LCATR_SCHEMA_PATH=${HARNESSEDJOBSDIR}/schemas:${LCATR_SCHEMA_PATH}\n')
self.assertEqual(installer._jh_config(),
'''export HARNESSEDJOBSDIR=${INST_DIR}/harnessed-jobs-0.3.49-slac
export VIRTUAL_ENV=${INST_DIR}
source ${INST_DIR}/Modules/3.2.10/init/bash
export PATH=${INST_DIR}/bin:${PATH}
export SITENAME=%s
''' % self.site)
with warnings.catch_warnings():
warnings.filterwarnings('ignore')
self.assertEqual(installer._python_configs(),
'''export DATACATDIR=/afs/slac/u/gl/srs/datacat/dev/0.4/lib
export DATACAT_CONFIG=/nfs/farm/g/lsst/u1/software/datacat/config.cfg
export PYTHONPATH=${OFFLINEJOBSDIR}/python:${METROLOGYDATAANALYSISDIR}/python:${DATACATDIR}:${HARNESSEDJOBSDIR}/python::${PYTHONPATH}
''')
self.assertEqual(installer._package_env_vars(),
'''export OFFLINEJOBSDIR=${INST_DIR}/offline-jobs-0.0.16
export METROLOGYDATAANALYSISDIR=${INST_DIR}/metrology-data-analysis-0.0.10
''')
self.assertEqual(set(installer.package_dirs.keys()),
set(('metrology-data-analysis', 'offline-jobs')))
def test_set_cache_only(self):
installer = Installer.fromYaml(config, Cache(wd), "linux", "x86_64")
installer.set_cache_only(False)
assert installer.cache_only == False
installer.set_cache_only(True)
assert installer.cache_only == True
def test_recache(self):
installer = Installer.fromYaml(config, Cache(wd), "linux", "x86_64")
installer.set_recache(False)
assert installer.recache == False
installer.set_recache(True)
assert installer.recache == True
def cleanup_build(self):
# Always remove the build when it has been installed
if self.options.install:
Installer().uninstall(self.options.folder)
def test_handle_set_env(self):
installer = Installer.fromYaml(config, Cache(wd), "linux", "x86_64")
installer.symbols["foo"] = "bar"
os.environ["test_handle_set_env"] = ""
installer.handle_set_env({ "test_handle_set_env": "xxx" })
assert os.environ["test_handle_set_env"] == "xxx"
def test_copy(self):
installer = Installer.fromYaml(config, Cache(wd), "linux", "x86_64")
assert type(installer.copy()) == Installer
def test_templatize(self):
installer = Installer.fromYaml(config, Cache(wd), "linux", "x86_64")
installer.symbols["ALPHA"] = "ABC"
installer.symbols["NUMERIC"] = "123"
assert installer.templatize("${ALPHA}-${NUMERIC}") == "ABC-123"
def test_get_installer_file(self):
installer = Installer.fromYaml(config, Cache(wd), "linux", "x86_64")
installer.installer_file = "foo"
assert installer.get_installer_file() == "foo"
