def setUp(self):
self.environ0 = os.environ.copy()
os.environ["EUPS_PATH"] = testEupsStack
os.environ["EUPS_FLAVOR"] = "Linux"
if eups.Eups().isSetup("python"):
eups.unsetup("python")
self.dbpath = os.path.join(testEupsStack, "ups_db")
def get_stack_products(product_names=None):
"""Get the LSST Stack products corresponding to a set of product
names.
Parameters
----------
product_names: dict [None]
A dict with LSST Stack package names as keys and either
'metapackage' or None as values. The setup eups.Product
corresponding to each of these packages will be returned in a
dictionary. If None, then use the products listed in the
config file.
Returns
-------
dict of eups.Products keyed by package name.
"""
config = get_config()
stack_packages = config['stack_packages'] if product_names is None \
else product_names
eupsenv = eups.Eups()
products = dict()
for product_name, product_type in stack_packages.items():
products[product_name] = eupsenv.getSetupProducts(product_name)[0]
products[product_name].type = product_type
return products
def __init__(self, dataRoot=None, configRoot=None, specIds=None):
"""A data and configuration manager, inspired by the LSST/DRP butler.
This provides only the most minimal functionality required by the PFS ICS software.
Args
----
dataRoot : path-like
The root of the data directory tree. Defaults to /data
configRoot : path-like
The root of the configuration directory tree.
Defaults to $PFS_INSTDATA_DIR/data
specIds : `pfs.utils.spectroIds.SpectroIds`
Contains our identity: site, arm, module, etc.
Usually created dynamically, from the hostname.
"""
self.logger = logging.getLogger('butler')
self.logger.setLevel(logging.WARN)
self.dataRoot = pathlib.Path(
dataRoot) if dataRoot is not None else defaultDataRoot
if configRoot is not None:
self.configRoot = pathlib.Path(configRoot)
else:
import eups
eupsEnv = eups.Eups()
eupsProd = eupsEnv.findSetupProduct('pfs_instdata')
if eupsProd is None:
raise ValueError("either configRoot must be passed in "
"or the pfs_instdata product must be setup.")
self.configRoot = pathlib.Path(eupsProd.dir) / "data"
self._loadMaps(specIds)
def setUp(self):
#Load sample input from disk
srcSet = readSourceSet(
os.path.join(eups.productDir("meas_pipeline"), "tests",
"v695833-e0-c000.xy.txt"))
exp = afwImg.ExposureF(2048, 4612)
#Put them on the clipboard
fileName = pexPolicy.DefaultPolicyFile(
"meas_pipeline", "WcsDeterminationStageDictionary.paf", "policy")
self.policy = pexPolicy.Policy.createPolicy(fileName)
self.clipboard = pexClipboard.Clipboard()
self.clipboard.put(self.policy.get("inputExposureKey"), exp)
self.clipboard.put(self.policy.get("inputSourceSetKey"), srcSet)
# Set up local astrometry_net_data
adnDB = 'testTagAlong'
datapath = os.path.join(eups.productDir("meas_pipeline"), 'tests',
'astrometry_net_data', adnDB)
# scons doesn't set $HOME, so make sure it's set
os.environ['HOME'] = os.environ.get("HOME", os.path.expanduser("~"))
eupsObj = eups.Eups(root=datapath)
ok, version, reason = eupsObj.setup('astrometry_net_data')
if not ok:
raise ValueError(
"Need %s version of astrometry_net_data (from path: %s): %s" %
(adnDN, datapath, reason))
def getSetupPackages(self):
"""Create a string of all the currently setup LSST software packages,
excluding any locally setup packages (LOCAL:). Also include any
packages specified on the comand line. This string will be
used to substitute within a preJob Template to create an LSST stack
environment that jobs will use.
@return string containing all setup commands, one per line.
"""
e = eups.Eups()
setupProducts = e.getSetupProducts()
a = ""
# create a new list will all products and versions
allProducts = {}
for i in setupProducts:
allProducts[i.name] = i.version
# replace any existing products that we saw on the command line, adding
# them if they're not already there.
if self.opts.setup is not None:
for i, pkg in enumerate(self.opts.setup):
name = pkg[0]
version = pkg[1]
print("name = %s, version = %s" % (name, version))
allProducts[name] = version
# write out all products, except those that are setup locally.
for name in allProducts:
version = allProducts[name]
if self.platform == "lsst":
a = a + "setup -j %s %s\\n\\\n" % (name, version)
else:
if not version.startswith("LOCAL:"):
a = a + "setup -j %s %s\\n\\\n" % (name, version)
return a
def list_packages(eups_env=None):
"""Discover all packages in the stack.
Attempts to do something like the the printProducts function in Eups:
https://github.com/RobertLuptonTheGood/eups/blob/master/python/eups/app.py
However, the function currently uses repos.yaml, and thus is tied to
the packages that are available in lsstsw.
Parameters
----------
eups_env : :class:`eups.Eups` instance, optional
The Eups environment. This function must be run from an enviroment
where Eups is setup.
Returns
-------
packages : dict
A `dict` of :class:`Package` instances representing each Stack package,
keyed by package name.
"""
if eups_env is None:
eups_env = eups.Eups()
product_list = eups_env.findProducts(None, None, None)
# The set of product names available (remove duplicate versions)
product_names = list(set([p.name for p in product_list]))
packages = {}
for name in product_names:
pkg = Package(name, eups_env)
packages[name] = pkg
return packages
def kill(self, runId):
e = eups.Eups()
if not e.isSetup("ctrl_orca"):
print >> sys.stderr, "ctrl_orca not setup, using default version"
e.setup("ctrl_orca")
self.machineSet = self.findMachineSet(runId)
if self.machineSet is None:
raise RuntimeError("No current run with runId " + runId)
self._log("*** orca killed")
subprocess.check_call("$CTRL_ORCA_DIR/bin/shutprod.py 1 " + runId,
shell=True)
print >> sys.stderr, "waiting for production shutdown"
time.sleep(15)
print >> sys.stderr, "killing all remote processes"
for machine in RunConfiguration.machineSets[self.machineSet]:
machine = re.sub(r':.*', "", machine)
if machine.find(".") == -1:
machine = machine + "." + RunConfiguration.defaultDomain
processes = subprocess.Popen(
["ssh", machine, "/bin/ps", "-o", "pid:6,command"],
stdout=subprocess.PIPE)
for line in processes.stdout:
if line.find(runId) != -1:
pid = line[0:6].strip()
# Ignore exit status
subprocess.call(["ssh", machine, "/bin/kill", pid])
processes.wait()
time.sleep(5)
print >> sys.stderr, "unlocking machine set"
self.unlockMachines()
def setUp(self):
os.environ["EUPS_PATH"] = testEupsStack
os.environ["EUPS_FLAVOR"] = "Linux"
os.environ["EUPS_USERDATA"] = os.path.join(testEupsStack, "_userdata_")
self.dbpath = os.path.join(testEupsStack, "ups_db")
self.eups = eups.Eups(quiet=1)
self.eups.quiet = 0
self.eups.unsetupSetupProduct("python")
self.eups.tags.registerTag("beta")
self.eups.tags.registerUserTag("rhl")
def allSetupVersions():
eupsEnv = eups.Eups()
setupProds = eupsEnv.findProducts(tags='setup')
res = dict()
for p in setupProds:
prodInfo = eupsEnv.findSetupVersion(p.name)
if prodInfo[0] is None:
continue
res[p.name] = version(p.name)
return res
def run(args):
#
# Ensure build directory exists and is writable
#
build_dir = args.build_dir
if not os.access(build_dir, os.W_OK):
raise Exception(
"Directory '%s' does not exist or isn't writable." % build_dir)
refs = args.ref
#
# Wire-up the BuildDirectoryConstructor constructor
#
eups_obj = eups.Eups()
if args.exclusion_map:
with open(args.exclusion_map, encoding='utf-8') as fp:
exclusion_resolver = ExclusionResolver.from_file(fp)
else:
exclusion_resolver = ExclusionResolver([])
dependency_module = EupsModule(eups_obj, exclusion_resolver)
if args.version_git_repo:
version_db = VersionDbGit(args.version_git_repo,
args.sha_abbrev_len, eups_obj)
else:
version_db = VersionDbHash(args.sha_abbrev_len, eups_obj)
product_fetcher = ProductFetcher(
build_dir,
args.repos,
dependency_module=dependency_module,
version_db=version_db,
repository_patterns=args.repository_pattern,
no_fetch=args.no_fetch,
tries=args.tries)
p = BuildDirectoryConstructor(build_dir, eups_obj, product_fetcher,
version_db, exclusion_resolver)
#
# Run the construction
#
manifest = p.construct(args.products, refs)
version_db.commit(manifest, args.build_id)
#
# Store the result in build_dir/manifest.txt
#
manifest_fn = os.path.join(build_dir, 'manifest.txt')
with open(manifest_fn, 'w', encoding='utf-8') as fp:
manifest.to_file(fp)
def report(self, logFile):
with open(logFile, "r") as f:
for line in f:
print line,
if line.startswith("Run:"):
runId = re.sub(r'Run:\s+', "", line.rstrip())
if line.startswith("Output:"):
outputDir = re.sub(r'Output:\s+', "", line.rstrip())
e = eups.Eups()
if not e.isSetup("mysqlpython"):
print >> sys.stderr, "*** mysqlpython not setup, skipping log analysis"
else:
print self.orcaStatus(runId, outputDir)
def versions(product):
""" Try to return the current git and eups versions of the given product.
Args
----
product : string or eups Product instance
EUPS product name to query
Returns
-------
gitVersion : standard git version string or None
eupsVersion : eups version name or None
We expect three environments for a running program:
- a setup EUPS directory with a live git repo
- a live git repo in `pwd`
- a setup EUPS-tagged version.
"""
if isinstance(product, str):
eupsEnv = eups.Eups()
eupsProd = eupsEnv.findSetupProduct(product)
else:
eupsProd = product
if eupsProd is None:
eupsVersion = None
prodDir = pathlib.Path().cwd()
else:
eupsVersion = eupsProd.version
prodDir = eupsProd.dir
try:
gitRepo = git.Repo(prodDir)
gitVersion = gitRepo.git.describe(dirty=True, always=True)
except AttributeError:
gitRepo = None
gitVersion = None
except git.InvalidGitRepositoryError:
gitRepo = None
gitVersion = None
# Could check for git-eups conformabity or warn?
return gitVersion, eupsVersion
def run(args):
# Ensure build directory exists and is writable
build_dir = args.build_dir
if not os.access(build_dir, os.W_OK):
raise Exception(
"Directory '%s' does not exist or isn't writable." % build_dir)
# Build products
eups_obj = eups.Eups()
progress = ProgressReporter(sys.stdout)
manifest_fn = os.path.join(build_dir, 'manifest.txt')
with open(manifest_fn, encoding='utf-8') as fp:
manifest = Manifest.from_file(fp)
b = Builder(build_dir, manifest, progress, eups_obj)
b.rm_status()
retcode = b.build()
b.write_status()
sys.exit(retcode == 0)
def execute(self):
productName = versionName = None
if len(self.args) > 0:
productName = self.args[0]
if len(self.args) > 1:
versionName = self.args[1]
if self.opts.unsetup:
cmdName = "unsetup"
else:
cmdName = "setup"
if not self.opts.noCallbacks:
try:
eups.commandCallbacks.apply(None, cmdName, self.opts, self.args)
except eups.OperationForbidden as e:
e.status = 255
raise
except Exception as e:
e.status = 9
raise
if self.opts.exact_version and self.opts.inexact_version:
self.err("Specifying --exact --inexact confuses me, so I'll ignore both")
self.opts.exact_version = False
self.opts.inexact_version = False
if self.opts.tablefile: # we're setting up a product based only on a tablefile
if self.opts.unsetup:
self.err("Ignoring --table as I'm unsetting up a product")
self.opts.tablefile = None
else:
if not os.path.exists(self.opts.tablefile) and self.opts.tablefile != "none":
self.err("%s does not exist" % self.opts.tablefile)
print(self.clo.get_usage(), file=utils.stderr)
return 3
self.opts.tablefile = os.path.abspath(self.opts.tablefile)
if not productName:
self.opts.productDir = os.path.dirname(self.opts.tablefile)
productName = os.path.splitext(os.path.basename(self.opts.tablefile))[0]
if not self.opts.productDir and not productName:
self.err("please specify at least a product name or use -r")
print(self.clo.get_usage(), file=utils.stderr)
return 3
if self.opts.productDir:
self.opts.productDir = os.path.abspath(self.opts.productDir)
try:
productName = eups.utils.guessProduct(os.path.join(self.opts.productDir, "ups"), productName)
except EupsException as e:
e.status = 4
raise
except RuntimeError as e:
if self.opts.tablefile:
pass # They explicitly listed the table file to use, so trust them
else:
e.status = 4
raise
if not productName:
self.err("Please specify a product")
print(self.clo.get_usage(), file=utils.stderr)
return 3
if self.opts.nodepend:
if self.opts.max_depth > 0:
self.err("You may not specify both --just and --max_depth")
return 3
self.opts.max_depth = 0
path = eups.Eups.setEupsPath(self.opts.path, self.opts.dbz)
locks = lock.takeLocks("setup", path, lock.LOCK_SH,
nolocks=self.opts.nolocks, verbose=self.opts.verbose - self.opts.quiet)
#
# Do the work
#
status = 0
try:
try:
Eups = eups.Eups(flavor=self.opts.flavor, path=self.opts.path,
dbz=self.opts.dbz, # root=self.opts.productDir,
readCache=False, force=self.opts.force,
quiet=self.opts.quiet, verbose=self.opts.verbose,
noaction=self.opts.noaction, keep=self.opts.keep,
ignore_versions=self.opts.ignoreVer, setupType=self.opts.setupType,
max_depth=self.opts.max_depth, vro=self.opts.vro,
exact_version=self.opts.exact_version, cmdName="setup")
Eups._processDefaultTags(self.opts)
if not self.opts.noCallbacks:
try:
eups.commandCallbacks.apply(Eups, cmdName, self.opts, self.args)
except eups.OperationForbidden as e:
e.status = 255
raise
except Exception as e:
e.status = 9
raise
Eups.selectVRO(self.opts.tag, self.opts.productDir, versionName, self.opts.dbz,
inexact_version=self.opts.inexact_version, postTag=self.opts.postTag)
if self.opts.tag:
for t in self.opts.tag:
if Eups.isUserTag(t):
break
Eups.includeUserDataDirInPath()
for user in Eups.tags.owners.values():
Eups.includeUserDataDirInPath(eups.utils.defaultUserDataDir(user))
#
# If they specify a productDir in addition to a complete product + version specification
# Use that product + version's expanded table file, but this directory
#
if self.opts.productDir and not self.opts.tablefile and productName and versionName:
prod = Eups.findProduct(productName, versionName)
if not prod:
self.err("Unable to find %s %s" % (productName, versionName))
return 3
tablefile = prod.tablefile
else:
tablefile=self.opts.tablefile
cmds = eups.setup(productName, versionName, self.opts.tag, self.opts.productDir,
Eups, fwd=not self.opts.unsetup, tablefile=tablefile,
postTags=self.opts.postTag)
except EupsException as e:
e.status = 1
raise
except Exception as e:
e.status = -1
raise
finally:
lock.giveLocks(locks, self.opts.verbose)
if Eups.verbose > 3:
print("\n\t".join(["Issuing commands:"] + cmds), file=sys.stderr)
print(";\n".join(cmds))
return status
#! /usr/bin/env python
#
import os, sys, re
import eups
from eups import distrib
baseURL = re.sub(r'\|.*$', '', os.environ.get('EUPS_PKGROOT'))
eupsenv = eups.Eups()
repos = distrib.Repository(eupsenv, baseURL)
current = repos.listPackages(tag='current', flavor='generic', queryServer=True)
for prod, version, flavor in current:
installed = eupsenv.findProduct(prod, version)
if not installed:
# print >> sys.stderr, "Current product not installed:", prod, version
continue
if not installed.isTagged("current"):
print >> sys.stderr, "Synchronizing product:", prod, version
eupsenv.assignTag('current', prod, version)
print >> sys.stderr, "Stack is synchronized with tag=current on server."
def setupAstrometryNetData(self):
if self.astrometryNetData is not None:
ok, version, reason = eups.Eups().setup('astrometry_net_data', self.astrometryNetData)
def main(qaName,
wwwRoot=None,
force=False,
forceClean=False,
color="blue",
project='lsst'):
# verify that we have WWW_ROOT and TESTING_DISPLAYQA_DIR
envVars = ['TESTING_DISPLAYQA_DIR']
if wwwRoot is None:
envVars.append('WWW_ROOT')
missing = []
for envVar in envVars:
if not os.environ.has_key(envVar):
missing.append(envVar)
if len(missing) > 0:
raise Exception("Missing environment variable(s):\n",
"\n".join(missing))
if wwwRoot is None:
wwwRoot = os.environ['WWW_ROOT']
dqaDir = os.environ['TESTING_DISPLAYQA_DIR']
# get our version
if haveEups:
e = eups.Eups()
version, eupsPathDir, productDir, tablefile, flavour = e.findSetupVersion(
'testing_displayQA')
else:
path, v = os.path.split(dqaDir)
if re.search("^v[\d.-_]+$", v):
version = v
else:
version = "working-copy-%s" % (v)
# check to see if qaname already exists with different version
dest = os.path.join(wwwRoot, qaName)
if os.path.exists(dest):
vFile = os.path.join(dest, "version")
if os.path.exists(vFile):
fp = open(vFile)
v = fp.readlines()[0]
v = v.strip()
fp.close()
if not force:
if (v != version):
raise Exception(
qaName,
"already exists with different version. Use -f to force."
)
else:
print "QA site '" + qaName + "' already exists at:"
print " ", dest
print "Exiting (nothing done)."
sys.exit()
else:
fp = open(os.path.join(dest, "version"), 'w')
fp.write("%s\n" % version)
fp.close()
else:
os.mkdir(dest)
fp = open(os.path.join(dest, "version"), 'w')
fp.write("%s\n" % version)
fp.close()
# copy the www/ to the destination
src = os.path.join(dqaDir, "www")
patterns = ["php", "js"]
files = []
for p in patterns:
files += glob.glob(os.path.join(src, "[a-zA-Z]*." + p))
doc = os.path.join(dqaDir, "doc")
files += glob.glob(os.path.join(doc, "README"))
for f in files:
dir, script = os.path.split(f)
print "installing: ", script
cmd = "cp -r %s %s" % (f, dest)
os.system(cmd)
# handle the css and favicon files based on color chosen
style_base = "style_" + color + ".css"
favicon_base = project + "_favicon_" + color + ".png"
files = [
[
style_base,
os.path.join(src, style_base),
os.path.join(dest, "style.css")
],
[
favicon_base,
os.path.join(src, favicon_base),
os.path.join(dest, "favicon.ico")
],
]
for file_base, file_color, file_dest in files:
if os.path.exists(file_color):
print "installing: ", file_base
os.system("cp %s %s" % (file_color, file_dest))
else:
color_files = glob.glob(os.path.join(src, "style_*.css"))
colors = []
for f in color_files:
m = re.search("style_(.*).css", f)
if m:
colors.append(m.group(1))
msg = "Cannot install color '" + color + "'. "
msg += "Available colors: " + ", ".join(colors)
print msg
sys.exit()
print ""
print "Created new QA site served from:"
print " ", dest
envFile = os.path.join(dest, "environment.php")
if forceClean:
print ""
print "Cleaning existing data from", dest, ":"
dbFile = os.path.join(dest, "db.sqlite3")
for f in [dbFile, envFile]:
if os.path.exists(f):
print " ", os.path.split(f)[1]
os.remove(f)
for testDir in glob.glob(os.path.join(dest, "test_*")):
print " ", os.path.split(testDir)[1]
shutil.rmtree(testDir)
# touch the environment file to make sure it's there.
with file(envFile, 'a'):
os.utime(envFile, None)
def __init__(self, *args, **kwargs):
super(DreamServer, self).__init__(*args, **kwargs)
self.Eups = eups.Eups()
# Working with real files from here on out
assert self.base.startswith("dream:")
self.base = self.base[len("dream:"):]
def testEndToEnd(self):
"""Test ISR, CcdAssembly, CrSplit, ImgChar, SFM pipelines"""
#Setup up astrometry_net_data
# Note - one of datarel's dependencies causes setup of
# 'astrometry_net_data cfhttemplate' version;
# datarel needs imsim_*.
ver = 'imsim-2010-12-17-1'
print "Setting up astrometry_net_data", ver
# XXX what is actually used from this setup -- a path in the env?
ok, version, reason = eups.Eups().setup("astrometry_net_data", versionName=ver)
if not ok:
raise ValueError("Couldn't set up version '%s' of astrometry_net_data: %s" % (ver, reason))
afwdataDir = lsst.utils.getPackageDir("afwdata")
inputRoot = os.path.join(afwdataDir, "ImSim")
if os.path.exists("endToEnd.py"):
outputRoot = "."
else:
outputRoot = "tests"
registryPath = os.path.join(inputRoot, "registry.sqlite3")
bf = dafPersist.ButlerFactory(mapper=LsstSimMapper(root=inputRoot))
inButler = bf.create()
obf = dafPersist.ButlerFactory(mapper=LsstSimMapper(root=outputRoot,
registry=registryPath))
outButler = obf.create()
stat = subprocess.call(["runImSim.py", "-T", "--force",
"-i", inputRoot, "-o", outputRoot,
"-v", "85408556", "-r", "2,3", "-s", "1,1"])
self.assertEqual(stat, 0, "Error while running end to end test")
fname = "psf/v85408556-fr/R23/S11.boost"
stat = subprocess.call(["cmp",
os.path.join(outputRoot, fname), os.path.join(inputRoot, fname)])
psfDiffers = (stat != 0)
if psfDiffers:
print 'PSF differs (but carrying on and failing later...)'
results = []
for datasetType in ("icSrc", "src", "calexp"):
msg = compare(outButler, inButler, datasetType,
visit=85408556, raft="2,3", sensor="1,1")
results.append((datasetType, msg))
if msg is not None:
print 'Dataset type', datasetType, 'differs (but carrying on and failing later...)'
print 'message:', msg
for snap in (0, 1):
msg = compare(outButler, inButler, "sdqaCcd",
visit=85408556, snap=snap, raft="2,3", sensor="1,1")
results.append(('sdqaCcd snap %i' % snap, msg))
if msg is not None:
print 'Snap', snap, 'sdqaCCD differs (but carrying on and failing later...)'
print 'message:', msg
for channel in inButler.queryMetadata("raw", "channel"):
msg = compare(outButler, inButler, "sdqaAmp",
visit=85408556, snap=snap, raft="2,3", sensor="1,1",
channel=channel)
print 'channel:', channel
results.append(('sdqaAmp snap %i channel ' % (snap) + str(channel), msg))
if msg is not None:
print 'Snap', snap, 'channel', channels, 'sdqaAmp differs (but carrying on and failing later...)'
print 'message:', msg
# Deferred failure!
self.assertFalse(psfDiffers)
for datasetType,msg in results:
self.assert_(msg is None, msg)
def setUp(self):
self.eups = eups.Eups()
def __init__(self,
name,
productName=None,
configFile=None,
makeCmdrConnection=True,
acceptCmdrs=True,
productPrefix="",
modelNames=(),
idDict=None):
""" Build an Actor.
Args:
name - the name of the actor: what name we are advertised as to tron.
productName - the name of the product; defaults to .name
configFile - the full path of the configuration file; defaults
to $PRODUCTNAME_DIR/etc/$name.cfg
makeCmdrConnection
- establish self.cmdr as a command connection to the hub.
This needs to be True if we send commands or listen to keys.
acceptCmdrs - allow incoming commander connections.
modelNames - a list of actor names, whose key dictionaries we want to
listen to and have in our .models[]
"""
# Define/save the actor name, the product name, the product_DIR, and the
# configuration file.
self.name = name
self.productName = productName if productName else self.name
# optional idDict
self.idDict = dict() if idDict is None else idDict
# Incomplete/missing eups environment will make us blow
# up. Note that we allow "actorName" as equivalent to
# "ics_actorName". This was a bad decision made a long time
# ago.
eupsEnv = eups.Eups()
setupProds = eupsEnv.findProducts(tags='setup', name=self.productName)
if len(setupProds) == 0:
setupProds = eupsEnv.findProducts(tags='setup',
name=f'ics_{self.productName}')
if len(setupProds) == 0:
raise RuntimeError(
f"cannot figure out what our eups product name is. {self.productName} is not useful"
)
self.product_dir = setupProds[0].dir
self.configFile = configFile if configFile else \
os.path.expandvars(os.path.join(self.product_dir, 'etc', '%s.cfg' % (self.name)))
# Missing config bits should make us blow up.
self.configFile = os.path.expandvars(self.configFile)
self._reloadConfiguration()
self.logger.info('%s starting up....' % (name))
self.parser = CommandParser()
self.acceptCmdrs = acceptCmdrs
self.models = {}
if modelNames and not makeCmdrConnection:
self.logger.warn(
"modelNames were requested but makeCmdrConnection is False. Forcing that to True."
)
makeCmdrConnection = True
# The list of all connected sources.
listenInterface = self.actorConfig['listen']['interface']
if listenInterface == 'None':
listenInterface = None
listenPort = self.actorConfig['listen']['port']
# Allow for a passive actor which does not accept external commands.
if not self.acceptCmdrs:
listenInterface = None
self.commandSources = cmdLinkManager.CommandLinkManager(
self, port=listenPort, interface=listenInterface)
# IDs to send commands to ourself.
self.selfCID = self.commandSources.fetchCid()
self.synthMID = 1
# The Command which we send uncommanded output to.
self.bcast = actorCmd.Command(self.commandSources,
'self.0',
0,
0,
None,
immortal=True)
# commandSets are the command handler packages. Each handles
# a vocabulary, which it registers when loaded.
# We gather them in one place mainly so that "meta-commands" (init, status)
# can find the others.
self.commandSets = {}
if acceptCmdrs:
self.logger.info("Creating validation handler...")
self.handler = validation.CommandHandler()
self.logger.info("Attaching actor command sets...")
self.attachAllCmdSets()
self.logger.info("All command sets attached...")
self.commandQueue = queue.Queue()
self.shuttingDown = False
if makeCmdrConnection:
self.cmdr = CmdrConnection.Cmdr(name, self)
self.cmdr.connectionMade = self._connectionMade
self.cmdr.connectionLost = self.connectionLost
self.cmdr.connect()
self._initializeHubModels(modelNames)
else:
self.cmdr = None
def main(argv):
parser = argparse.ArgumentParser(
description=
"Install a setuptools module in the current directory and declare it to EUPS",
)
parser.add_argument("name",
metavar="NAME",
type=str,
help="name of the EUPS product to declare")
parser.add_argument("version",
nargs="?",
type=str,
default="system",
help="EUPS version 'number'")
parser.add_argument("--productDir",
"-r",
metavar="DIR",
help="root directory for installed product",
default=None)
parser.add_argument(
"--dep",
"-d",
action="append",
metavar="PRODUCTS",
dest="deps",
default=["python"],
help=
"names of EUPS products on which this package depends (must be setup)")
args = parser.parse_args(argv)
eupsObj = eups.Eups()
if args.productDir is None:
productDir = os.path.join(eupsObj.path[0], eupsObj.flavor, args.name,
args.version)
else:
productDir = args.productDir
subprocess.call("python setup.py install --home=" + productDir, shell=True)
depDict = {}
tableFile = os.path.join(productDir, "ups", "{}.table".format(args.name))
with open(tableFile, "w") as table:
for dep in args.deps:
table.write("setupRequired({})\n".format(dep))
table.write("envPrepend(PYTHONPATH, {})".format(
os.path.join("${PRODUCT_DIR}", "lib", "python")))
if os.path.isdir(os.path.join(productDir, "bin")):
table.write("envPrepend(PATH, {})".format(
oos.path.join("${PRODUCT_DIR}", "bin")))
try:
os.makedirs(os.path.join(productDir, "ups"))
except OSError:
pass
subprocess.call("eups expandtable -i " + tableFile, shell=True)
eups.declare(productName=args.name,
versionName=args.version,
productDir=productDir,
tablefile=tableFile)
def recordEnvironment(self):
"""Record the software environment of the pipeline."""
setupList = eups.Eups().listProducts(setup=True)
# self._realRecordEnvironment(self._rundb, setupList)
self._realRecordEnvironment(self._globalDb, setupList)
def _get_eups_version(name: str, tag: str) -> str:
eups_product = eups.Eups().findProduct(name, eups.Tag(tag))
if eups_product is None:
raise LookupError(f"No metapackage {name!r} with tag {tag} found.")
return eups_product.version
nolocks=self.opts.nolocks,
verbose=self.opts.verbose - self.opts.quiet)
#
# Do the work
#
status = 0
try:
try:
Eups = eups.Eups(
flavor=self.opts.flavor,
path=self.opts.path,
dbz=self.opts.dbz, # root=self.opts.productDir,
readCache=False,
force=self.opts.force,
quiet=self.opts.quiet,
verbose=self.opts.verbose,
noaction=self.opts.noaction,
keep=self.opts.keep,
ignore_versions=self.opts.ignoreVer,
setupType=self.opts.setupType,
max_depth=self.opts.max_depth,
vro=self.opts.vro,
exact_version=self.opts.exact_version,
cmdName="setup")
Eups._processDefaultTags(self.opts)
if not self.opts.noCallbacks:
try:
eups.commandCallbacks.apply(Eups, cmdName, self.opts,
self.args)
except eups.OperationForbidden, e:
def __init__(self, args):
self.datetime = time.strftime("%Y_%m%d_%H%M%S")
self.user = pwd.getpwuid(os.getuid())[0]
if self.user == 'buildbot':
RunConfiguration.pipeQaBase = re.sub(r'dev', 'buildbot',
RunConfiguration.pipeQaBase)
RunConfiguration.pipeQaDir = re.sub(r'dev', 'buildbot',
RunConfiguration.pipeQaDir)
self.dbUser = DbAuth.username(RunConfiguration.dbHost,
str(RunConfiguration.dbPort))
self.hostname = socket.getfqdn()
self.fromAddress = "%s@%s" % (self.user, self.hostname)
self.options, self.args = self.parseOptions(args)
# Handle immediate commands
if self.options.printStatus:
self.printStatus()
sys.exit(0)
if self.options.report is not None:
self.report(
os.path.join(self.options.output, self.options.report, "run",
"run.log"))
sys.exit(0)
if self.options.listRuns:
self.listRuns(self.options.listRuns)
sys.exit(0)
if self.options.listInputs:
self.listInputs()
sys.exit(0)
if self.options.linkLatest is not None:
self.linkLatest(self.options.linkLatest)
sys.exit(0)
if self.options.kill is not None:
self.kill(self.options.kill)
sys.exit(0)
if self.options.hosts is not None:
self.hosts()
sys.exit(0)
if self.arch is None:
if self.options.arch is None:
raise RuntimeError("Architecture is required")
self.arch = self.options.arch
if re.search(r'[^a-zA-Z0-9_]', self.options.runType):
raise RuntimeError("Run type '%s' must be one word" %
(self.options.runType, ))
self.collectionName = re.sub(r'\.', '_', RunConfiguration.collection)
runIdProperties = dict(user=self.user,
dbUser=self.dbUser,
coll=self.collectionName,
runType=self.options.runType,
datetime=self.datetime)
# When resuming a run, use provided runID
if self.options.resumeRunId is None:
self.runId = RunConfiguration.runIdPattern % runIdProperties
else:
self.runId = self.options.resumeRunId
runIdProperties['runid'] = self.runId
dbNamePattern = "%(dbUser)s_%(coll)s_u_%(runid)s"
self.dbName = dbNamePattern % runIdProperties
self.inputBase = os.path.join(RunConfiguration.inputBase,
self.options.input)
self.inputDirectory = os.path.join(self.inputBase,
RunConfiguration.collection)
self.outputDirectory = os.path.join(self.options.output, self.runId)
self.outputDirectory = os.path.abspath(self.outputDirectory)
if self.options.resumeRunId is None:
if os.path.exists(self.outputDirectory):
raise RuntimeError("Output directory %s already exists" %
(self.outputDirectory, ))
os.mkdir(self.outputDirectory)
elif not os.path.exists(self.outputDirectory):
raise RuntimeError(
"Output directory %s does not exist for resumed run" %
(self.outputDirectory, ))
self.pipeQaUrl = RunConfiguration.pipeQaBase + self.dbName + "/"
self.eupsPath = os.environ['EUPS_PATH']
e = eups.Eups(readCache=False)
self.setups = dict()
for product in e.getSetupProducts():
if product.name != "eups":
self.setups[product.name] = \
re.sub(r'^LOCAL:', "-r ", product.version)
# TODO -- load policy and apply overrides
self.options.override = None
