def _get_lib_depends(self, lib_dict, lib_list):
for lib_name in listify(lib_list):
lib_keys = listify(self._get_matching_lib_keys(lib_name))
if len(lib_keys) == 1:
# Matched internal library
lib_key = lib_keys[0]
# Search for the found library dependencies
libs, cpp_paths, lib_paths = self._libs[lib_key][
'lib_obj'].GetLib()
lib_dict['CPPPATH'].extend(cpp_paths)
lib_dict['LIBS'].extend(libs)
lib_dict['LIBPATH'].extend(lib_paths)
# Get the dependencies
self._get_lib_depends(lib_dict,
self._libs[lib_key]['lib_deps'])
elif len(lib_keys) > 1:
# Matched multiple internal libraries - probably bad!
msg = "Library identifier \'%s\' matched %d libraries (%s). Please use a fully qualified identifier instead!" % (
lib_name, len(lib_keys), ', '.join(lib_keys))
Exit(msg)
else: # empty lib_keys
msg = "Library identifier \'%s\' didn\'t match any library. Is it a typo?" % (
lib_name)
Exit(msg)
def _build_alias_executable(self, alias, prog_name, module_name,
prog_version, sources, link_libs, *args,
**kwargs):
lib_nodes = []
# Store the kwarg flags
cpp_paths = listify(kwargs.pop('CPPPATH', list()))
ext_libs = listify(kwargs.pop('LIBS', list()))
lib_paths = listify(kwargs.pop('LIBPATH', list()))
# Get the dependencies for the executable
libs_dict = self._shared_libs.GetLibraries(link_libs)
# Extend the stored flags with the lib dependencies
cpp_paths.extend(libs_dict['CPPPATH'])
ext_libs.extend(libs_dict['LIBS'])
lib_paths.extend(libs_dict['LIBPATH'])
# Copy back the dependencies to the kwargs
kwargs['CPPPATH'] = cpp_paths
kwargs['LIBS'] = ext_libs
kwargs['LIBPATH'] = lib_paths
exec_path = self.env['BUILD_DIR'] + '/'
obj_nodes = (self._build_default_objects(module_name, sources))
prog_nodes = self.env.Program(exec_path + prog_name, obj_nodes,
**kwargs)
# Create an alias target for the lib
#if self._target in BUILD_TARGETS:
self.createAlias(alias, prog_nodes)
self.createAlias(module_name, prog_nodes)
def update_task(self, task, properties=None, event_name=None,
event_type=EVENT_TYPE_INFO, message=None, details=None):
"""
Updates the specified properties of the specified MBSTask object
"""
task_doc = task.to_document()
q = {
"_id": task.id
}
u = {}
# construct $set operator
if properties:
properties = listify(properties)
u["$set"] = {}
for prop in properties:
u["$set"][prop] = task_doc.get(prop)
# construct the $push
if event_name or message:
log_entry = task.log_event(name=event_name, event_type=event_type,
message=message, details=details)
u["$push"] = {"logs": log_entry.to_document()}
self.update(spec=q, document=u)
def _build_alias_lib(self, alias, lib_name, module_name, module_version,
sources, headers, libs, *args, **kwargs):
# TODO: Append the module version to the key, modifiy the querry to be able to search for specific version, or latest version (dont care version)
# Store resulting library node in shared dictionary
obj_targets = self._build_default_objects(module_name, sources)
# Create a lib node.
lib_node = self.env.Library(target=lib_name,
source=obj_targets,
*args,
**kwargs)
# Create unique library key from module and library name
key = LibraryList.CreateLibraryKey(module_name, lib_name)
# Store the library key in shared libs list
self._shared_libs.add(key, InternalLibrary(lib_name, lib_node), libs)
# Create an alias target for the lib
self.createAlias(alias, lib_node)
# Make sure headers are list
headers = listify(headers)
if len(headers) > 0:
# Remove redundant elements
headers = remove_redundant(headers)
# Get the isntall directory for the active variant
inc_path = self.env['INC_DIR'] + '/'
# Install every header file to the shared <INC_DIR> keep the folder hierarchy for the header files
inc_node = [
self.env.Install(
self.createNode([os.path.split(inc_path + h)[0]]), h)
for h in headers
]
# Create alias targets for the header nodes
self.createAlias(alias, inc_node)
self.createAlias('install', inc_node)
def update_task(self, task, properties=None, event_name=None,
event_type=EventType.INFO, message=None, details=None,
error_code=None,
**update_kwargs):
"""
Updates the specified properties of the specified MBSTask object
"""
task_doc = task.to_document()
q = {
"_id": task.id
}
u = {}
# log the event as needed
if event_name or message:
log_entry = task.log_event(name=event_name, event_type=event_type,message=message, details=details,
error_code=error_code)
# push if "logs" property is not included
if not (properties and "logs" in properties):
u["$push"] = {"logs": log_entry.to_document()}
# construct $set operator
if properties:
properties = listify(properties)
u["$set"] = {}
for prop in properties:
u["$set"][prop] = task_doc.get(prop)
self.update(spec=q, document=u, **update_kwargs)
def _build_default_objects(self, module, sources):
# Make sure headers are list
sources = listify(sources)
# Remove redundant elements
sources = remove_redundant(sources)
obj_targets = []
if len(sources) > 0:
obj_path = self.env['OBJ_PATH'] + '/' + module + '/'
# TODO: Object files maybe should placed into $OBJ_DIR/module/path/src.o. Now it will be just placed to: $OBJ_DIR/module/src.o
for item in sources:
# '#' sign here is mandatory, otherwise the include path is wrong
# TODO: Why this is not working without the '#' sign?
# Create a node from the include directory
inc_dir = [self.createNode(['#' + module])
] + [self.createNode([self.env['CPPPATH']])]
# Get the base fileanem from the hive source
base_item = os.path.splitext(os.path.basename(item))[
0] # strip relative path part and extension
# TODO: Cannot create nodes from item,s Why?
obj_node = self.env.Object(target=obj_path + base_item,
source=item,
CPPPATH=inc_dir)
# Append the object node to targets
obj_targets.append(obj_node)
# Create an alias target from object node
self.createAlias(module + '/' + item, obj_node)
return obj_targets
def update_task(self,
task,
properties=None,
event_name=None,
event_type=EVENT_TYPE_INFO,
message=None,
details=None):
"""
Updates the specified properties of the specified MBSTask object
"""
task_doc = task.to_document()
q = {"_id": task.id}
u = {}
# construct $set operator
if properties:
properties = listify(properties)
u["$set"] = {}
for prop in properties:
u["$set"][prop] = task_doc.get(prop)
# construct the $push
if event_name or message:
log_entry = task.log_event(name=event_name,
event_type=event_type,
message=message,
details=details)
u["$push"] = {"logs": log_entry.to_document()}
self.update(spec=q, document=u)
def __init__(self,
name,
libs=list(),
include_paths=list(),
lib_paths=list()):
"""Initialize external library instance.
@param lib_name Symbolic name of library (or library-group)
@param libs Identifiers of libraries to link with
(if not specified, `lib_name` is used)
@param include_paths Additional include search paths
@param lib_paths Additional library search paths
"""
super(ExternalLibrary, self).__init__(name)
self._libs = listify(libs) if len(libs) else [name]
self._cpp_paths = listify(include_paths)
self._lib_paths = listify(lib_paths)
def scan_modules(modules_list, max_depth, file_skip_list):
_CACHED_MODULES = listify(modules_list)
def modules():
"""Generate modules to build.
Each module is a directory with a SConscript file.
"""
if not _CACHED_MODULES:
# Build the cache
def build_dir_skipper(dirpath):
"""Return True if `dirpath` is the build base dir."""
return os.path.normpath('out') == os.path.normpath(dirpath)
def hidden_dir_skipper(dirpath):
"""Return True if `dirpath` last dir component begins with '.'"""
last_dir = os.path.basename(dirpath)
return last_dir.startswith('.')
for module_path in module_dirs_generator(
max_depth=max_depth,
followlinks=False,
dir_skip_list=[build_dir_skipper, hidden_dir_skipper],
file_skip_list=file_skip_list):
_CACHED_MODULES.append(module_path)
# Yield modules from cache
for module in _CACHED_MODULES:
yield module
return modules
def __init__(self, name, title, files, stype, cfg, trees,
cuts = DEFAULT_TRUE, weights = '1.0', scale = 1.0, lumi = 1.0,
channel = '', roottree = 'HWWTree'):
assert(type(name) is str)
assert(type(title) is str)
assert(type(cuts) is str)
assert(type(weights) is str)
assert(type(scale) is float)
self.__root = roottree # << the root tree (all other trees will be friends)
self.__trees = None # << list of TChain's controlled by this sample
self.__tree_names = utils.uniqify(utils.listify(trees) + [self.__root])
self.__name = name # << name of the sample
self.__title = title # << title (used in legends, &c.)
self.__cuts = cuts.replace(' ','') # << cuts that define sample selection
self.__cuts_base = self.__cuts
self.__applied_cuts = None
self.__weights = weights.replace(' ','') # << event weights
self.__weights_base = self.__weights
self.__scale = scale # << product of scale factors
self.__counters = {} # << counters maintained by this sample
self.__histograms = {} # << histograms produced by this sample
self.__internals = [] # << other internal objects to keep
self.__locked = False # << whether the sample is mutable (ie., can produce histograms)
self.__selections = {} # << named selections of this sample
self.__uid = utils.uuid()
samplebase.__init__(self, files, lumi, stype, cfg, channel, title)
def configure(self):
''' Load trees contained in this sample, set root tree (all other trees are friends) '''
self.set_style()
if len(self.__tree_names) == 0:
self.__locked = True
return
trees = { }
for tname in utils.listify(self.__tree_names): # << load the TChain's using all files in sample
nfiles = 0
t = ROOT.TChain(tname, tname)
ROOT.SetOwnership(t, False)
for fpattern in self.get_files():
for fname in glob(os.sep.join([self.get_cfg().base,fpattern])):
n = t.Add(fname.replace('//','/'))
if n <= 0:
raise IOError('[%s] does not exist or does not contain [%s]'%(fname,tname))
nfiles += n
if nfiles != 0:
trees[tname] = t
else:
self.log.info('no files matching patterns [%s] exist, sample not defined'%(str(self.get_files())) )
self.__locked = True
return
trees[self.__root].LoadTree(0)
self.__trees = trees
if self.__root not in self.__trees.keys():
self.log.warning('it seems that [%s] is not in list of input trees'%(self.__root))
self.__root = self.__tree_names[0]
for k, v in trees.items():
if k == self.__root:
continue
self.log.verbose('setting tree [%s] as friend to root tree [%s]'%(tname, self.__root))
trees[self.__root].AddFriend(v)
return
def send_notification(self, subject, message, recipient=None):
try:
logger.info("Sending notification email...")
msg = MIMEText(message.encode('utf-8'), 'plain', 'UTF-8')
to_address = listify(recipient or self._to_address)
msg['From'] = self.from_address
msg['To'] = ",".join(to_address)
if subject:
msg['Subject'] = subject
smtp = smtplib.SMTP(self.smtp_host)
if (self.smtp_username is not None or
self.smtp_password is not None):
smtp.login(self.smtp_username, self.smtp_password)
smtp.sendmail(self.from_address, to_address, msg.as_string())
smtp.quit()
logger.info("Email sent successfully!")
except Exception, e:
logger.error("Error while sending email:\n%s" %
traceback.format_exc())
def _resolve_sources(self, sources):
"""
Add a file, glob pattern or another bundle to this bundle's
contents.
"""
sources = listify(sources)
contents = []
for source in sources:
# Glob pattern, so expand into files
if isinstance(source, str) and '*' in source:
contents.extend(self._resolve_sources(glob(join(self.static_folder, source))))
# Nested bundles are cool
elif type(source) is Bundle:
contents.append(source)
# Files are fine; check the static_folder
elif isfile(realpath(join(self.static_folder, source))):
contents.append(realpath(join(self.static_folder, source)))
# Check for direct path
elif isfile(realpath(source)):
contents.append(realpath(source))
else:
raise Exception('Cannot find file \'%s\'' % source)
return contents
def links(self, page, exclude = False, require = None):
exclude, require = map(lambda m: make_chain(map(matcher, listify(m)),
merge = lambda returns: reduce(lambda a, b: bool(a) | bool(b), returns, False)),
[exclude, require])
return BeautifulSoup(page).findAll(
name = 'a',
href = lambda value: value != None and (require(value) and not exclude(value)))
def links(self, page, exclude=False, require=None):
exclude, require = map(
lambda m: make_chain(map(matcher, listify(m)),
merge=lambda returns: reduce(
lambda a, b: bool(a) | bool(b), returns,
False)), [exclude, require])
return BeautifulSoup(page).findAll(
name='a',
href=lambda value: value != None and
(require(value) and not exclude(value)))
def __init__(self, files, lumi, stype, cfg, chan, title):
self.log = msg.msglog(title, 'debug')
if stype not in self.alltypes:
self.log.error('invalid sample mode: [%s] not one of %s'%(stype, str(self.alltypes)))
raise RuntimeError
self.__files = utils.listify(files) # << list of files
self.__stype = stype # << type of sample: bkg, sig, or dat
self.__cfg = cfg # << configuration tag object
self.__lumi = lumi # << luminosity of the sample
self.__channel = chan # << channel identifier
self.configure()
def __init__(self, samples = [], cfg = None, name = 'samplelist', fromfile = None):
self.__name = name
self.__cfg = cfg
self.__samples = {}
self.__file = None
self.log = msg.msglog(name, 'debug')
if fromfile != None:
self.load(fromfile)
else:
for item in utils.listify(samples):
self.__samples[item.get_name()] = item
pass
def combine_scheds(pcts, scheds):
assert sum(pcts) == 1.
pcts = tensor([0] + listify(pcts))
assert torch.all(pcts >= 0)
pcts = torch.cumsum(pcts, 0)
def _inner(pos):
idx = (pos >= pcts).nonzero().max()
actual_pos = (pos - pcts[idx]) / (pcts[idx + 1] - pcts[idx])
return scheds[idx](actual_pos)
return _inner
def get_document_text(self, ids):
text = ''
for doc in self.get_document_by_ids(ids):
for f in self.more_like_this_params['fields']:
if f not in doc:
continue
doc[f] = utils.listify(doc[f])
# keep intact for tagger analyzer
if f.endswith('.terms') or f.endswith('.raw'):
text += ' ' + ' '.join('"%s"' % term for term in doc[f])
else:
text += ' ' + ' '.join(doc[f])
return text.strip()
def get_detailed_scores(self, item_ids, query_item_ids=None, max_terms=20):
"""Returns detailed statistics about the matched items.
This will assume the same items previously queried unless otherwise
specified by 'query_item_ids'.
"""
item_ids = utils.listify(item_ids)
logger.info('Computing detailed scores ...')
scores = self._compute_detailed_scores(item_ids, query_item_ids, max_terms)
self._update_time_taken()
return scores
def sparsify(self, P, epsilon=SMALL_VALUE):
"""
Returns the sparsified distribution, anything smaller than epsilon is
considered as zero and NOT recorded.
"""
Z = {}
total = sum(v for v in P.values())
for k, v in P.items():
if v < SMALL_VALUE:
continue
k = [x / self.period for x in listify(k)]
k = ",".join(str(x) for x in k)
Z[k] = v / total
return Z
def config_lookup_cluster_by_server(server, lookup_type=LOOKUP_TYPE_ANY):
clusters = get_configured_clusters()
lookup_type = listify(lookup_type)
for t in lookup_type:
result = None
if t == LOOKUP_TYPE_MEMBER:
result = filter(lambda c: c.has_member_server(server), clusters.values())
elif t == LOOKUP_TYPE_CONFIG_SVR:
result = filter(lambda c: cluster_has_config_server(c, server), clusters.values())
elif t == LOOKUP_TYPE_SHARDS:
result = filter(lambda c: cluster_has_shard(c, server), clusters.values())
if result:
return result[0]
def get_detailed_scores(self, item_ids, query_item_ids=None, max_terms=20):
"""Returns detailed statistics about the matched items.
This will assume the same items previously queried unless otherwise
specified by 'query_item_ids'.
"""
item_ids = utils.listify(item_ids)
logger.info('Computing detailed scores ...')
scores = self._compute_detailed_scores(item_ids, query_item_ids,
max_terms)
self._update_time_taken()
return scores
def query(self, item_ids, max_results=100):
"""Queries the given computed against the given item ids.
"""
item_ids = utils.listify(item_ids)
if not self.is_valid_query(item_ids):
return self.empty_results
logger.info('Computing the query vector ...')
self._make_query_vector()
logger.info('Computing log scores ...')
self._compute_scores()
logger.info('Get the top %s log scores ...', max_results)
self._order_indexes_by_scores(max_results)
return self.results
def query(self, item_ids, max_results=100):
"""Queries the given computed against the given item ids.
"""
item_ids = utils.listify(item_ids)
if not self.is_valid_query(item_ids):
return self.empty_results
logger.info('Computing the query vector ...')
self._make_query_vector()
logger.info('Computing log scores ...')
self._compute_scores()
logger.info('Get the top %s log scores ...', max_results)
self._order_indexes_by_scores(max_results)
return self.results
def __init__(self,
wrapper,
data,
loss_func,
lr=1e-2,
cbs=None,
cb_funcs=None):
self.wrapper, self.data, self.loss_func, self.lr = wrapper, data, loss_func, lr
self.print, self.logger, self.opt = False, print, None
self.model = self.wrapper.model
self.cbs = []
self.add_cb(TrainEvalCallback())
self.add_cbs(cbs)
self.add_cbs(cbf() for cbf in listify(cb_funcs))
def db_lookup_cluster_by_server(server, lookup_type=LOOKUP_TYPE_ANY):
cluster_collection = get_mongoctl_cluster_db_collection()
lookup_type = listify(lookup_type)
type_query = []
for t in lookup_type:
prop_query = {"%s.server.$id" % t: server.id}
type_query.append(prop_query)
query = {"$or": type_query}
cluster_doc = cluster_collection.find_one(query)
if cluster_doc is not None:
return new_cluster(cluster_doc)
else:
return None
def db_lookup_cluster_by_server(server, lookup_type=LOOKUP_TYPE_ANY):
cluster_collection = get_mongoctl_cluster_db_collection()
lookup_type = listify(lookup_type)
type_query = []
for t in lookup_type:
prop_query = {"%s.server.$id" % t: server.id}
type_query.append(prop_query)
query = {"$or": type_query}
cluster_doc = cluster_collection.find_one(query)
if cluster_doc is not None:
return new_cluster(cluster_doc)
else:
return None
def update_task(self,
task,
properties=None,
event_name=None,
event_type=EventType.INFO,
message=None,
details=None,
error_code=None,
**update_kwargs):
"""
Updates the specified properties of the specified MBSTask object
"""
task_doc = task.to_document()
q = {"_id": task.id}
u = {}
# log the event as needed
if event_name or message:
log_entry = task.log_event(name=event_name,
event_type=event_type,
message=message,
details=details,
error_code=error_code)
# push if "logs" property is not included
if not (properties and "logs" in properties):
u["$push"] = {"logs": log_entry.to_document()}
# construct $set operator
if properties:
properties = listify(properties)
u["$set"] = {}
for prop in properties:
u["$set"][prop] = task_doc.get(prop)
if not u or ("$set" not in u and "$push" not in u):
import mbs
import notification.handler
mbs.get_mbs().notifications.send_event_notification(
"BAD UPDATE",
"BAD UPDATE for task %s: %s , %s" %
(task.id, u, traceback.format_exc()),
priority=notification.handler.NotificationPriority.CRITICAL)
raise Exception("BAD UPDATE!!!!!")
self.update(spec=q, document=u, **update_kwargs)
def config_lookup_cluster_by_server(server, lookup_type=LOOKUP_TYPE_ANY):
clusters = get_configured_clusters()
lookup_type = listify(lookup_type)
for t in lookup_type:
result = None
if t == LOOKUP_TYPE_MEMBER:
result = filter(lambda c: c.has_member_server(server),
clusters.values())
elif t == LOOKUP_TYPE_CONFIG_SVR:
result = filter(lambda c: cluster_has_config_server(c, server),
clusters.values())
elif t == LOOKUP_TYPE_SHARDS:
result = filter(lambda c: cluster_has_shard(c, server),
clusters.values())
if result:
return result[0]
def __init__(self,
model,
data,
loss_func,
opt_func=sgd_opt,
lr=1e-2,
splitter=param_getter,
cbs=None,
cb_funcs=None,
path=None):
self.model, self.data, self.loss_func, self.opt_func, self.lr, self.splitter = model, data, loss_func, opt_func, lr, splitter
self.in_train, self.logger, self.opt = False, print, None
self.path = Path(ifNone(path, self.data.path))
self.cbs = []
self.add_cb(TrainEvalCallback())
self.add_cbs(cbs)
self.add_cbs(cbf() for cbf in listify(cb_funcs))
def _urlize_paths(self, paths):
"""
Return a list of URLs for the given paths.
"""
paths = listify(paths)
static_folder = realpath(self.static_folder)
url_paths = []
for path in paths:
# Ensure this is a full, real path (ie. not relative)
path = realpath(path)
# Strip off path to the static folder from the beginning
if path.startswith(static_folder):
path = path[len(static_folder):]
# Prefix with the static folder URL path
path = self.static_url_path + path
url_paths.append(path)
return url_paths
def upload_backup(self, backup, file_name, target, destination_path=None):
targets = listify(target)
workspace = self.get_task_workspace_dir(backup)
file_path = os.path.join(workspace, file_name)
metadata = {
"Content-Type": "application/x-compressed"
}
uploaders = multi_target_upload_file(targets, file_path, destination_path=destination_path, metadata=metadata)
errored_uploaders = filter(lambda uploader: uploader.error is not None,
uploaders)
if errored_uploaders:
raise errored_uploaders[0].error
target_references = map(lambda uploader: uploader.target_reference, uploaders)
if isinstance(target, list):
return target_references
else:
return target_references[0]
def update_task(self, task, properties=None, event_name=None,
event_type=EventType.INFO, message=None, details=None,
error_code=None,
**update_kwargs):
"""
Updates the specified properties of the specified MBSTask object
"""
task_doc = task.to_document()
q = {
"_id": task.id
}
u = {}
# log the event as needed
if event_name or message:
log_entry = task.log_event(name=event_name, event_type=event_type,message=message, details=details,
error_code=error_code)
# push if "logs" property is not included
if not (properties and "logs" in properties):
u["$push"] = {"logs": log_entry.to_document()}
# construct $set operator
if properties:
properties = listify(properties)
u["$set"] = {}
for prop in properties:
u["$set"][prop] = task_doc.get(prop)
if not u or ("$set" not in u and "$push" not in u):
import mbs
import notification.handler
mbs.get_mbs().notifications.send_event_notification(
"BAD UPDATE", "BAD UPDATE for task %s: %s , %s" % (task.id, u, traceback.format_exc()),
priority=notification.handler.NotificationPriority.CRITICAL)
raise Exception("BAD UPDATE!!!!!")
self.update(spec=q, document=u, **update_kwargs)
def upload_backup(self, backup, file_name, target, destination_path=None):
targets = listify(target)
workspace = self.get_task_workspace_dir(backup)
file_path = os.path.join(workspace, file_name)
metadata = {"Content-Type": "application/x-compressed"}
uploaders = multi_target_upload_file(targets,
file_path,
destination_path=destination_path,
metadata=metadata)
errored_uploaders = filter(lambda uploader: uploader.error is not None,
uploaders)
if errored_uploaders:
raise errored_uploaders[0].error
target_references = map(lambda uploader: uploader.target_reference,
uploaders)
if isinstance(target, list):
return target_references
else:
return target_references[0]
def hlfet(dag, no_nodes):
scheduling = {}
# initialize empty task list for each processor
for node in range(0, no_nodes):
scheduling[node] = []
L = listify(dag)
L.sort(key=lambda task: task.static_level)
while len(L) > 0:
task = L.pop()
min_node = 0
min_val = processor_finish(scheduling[0])
for node in scheduling:
t = processor_finish(scheduling[node])
m = max(t, task.est)
if m < min_val:
min_val = m
min_node = node
scheduling[min_node].append(task)
task.start = min_val
return scheduling
def send_notification(self, subject, message, recipient=None):
try:
logger.info("Sending notification email...")
msg = MIMEText(message.encode('utf-8'), 'plain', 'UTF-8')
to_address = listify(recipient or self._to_address)
msg['From'] = self.from_address
msg['To'] = ",".join(to_address)
if subject:
msg['Subject'] = subject
smtp = smtplib.SMTP(self.smtp_host)
if (self.smtp_username is not None
or self.smtp_password is not None):
smtp.login(self.smtp_username, self.smtp_password)
smtp.sendmail(self.from_address, to_address, msg.as_string())
smtp.quit()
logger.info("Email sent successfully!")
except Exception, e:
logger.error("Error while sending email:\n%s" %
traceback.format_exc())
def test(MAXIMUM_GENERATORS=255):
""" Run the inline tests for the halogen.compile module """
import tempfile
from contextlib import ExitStack
from pprint import pprint
if __package__ is None or __package__ == '':
import api # type: ignore
from utils import terminal_width
else:
from . import api
from .utils import terminal_width
directory = Directory(pth="/Users/fish/Dropbox/halogen/tests/generators")
destination = Directory(pth=os.path.join(tempfile.gettempdir(), "yodogg"))
zip_destination = os.path.realpath("/tmp")
with TemporaryDirectory(prefix='yo-dogg-') as td:
if not td.exists:
print("X> TemporaryDirectory DOES NOT EXIST:")
print(f"X> {td}")
# We use a contextlib.ExitStack instance to separate out the construction
# of the halogen.compile.Generators instance (q.v. immediately below) and
# the call to __enter__ (q.v. right after that) so as to trap any and all
# exceptions that may be thrown individually in either the constructor call
# -- e.g. Generators.__init__ -- or Generators.__enter__ …
stack = ExitStack()
gens = Generators(CONF, directory=directory,
destination=td,
intermediate=td.subdirectory(".intermediate"),
maximum=MAXIMUM_GENERATORS,
verbose=DEFAULT_VERBOSITY,
use_cdb=True)
# Preserve compilation artifacts:
# td.do_not_destroy()
try:
# Calls Generators.__enter__(self=gens):
stack.enter_context(gens)
except CompilerError as exc:
print_exception(exc)
# gens.precompile() and gens.compile()
except CompileDatabaseError as exc:
print_exception(exc)
# gens.precompile() and gens.compile()
except LinkerError as exc:
print_exception(exc)
# if gens.compiled and gens.do_static:
# gens.arch()
except ArchiverError as exc:
print_exception(exc)
# if gens.compiled and gens.do_shared:
# gens.link()
# if gens.linked and gens.do_preload:
# gens.preload_all()
except GeneratorLoaderError as exc:
print_exception(exc)
except GenerationError as exc:
print_exception(exc)
else:
with stack: # Exiting this scope calls Generators.__exit__(self=gens):
precompiled = gens.precompiled and "YES" or "no"
compiled = gens.compiled and "YES" or "no"
postcompiled = gens.postcompiled and "YES" or "no"
linked = gens.linked and "YES" or "no"
archived = gens.archived and "YES" or "no"
preloaded = gens.preloaded and "YES" or "no"
print("")
print(f"IS IT PRECOMPILED? -- {precompiled}")
print(f"IS IT COMPILED? -- {compiled}")
print(f"IS IT POSTCOMPILED? -- {postcompiled}")
print(f"IS IT LINKED? -- {linked}")
print(f"IS IT ARCHIVED? -- {archived}")
print(f"IS IT PRELOADED? -- {preloaded}")
print("")
print(f"LIBRARY: {gens.library}")
if gens.linked and os.path.exists(gens.library):
print("LIBRARY FILE EXISTS")
print(f"ARCHIVE: {gens.archive}")
if gens.archived and os.path.exists(gens.archive):
print("ARCHIVE FILE EXISTS")
print(f"REGISTERED GENERATORS: {api.registered_generators()}")
# loaded_generators = gens.loaded_generators()
if DEFAULT_VERBOSITY:
if gens.loaded_count > 0:
print(f"... SUCCESSFULLY LOADED GENERATORS FROM LIBRARY {gens.library}")
print(f"... THERE ARE {gens.loaded_count} GENERATORS LOADED FROM THAT LIBRARY, DOGG")
else:
print(f"... NO GENERATORS COULD BE LOADED FROM LIBRARY {gens.library}")
# Run generators:
generated = gens.run(emit='expanded')
print('')
pprint(generated, indent=4,
width=terminal_width)
print('')
# Copy the library and archive files to $TMP/yodogg:
if destination.exists:
if DEFAULT_VERBOSITY:
print(f"Removing destination: {destination} …")
rm_rf(destination)
if DEFAULT_VERBOSITY:
print(f"Copying from {td} to {destination} …")
td.copy_all(destination)
with TemporaryName(suffix="zip", parent=zip_destination) as tz:
if DEFAULT_VERBOSITY:
print(f"Zip-archiving destination contents to zipfile: {tz} …")
destination.zip_archive(tz)
if gens.intermediate.exists:
if CDBJsonFile.in_directory(gens.intermediate):
if DEFAULT_VERBOSITY:
print("")
print(f"Found compilation DB file “{CDBJsonFile.filename}” in intermediate: {gens.intermediate}:")
with CDBJsonFile(directory=gens.intermediate) as cdb:
pprint(cdb.entries, indent=4,
width=terminal_width)
if DEFAULT_VERBOSITY:
print("")
print(f"Listing files at intermediate: {gens.intermediate} …")
intermediate_list = OCDList(gens.intermediate.subpath(listentry) \
for listentry in gens.intermediate.ls())
pprint(listify(*intermediate_list), indent=4,
width=terminal_width)
else:
print("X> Intermediate directory DOES NOT EXIST")
print(f"X> {gens.intermediate}")
if destination.exists:
if DEFAULT_VERBOSITY:
print("")
print(f"Listing files at destination: {destination} …")
destination_list = OCDList(destination.subpath(listentry) \
for listentry in destination.ls())
pprint(listify(*destination_list), indent=4,
width=terminal_width)
if DEFAULT_VERBOSITY:
print(f"Removing destination: {destination} …")
rm_rf(destination)
else:
print("X> Destination directory DOES NOT EXIST")
print(f"X> {destination}")
def set_files(self, files):
self.__files = utils.listify(files)
self.configure()
def compose(x, funcs, *args, order_key='_order', **kwargs):
key = lambda o: getattr(o, order_key, 0)
for f in sorted(listify(funcs), key=key): x = f(x, **kwargs)
return x
def obj(self, items, idx, procs):
isint = isinstance(idx, int) or (isinstance(idx,torch.LongTensor) and not idx.ndim)
item = items[idx]
for proc in reversed(listify(procs)):
item = proc.deproc1(item) if isint else proc.deprocess(item)
return item
def remove_cbs(self, cbs):
for cb in listify(cbs):
self.cbs.remove(cb)
def add_cbs(self, cbs):
for cb in listify(cbs):
self.add_cb(cb)
def __init__(self, items): self.items = listify(items) def __getitem__(self, idx):
def process_sz(sz):
sz = listify(sz)
return tuple(sz if len(sz) == 2 else [sz[0], sz[0]])
def __init__(self, metrics, in_train):
self.metrics, self.in_train = listify(metrics), in_train
def add(self, key, lib_obj, lib_deps=list()):
assert key not in self._libs
lib_deps = {'lib_obj': lib_obj, 'lib_deps': listify(lib_deps)}
self._libs[key] = lib_deps
def __init__(self, param, sched_funcs):
self.param = param
self.sf = listify(sched_funcs)
def main():
# Arseparse
parser = argparse.ArgumentParser(description='Manage some builds')
parser.add_argument('action', metavar='action', nargs='?', action="store")
parser.add_argument('environments', metavar='environment', nargs='*', action="store",default=[])
parser.add_argument('-c', dest='config_path', action='store',help='Specify a config file')
args = parser.parse_args()
# Dig out the config path
if args.config_path is None:
# TODO: Search environ too
args.config_path = "motley.cfg"
# Read the raw config
config = ConfigParser()
config.read(args.config_path)
base_path = config.get('motley','directory')
# Build the list of jobs to run
jobs = []
for section in config.sections():
if section.startswith('environment:'):
# If no arguments were specified we want all environments
if not args.environments or section[12:] in args.environments:
options = expand_section(config,section)
options['environment'] = section[12:]
# Inject the path
if 'path' not in options:
options['path'] = os.path.join(base_path,options['environment'])
# Iterate through the actions looking for
for action_name in listify(options.get('actions','')):
# Expand each action (this duplicates a lot unfortunately)
action = expand_section(config,'action:%s' % action_name)
key = action.get('name',action_name)
# Add a job when we find the first action whose name matches
if key == args.action:
job = {'options':options,
'actions':[]}
for sn in expand_dependencies(config,action_name):
command = expand_section(config,'action:%s' % sn)['command']
job['actions'].append(command)
jobs.append(job)
break
batch = Batch(jobs)
batch.start()
for result in batch:
if result is None:
print "Something bad has happened"
else:
code,stdout,stderr = result
if int(code) != 0:
print "============================================"
print "A job failed"
print stdout
print stderr
else:
pass
def setify(o): return o if isinstance(o,set) else set(listify(o)) def _get_files(p, fs, extensions=None):
