def calc_stats(self, just_save=False):
if just_save:
logging.trace('just_save')
save(
os.path.join(self.save_dir, 'pat_stats%s.npy') %
self.file_postfix, self.stats)
return self.stats
t0 = time.time()
divs = self.divide_parts(self.hps.train_its, self.n_threads)
out_que = queue.Queue(self.n_threads)
cnt_mb_que = queue.Queue()
# initialize threads
for t in range(self.n_threads):
self.threads.append(
Thread(target=self.calc_patch_stats,
args=(t, self.mb_queue, divs[t], out_que, cnt_mb_que)))
self.threads[t].start()
print('')
for t in range(self.n_threads):
self.threads[t].join()
self.weighted_stats(out_que)
self.calc_scalar_stats()
save(
os.path.join(self.save_dir, 'pat_stats%s.npy') % self.file_postfix,
self.stats)
logging.trace('calc. stats: time = %3.0f s ' % (time.time() - t0))
return self.stats
def process(source_assets,
staged_assets,
output_path,
droid_key,
py_exe=None,
py_version=(2, 7)):
logging.info("Processing client python...")
def iter_python_paks():
for client_path, source_asset in source_assets.items():
if "python" in source_asset.categories and client_path.suffix.lower(
) == ".pak":
yield client_path, source_asset
# Check for any Python paks -- if they exist, bail.
if any(iter_python_paks()):
logging.warning(
"Using prebuilt Python packages -- this is not recommended!")
for client_path, source_asset in iter_python_paks():
logging.trace(f"Prebuilt Python: '{client_path.name}'")
staged_assets[client_path].file_name = client_path
return
# The compyler was written assuming a minimum of Python 2.3
if py_version[0] == 2 and py_version[1] < 3:
logging.critical(
f"Python {'.'.join(py_version)} is not supported by the compyler.")
logging.critical("No Python.pak will be generated!")
return
module_code = _compyle_all(source_assets, staged_assets, py_exe)
if module_code:
_package(source_assets, staged_assets, module_code, output_path,
droid_key)
def _onCookieAdded(self, cookie):
cookie_str = str(cookie.toRawForm(QNetworkCookie.NameAndValueOnly), encoding='utf-8')
cookie_name = str(cookie.name(), encoding='utf-8')
cookie_val = str(cookie.value(), encoding='utf-8')
if (cookie_name == self.authn_cookie_name) and (cookie.domain() == self.config.get("host")):
logging.trace("%s cookie added:\n\n%s\n\n" % (self.authn_cookie_name, cookie_str))
self.credential["cookie"] = "%s=%s" % (self.authn_cookie_name, cookie_val)
host = self.auth_url.host()
cred_entry = dict()
cred_entry[host] = self.credential
if self.credential_file:
creds = read_credential(self.credential_file, create_default=True)
creds.update(cred_entry)
write_credential(self.credential_file, creds)
self.token = cookie_val
self._session.cookies.set(self.authn_cookie_name, cookie_val, domain=host, path='/')
if self.cookie_jar is not None:
self.cookie_jar.set_cookie(
create_cookie(self.authn_cookie_name,
cookie_val,
domain=host,
path='/',
expires=0,
discard=False,
secure=True))
for path in self.config.get("cookie_jars", DEFAULT_CONFIG["cookie_jars"]):
path_dir = os.path.dirname(path)
if os.path.isdir(path_dir):
logging.debug("Saving cookie jar to: %s" % path)
self.cookie_jar.save(path, ignore_discard=True, ignore_expires=True)
else:
logging.debug("Cookie jar save path [%s] does not exist." % path_dir)
def load_asset_database(mfs_path, list_path, db_type):
logging.info("Reading asset database...")
db_cls = manifest.ManifestDB.get(db_type)
if db_cls is None:
raise AssetError(f"Invalid asset db type '{db_type}'")
manifests, lists = db_cls.load_db(mfs_path, list_path)
# Merge assets into case insensitive dict and verify hashes. Use a custom type so we don't
# compare the file flags, which can legally differ (eg sound decompression)
asset = namedtuple(
"AssetEntry",
("file_hash", "download_hash", "file_size", "download_size"))
assets, conflicts = {}, 0
for mfs_name, mfs_entries in manifests.items():
for mfs_entry in mfs_entries:
mfs_asset = asset(mfs_entry.file_hash, mfs_entry.download_hash,
mfs_entry.file_size, mfs_entry.download_size)
if assets.setdefault(mfs_entry.file_name, mfs_asset) != mfs_asset:
logging.warn(f"CONFLICT: '{mfs_entry.file_name}'")
conflicts += conflicts
assets[mfs_entry.file_name] = None
if conflicts:
logging.warn(f"Discarded {conflicts} conflicting asset entries!")
logging.trace(
f"Loaded {len(assets)} asset entries from {len(manifests)} manifests, with {len(lists)} legacy auth-lists."
)
db = namedtuple("AssetDb", ("assets", "manifests", "lists"))
return db(assets, manifests, lists)
def _onCookieRemoved(self, cookie):
cookie_str = str(cookie.toRawForm(QNetworkCookie.NameAndValueOnly), encoding='utf-8')
cookie_name = str(cookie.name(), encoding='utf-8')
if cookie_name == self.authn_cookie_name and cookie.domain() == self.url().host():
logging.trace("%s cookie removed:\n\n%s\n\n" % (self.authn_cookie_name, cookie_str))
if self.cookie_jar:
self.cookie_jar.clear(cookie_name, path=cookie.path(), domain=cookie.domain())
def check_python_version(py_exe, py_version=(2, 7)):
logging.debug(f"Checking Python interpreter version: {py_exe}")
if not py_exe or not py_exe.is_file():
logging.debug("Non-file input")
return False
args = (str(py_exe), "-V")
result = subprocess.run(args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
encoding="ascii")
if result.returncode == 0:
logging.trace(f"{py_exe}: {result.stdout}")
expected_version = f"Python {py_version[0]}.{py_version[1]}"
result_version = result.stdout.strip()
if not result_version.startswith(expected_version):
logging.error(
f"Python interpreter '{py_exe}' is wrong version--expected '{expected_version}' got '{result_version}'"
)
return False
return True
else:
logging.debug("Nonzero returncode")
return False
def read_param_string(self, param_str):
"""
Parse the dict with the parametrization strings and evaluate for
the bin energies needed.
"""
evals = get_bin_centers(self.ebins)
n_e = len(self.ebins) - 1
n_cz = len(self.czbins) - 1
parametrization = {}
for flavour in param_str:
parametrization[flavour] = {}
for int_type in param_str[flavour]: #['cc', 'nc']
logging.debug('Parsing function strings for %s %s' %
(flavour, int_type))
parametrization[flavour][int_type] = {}
for axis in param_str[flavour][
int_type]: #['energy', 'coszen']
parameters = {}
for par, funcstring in param_str[flavour][int_type][
axis].items():
# this should contain a lambda function:
function = eval(funcstring)
logging.trace(' function: %s' % funcstring)
# evaluate the function at the given energies
vals = function(evals)
# repeat for all cos(zen) bins
parameters[par] = np.repeat(vals, n_cz).reshape(
(n_e, n_cz))
parametrization[flavour][int_type][axis] = copy(parameters)
return parametrization
def read_param_string(self, param_str):
"""
Parse the dict with the parametrization strings and evaluate for
the bin energies needed.
"""
evals = get_bin_centers(self.ebins)
n_e = len(self.ebins)-1
n_cz = len(self.czbins)-1
parametrization = {}
for flavour in param_str:
parametrization[flavour] = {}
for int_type in param_str[flavour]: #['cc', 'nc']
logging.debug('Parsing function strings for %s %s'
%(flavour, int_type))
parametrization[flavour][int_type] = {}
for axis in param_str[flavour][int_type]: #['energy', 'coszen']
parameters = {}
for par, funcstring in param_str[flavour][int_type][axis].items():
# this should contain a lambda function:
function = eval(funcstring)
logging.trace(' function: %s'%funcstring)
# evaluate the function at the given energies
vals = function(evals)
# repeat for all cos(zen) bins
parameters[par] = np.repeat(vals,n_cz).reshape((n_e,n_cz))
parametrization[flavour][int_type][axis] = copy(parameters)
return parametrization
def collect_root_files(self):
root_files = copy.copy(self.root_files)
if len(self.scandirs) > 0:
for scandir in self.scandirs[::-1]:
if not scandir.endswith('/'): scandir += '/'
root_files_this_scandir = glob.glob(scandir + self.globpat +
'.root')
n_found = len(root_files_this_scandir)
logging.debug('Found {0} root files in {1}'.format(
n_found, scandir))
root_files.extend(root_files_this_scandir)
if self.read_one_scandir and len(root_files_this_scandir) > 0:
logging.warning(
'Using only root files found in {0} (ignoring others for globpat {1})'
.format(scandir, self.globpat))
self.scandirs = [scandir]
break
logging.debug('Found {0} root files in {1}'.format(
len(root_files), ', '.join(self.scandirs)))
if len(root_files) == 0:
raise RuntimeError(
'Attemped to retrieve scan for x:{0} y:{1}, '
'but no .root files were found. Passed list of dirs to look in:\n'
.format(self.x_variable, self.y_variable) +
'\n'.join(self.scandirs))
logging.trace('List of root files:\n' + '\n'.join(root_files))
return root_files
def _linux_pkg_checks() -> None:
"""Check that all the packages required by SIERRA are installed on whatever
flavor of Linux SIERRA is running on.
"""
import distro
dist = distro.id()
os_info = distro.os_release_info()
if os_info['id_like'] in ['debian', 'ubuntu']:
import apt
cache = apt.Cache()
missing = []
for pkg in kRequiredDebPackages:
logging.trace("Checking for .deb package '%s'", pkg)
if pkg not in cache or not cache[pkg].is_installed:
missing.append(pkg)
if missing:
raise RuntimeError(
(f"Required .deb packages {missing} missing on "
f"Linux distribution '{dist}'. Install all "
"required packages before running SIERRA! "
"(Did you read the \"Getting Started\" docs?)"))
else:
logging.warning(
("Unknown Linux distro '%s' detected: skipping package "
"check"), dist)
logging.warning(
("If SIERRA crashes it might be because you don't have "
"all the required packages installed"))
def drop_pdfindices(card_file, category_pats=None):
if differentials.core.is_testmode():
return
with open(card_file, 'r') as card_fp:
card = card_fp.read()
if category_pats is None:
category_pats = ['recoPt_600p0_10000p0']
lines = []
for line in card.split('\n'):
for category_pat in category_pats:
if re.match(r'pdfindex_.*{0}'.format(category_pat), line):
logging.debug(
'Dropping following line from {0} (matched to {2}):\n{1}'.
format(card_file, line, category_pat))
break
else:
lines.append(line)
new_card = '\n'.join(lines)
logging.trace('Datacard after removing lines:\n{0}'.format(new_card))
logging.info(
'Writing new card after deleting lines to {0}'.format(card_file))
if not core.is_testmode():
with open(card_file, 'w') as card_fp:
card_fp.write(new_card)
def get_next_address(list_obj, item):
if list_obj == None: return None
if item == None: return None
if type(list_obj) is dict:
list_obj = list(list_obj.keys())
list_obj.sort()
if item not in list_obj:
logging.trace('Item not in list. Estimating position.')
for x in range(len(list_obj)-1, -1, -1):
address = list_obj[x]
if item > address:
item = address
break
if item not in list_obj:
logging.trace('Item not in list. Returning None.')
return None
# Find index of the address and get next one up.
if (list_obj.index(item)) < (len(list_obj) - 1):
next_address = list_obj[list_obj.index(item) + 1]
else:
next_address = None
return next_address
def __init__(self,
identifier,
execparams,
propertydefs=(),
loggerName=None):
"""propertydefs is a iterable of tuples that contain
(propid, propname, type, mode, defaultvalue, units, action, kinds)
defaultvalue should be None if there is no default, otherwise:
simple - a single str, int, float, bool
simple sequence - an iterable of str, int, float, or bool
struct - a dictionary there the key is the "id" and the value is a tuple
of (propname, type, mode)
struct sequence - a iterable of dictionaries following the struct format
"""
self.propertySetAccess = threading.Lock()
self._id = identifier
self._started = False
if loggerName == None:
self._log = logging.getLogger(self._id)
else:
self._log = logging.getLogger(loggerName)
self._name = execparams.get("NAME_BINDING", "")
# The base resource class manages properties ...
self._props = PropertyStorage(self, propertydefs, execparams)
self._props.initialize()
# ... and also manages ports
self.__loadPorts()
logging.trace("Initial property storage %s", self._props)
def handle_dump(self, interfaces, neighbours, xroutes, routes):
# neighbours = {neigh_prefix: (neighbour, {dst_prefix: route})}
n = dict(((n.address, n.ifindex), (n, {})) for n in neighbours)
unidentified = set(n)
self.neighbours = neighbours = {}
a = len(self.network)
for route in routes:
assert route.flags & 1, route # installed
if route.prefix.startswith("\0\0\0\0\0\0\0\0\0\0\xff\xff"):
continue
assert route.neigh_address == route.nexthop, route
address = route.neigh_address, route.ifindex
neigh_routes = n[address]
ip = utils.binFromRawIp(route.prefix)
if ip[:a] == self.network:
prefix = ip[a : route.plen]
if prefix and not route.refmetric:
neighbours[prefix] = neigh_routes
unidentified.remove(address)
else:
prefix = None
neigh_routes[1][prefix] = route
self.locked.clear()
if unidentified:
routes = {}
for address in unidentified:
neigh, r = n[address]
if not neigh.cost_multiplier:
self.locked.add(address)
routes.update(r)
if routes:
neighbours[None] = None, routes
logging.trace("Routes via unidentified neighbours. %r", neighbours)
self.interfaces = dict((i.index, name) for i, name in interfaces)
self.handler.babel_dump()
def read_manifest(cls, path):
logging.debug(f"Reading manifest: {path}")
with path.open(mode="r") as f:
for i, line in enumerate(f):
line = line.strip()
if not line:
continue
data = line.split(',')
if len(data) != 7:
logging.error(f"Malformed manifest '{path}' line #{i}")
continue
try:
entry = manifest.ManifestEntry()
entry.file_name = Path(PureWindowsPath(data[0]))
entry.download_name = Path(PureWindowsPath(data[1]))
entry.file_hash = data[2]
entry.download_hash = data[3]
entry.file_size = int(data[4])
entry.download_size = int(data[5])
entry.flags = int(data[6])
except Exception as e:
logging.error(f"Malformed manifest '{path}' line #{i}")
else:
logging.trace(manifest.pformat(entry))
yield entry
def _osx_pkg_checks() -> None:
"""Check that all the packages required by SIERRA are installed on whatever
version of OSX SIERRA is running on.
"""
missing = []
for pkg in kRequiredOSXPackages:
logging.trace("Checking for homebrew package '%s'", pkg)
p1 = subprocess.Popen('brew list | grep {pkg}',
shell=True,
stderr=subprocess.DEVNULL,
stdout=subprocess.DEVNULL)
p2 = subprocess.Popen('brew list --cask | grep {pkg}',
shell=True,
stderr=subprocess.DEVNULL,
stdout=subprocess.DEVNULL)
p1.wait()
p2.wait()
if p1.returncode != 0 and p2.returncode != 0:
missing.append(pkg)
if missing:
raise RuntimeError((f"Required brew package {missing} missing on OSX. "
"Install all required packages before running"
"SIERRA! (Did you read the \"Getting Started\" "
"docs?)"))
def read_pht(matched_files_tuple, star_range_0, apertureidx: int, fake_references=False):
fileidx = matched_files_tuple[0]
file_entry = matched_files_tuple[1]
# open the file for reading
with open(file_entry, mode='rb') as file: # b is important -> binary
fileContent = file.read()
photheader, _, nrstars, stars, stardata = read_pht_file(file_entry, fileContent, only_apertureidx=int(apertureidx))
collect = np.full([len(init.star_list), 2],np.inf, dtype=float) # we don't use nrstars because this could be e.g. 1000, but with stars which have id's > 1000
fwhm = [photheader.fwhm_exp, photheader.fwhm_mean, photheader.fwhm_err]
jd = photheader.jd
# for every star
for staridx, starentry in enumerate(stardata):
ref_id_0 = stars[staridx].ref_id - 1 # if photometry then all is -1
if ref_id_0 is -2: # -1 - 1
if fake_references:
ref_id_0 = staridx
else:
continue
try:
if ref_id_0 > len(collect):
logging.trace(f"Star idx too big for collect shape: staridx: {staridx}, ref_id_0: {ref_id_0}, shape: {collect.shape}")
collect[ref_id_0] = [starentry.mag, starentry.err]
except:
logging.error(f"staridx: {staridx}, ref_id_0: {ref_id_0}, shape: {collect.shape}")
collected = collect[star_range_0]
return fileidx, jd, fwhm, nrstars, collected
def _onCookieAdded(self, cookie):
cookie_str = str(cookie.toRawForm(QNetworkCookie.NameAndValueOnly),
encoding='utf-8')
cookie_name = str(cookie.name(), encoding='utf-8')
cookie_val = str(cookie.value(), encoding='utf-8')
if (cookie_name
== self.authn_cookie_name) and (cookie.domain()
== self.config.get("host")):
logging.trace("%s cookie added:\n\n%s\n\n" %
(self.authn_cookie_name, cookie_str))
self.credential["cookie"] = "%s=%s" % (self.authn_cookie_name,
cookie_val)
host = self.auth_url.host()
cred_entry = dict()
cred_entry[host] = self.credential
if self.credential_file:
creds = read_credential(self.credential_file,
create_default=True)
creds.update(cred_entry)
write_credential(self.credential_file, creds)
self.token = cookie_val
self._session.cookies.set(self.authn_cookie_name,
cookie_val,
domain=host,
path='/')
self.authn_session_page.setUrl(
QUrl(self.auth_url.toString() + "/authn/session"))
def find_sdls(sdl_mgrs,
descriptor_name,
embedded_sdr=False,
optional=False):
dependencies = set()
descriptors = set()
for sdl_file, mgr in sdl_mgrs.items():
# Be sure to loop over all files in case someone moves a record to a new file
# from version to version. Please don't do that, though. It's mean :<
for descriptor in mgr.find_descriptors(descriptor_name):
dependencies.add(sdl_file)
descriptors.add(descriptor.name)
# We need to see if there are any embedded state descriptor variables...
sdrs = (i for i in descriptor.variables
if i.descriptor is not None
and i.descriptor not in descriptors)
for variable in sdrs:
more_dependencies, more_descriptors = find_sdls(
sdl_mgrs, variable.descriptor, True)
dependencies.update(more_dependencies)
descriptors.update(more_descriptors)
if descriptor_name not in descriptors:
if embedded_sdr:
raise AssetError(
f"Embedded SDL Descriptor '{descriptor_name}' is missing.")
elif not optional:
raise AssetError(
f"Top-level SDL '{descriptor_name}' is missing.")
else:
logging.trace(
f"Optional SDL Descriptor '{descriptor_name}' not found.")
return dependencies, descriptors
def execute(cmd,
capture_output=False,
ignore_testmode=False,
py_capture_output=False):
# Allow both lists and strings to be passed as the cmd
if not isinstance(cmd, basestring):
cmd = [l for l in cmd if not len(l.strip()) == 0]
cmd_str = '\n '.join(cmd)
cmd_exec = ' '.join(cmd)
else:
cmd_str = cmd
cmd_exec = cmd
logging.info('Executing the following command:\n{0}'.format(cmd_str))
logging.trace('Actual passed command: {0}'.format(cmd_exec))
if not (is_testmode()) and not (ignore_testmode):
if py_capture_output:
return subprocess.check_output(cmd_exec, shell=True)
elif capture_output:
with RedirectStdout() as redirected:
os.system(cmd_exec)
output = redirected.read()
return output
else:
os.system(cmd_exec)
def __init__(self, identifier, execparams, propertydefs=(), loggerName=None):
"""propertydefs is a iterable of tuples that contain
(propid, propname, type, mode, defaultvalue, units, action, kinds)
defaultvalue should be None if there is no default, otherwise:
simple - a single str, int, float, bool
simple sequence - an iterable of str, int, float, or bool
struct - a dictionary there the key is the "id" and the value is a tuple
of (propname, type, mode)
struct sequence - a iterable of dictionaries following the struct format
"""
self.propertySetAccess = threading.Lock()
self._id = identifier
self._started = False
if loggerName == None:
self._log = logging.getLogger(self._id)
else:
self._log = logging.getLogger(loggerName)
self._name = execparams.get("NAME_BINDING", "")
# The base resource class manages properties ...
self._props = PropertyStorage(self, propertydefs, execparams)
self._props.initialize()
# ... and also manages ports
self.__loadPorts()
logging.trace("Initial property storage %s", self._props)
def raspberry_pi_infos():
"""
Returns infos about current raspberry pi board
Note:
https://elinux.org/RPi_HardwareHistory#Board_Revision_History
Returns:
dict: raspberry pi board infos::
{
date (string): release date
model (string): raspberry pi model
pcbrevision (string): PCB revision
ethernet (bool): True if ethernet is natively available on board
wireless (bool): True if wifi is natively available on board,
audio (bool): True if audio is natively available on board
gpiopins (int): number of pins available on board
memory (string): memory amount
notes (string): notes on board
revision (string): raspberry pi revision
}
"""
if not platform.machine().startswith('arm'):
raise Exception('Not arm platform')
cmd = u'/usr/bin/awk \'/^Revision/ {sub("^1000", "", $3); print $3}\' /proc/cpuinfo'
p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
revision = p.communicate()[0].decode('utf-8').replace(u'\n', u'')
logging.trace('Raspberrypi revision=%s' % revision)
infos = RASPBERRY_PI_REVISIONS[revision] if revision and revision in RASPBERRY_PI_REVISIONS else RASPBERRY_PI_REVISIONS[u'unknown']
infos[u'revision'] = revision
return infos
def compare_memory_objects(self, test_set, pattern_memory_obj,
function_memory_obj):
if function_memory_obj == None:
return False
pattern_keys = list(pattern_memory_obj.keys())
for pattern_key in pattern_keys:
if pattern_key.startswith("group"):
is_group_present = self.check_for_output_memory_block(
test_set, function_memory_obj,
pattern_memory_obj[pattern_key])
if is_group_present == False:
return False
else:
memory_address = int(pattern_key, 16)
if memory_address not in function_memory_obj:
logging.trace('Memory address ' + pattern_key +
' not present in function execution output.')
return False
if (pattern_memory_obj[pattern_key] !=
function_memory_obj[memory_address]):
logging.trace(
'Value at memory address ' + pattern_key +
' doesn\'t match in function execution output.')
return False
return True
def __init__(self, identifier, execparams, propertydefs=(), loggerName=None):
"""propertydefs is a iterable of tuples that contain
(propid, propname, type, mode, defaultvalue, units, action, kinds)
defaultvalue should be None if there is no default, otherwise:
simple - a single str, int, float, bool
simple sequence - an iterable of str, int, float, or bool
struct - a dictionary there the key is the "id" and the value is a tuple
of (propname, type, mode)
struct sequence - a iterable of dictionaries following the struct format
"""
#print "resource __init__:"
#print "id:" + str(identifier)
#print "loggerName:" + str(loggerName)
#print "execparams:" + str(execparams)
self.propertySetAccess = threading.Lock()
self._id = identifier
self._started = False
self._domMgr = None
self._ecm = None
##
## logging context for the resource
##
self.logLevel = logging.INFO
self._logLevel = CF.LogLevels.INFO
self.logConfig = ""
self.loggingMacros = ossie.logger.GetDefaultMacros()
ossie.logger.ResolveHostInfo( self.loggingMacros )
self.loggingCtx = None
self.loggingURL=None
if loggerName == None:
self._logid = execparams.get("NAME_BINDING", self._id )
self._logid = self._logid.rsplit("_", 1)[0]
else:
self._logid = loggerName
self._logid = self._logid.replace(":","_")
self._log = logging.getLogger(self._logid)
self._log.setLevel(self.logLevel)
self.logListenerCallback=None
self._name = execparams.get("NAME_BINDING", "")
# The base resource class manages properties ...
self._props = PropertyStorage(self, propertydefs, execparams)
self._props.initialize()
# property change listener registry and monitoring thread
self._propChangeRegistry = {}
self._propChangeThread = _PropertyChangeThread(self)
self._propMonitors = {}
# ... and also manages ports
self.__loadPorts()
logging.trace("Initial property storage %s", self._props)
self.__initialized = False
self.__propertiesInitialized = False
def log(self):
if logging.getLogger().isEnabledFor(5):
logging.trace("Cache:")
for prefix, address, _try in self._db.execute(
"SELECT peer.*, try FROM peer, volatile.stat"
" WHERE prefix=peer ORDER BY prefix"):
logging.trace("- %s: %s%s", prefix, address,
' (blacklisted)' if _try else '')
def log(self):
if logging.getLogger().isEnabledFor(5):
logging.trace("Cache:")
for prefix, address, _try in self._db.execute(
"SELECT peer.*, try FROM peer, volatile.stat"
" WHERE prefix=peer ORDER BY prefix"):
logging.trace("- %s: %s%s", prefix, address,
' (blacklisted)' if _try else '')
def run(self) -> bool:
'''
Make the input file standalone.
:return: True if the execution is a success, otherwise False.
'''
self.__init()
content = self._analyze_document()
logging.trace(_T("File content: %s") % (content))
return self._generate_flat_document(content)
def write_list(cls, path, name, entries):
out_path = path.joinpath(name).with_suffix(".list")
logging.debug(f"Writing secure list: {out_path}")
with out_path.open("w") as f:
for i in entries:
fn = PureWindowsPath(i.file_name)
ln = f"{fn},{i.file_size}"
logging.trace(ln)
f.write(f"{ln}\n")
def storage_sanity_checks(module) -> None:
logging.trace("Verifying selected --storage-medium plugin interface")
functions = ['df_read', 'df_write']
in_module = inspect.getmembers(module, inspect.isfunction)
for f in functions:
assert (any([f in name for name, _ in in_module])),\
f"Storage medium plugin does not define {f}"
def write(self, buffer):
logging.trace("send %s%r", self.__class__.__name__, (self.id,) + self.args)
offset = len(buffer)
buffer += "\0" * header.size
r = self.request
if isinstance(r, Struct):
r.encode(buffer, self.args)
else:
r.encode(buffer, *self.args)
buffer.pack_into(header, offset, self.id, len(buffer) - header.size - offset)
def write_manifest(cls, path, name, entries):
out_path = path.joinpath(name).with_suffix(".mfs")
logging.debug(f"Writing manifest: {out_path}")
with out_path.open("w") as f:
for i in entries:
fn, dn = PureWindowsPath(i.file_name), PureWindowsPath(
i.download_name)
ln = f"{fn},{dn},{i.file_hash},{i.download_hash},{i.file_size},{i.download_size},{int(i.flags) & 0xFFFF}"
logging.trace(ln)
f.write(f"{ln}\n")
def generate(self) -> Generator[APData, None, None]:
logging.debug(
f"Pulling the rss feed at {self.url}, last etag: {self.last_etag}, modif: {self.last_modified}"
)
feed = feedparser.parse(self.url,
etag=self.last_etag,
modified=self.last_modified)
if feed.status != 304:
for entry in feed.entries:
logging.trace(f"Rss entry: {json.dumps(entry, indent=4)}")
yield self.mapper(entry)
def exec_env_sanity_checks(module) -> None:
logging.trace("Verifying selected --exec-env plugin interface")
classes = [
'ParsedCmdlineConfigurer', 'ExpRunShellCmdsGenerator',
'ExpShellCmdsGenerator', 'ExecEnvChecker'
]
in_module = inspect.getmembers(module, inspect.isclass)
for c in classes:
assert (any([c in name for name, _ in in_module])),\
f"Execution environment plugin '{module.__name__}' does not define '{c}'"
def match_function_to_pattern(self, start_address):
end_address = utils.id_function_block_end(start_address)
has_unsupported_operations = self.unsupported_operations(
start_address, end_address)
if has_unsupported_operations == True:
logging.trace('Unsupported function.')
return False
is_match = self.analyse_function(start_address, end_address)
return is_match
def run(self):
self.logger.notice(
'brozzler %s - brozzler-worker starting', brozzler.__version__)
last_nothing_to_claim = 0
try:
while not self._shutdown.is_set():
self._service_heartbeat_if_due()
if time.time() - last_nothing_to_claim > 20:
try:
self._start_browsing_some_sites()
except brozzler.browser.NoBrowsersAvailable:
logging.trace(
"all %s browsers are in use",
self._max_browsers)
except brozzler.NothingToClaim:
last_nothing_to_claim = time.time()
logging.trace(
"nothing to claim, all available active sites "
"are already claimed by a brozzler worker")
time.sleep(0.5)
self.logger.notice("shutdown requested")
except r.ReqlError as e:
self.logger.error(
"caught rethinkdb exception, will try to proceed",
exc_info=True)
except brozzler.ShutdownRequested:
self.logger.info("shutdown requested")
except:
self.logger.critical(
"thread exiting due to unexpected exception",
exc_info=True)
finally:
if self._service_registry and hasattr(self, "status_info"):
try:
self._service_registry.unregister(self.status_info["id"])
except:
self.logger.error(
"failed to unregister from service registry",
exc_info=True)
self.logger.info(
'shutting down %s brozzling threads',
len(self._browsing_threads))
with self._browsing_threads_lock:
for th in self._browsing_threads:
if th.is_alive():
brozzler.thread_raise(th, brozzler.ShutdownRequested)
self._browser_pool.shutdown_now()
# copy to avoid "RuntimeError: Set changed size during iteration"
thredz = set(self._browsing_threads)
for th in thredz:
th.join()
def run(self):
self.logger.notice(
'brozzler %s - brozzler-worker starting', brozzler.__version__)
last_nothing_to_claim = 0
try:
while not self._shutdown.is_set():
self._service_heartbeat_if_due()
if time.time() - last_nothing_to_claim > 20:
try:
self._start_browsing_some_sites()
except brozzler.browser.NoBrowsersAvailable:
logging.trace(
"all %s browsers are in use",
self._max_browsers)
except brozzler.NothingToClaim:
last_nothing_to_claim = time.time()
logging.trace(
"nothing to claim, all available active sites "
"are already claimed by a brozzler worker")
time.sleep(0.5)
self.logger.notice("shutdown requested")
except r.ReqlError as e:
self.logger.error(
"caught rethinkdb exception, will try to proceed",
exc_info=True)
except brozzler.ShutdownRequested:
self.logger.info("shutdown requested")
except:
self.logger.critical(
"thread exiting due to unexpected exception",
exc_info=True)
finally:
if self._service_registry and hasattr(self, "status_info"):
try:
self._service_registry.unregister(self.status_info["id"])
except:
self.logger.error(
"failed to unregister from service registry",
exc_info=True)
self.logger.info(
'shutting down %s brozzling threads',
len(self._browsing_threads))
with self._browsing_threads_lock:
for th in self._browsing_threads:
if th.is_alive():
brozzler.thread_raise(th, brozzler.ShutdownRequested)
self._browser_pool.shutdown_now()
# copy to avoid "RuntimeError: Set changed size during iteration"
thredz = set(self._browsing_threads)
for th in thredz:
th.join()
def _onPreAuthContent(self, content):
try:
if not content:
logging.debug("no preauth content")
return
preauth = json.loads(content)
logging.trace("webauthn preauth:\n%s\n", json.dumps(preauth, indent=2))
qApp.setOverrideCursor(Qt.WaitCursor)
self.authn_session_page.setUrl(QUrl(preauth["redirect_url"]))
except (ValueError, Exception) as e:
logging.error(format_exception(e))
self.set_current_html(ERROR_HTML % content)
def refreshEvent(self,dontRebuild=False,forceIx=-1):
currSeq=self.getter()
if len(currSeq)!=self.form.count()/2: #rowCount():
if dontRebuild: return # length changed behind our back, just pretend nothing happened and update next time instead
self.rebuild()
currSeq=self.getter()
for i in range(len(currSeq)):
item=self.form.itemAt(i,QFormLayout.FieldRole)
logging.trace('got item #%d %s'%(i,str(item.widget())))
widget=item.widget()
if not widget.hot:
widget.refresh()
if forceIx>=0 and forceIx==i: widget.setFocus()
def smooth(self, seq):
if len(seq) < self.window:
# Cannot smooth a sequence smaller than the smooth window
if log.is_trace(): log.trace("Not smoothing sequence; too small")
return seq
def smooth_beg(seq):
for e in self.beg_exprs:
m = e.match(seq)
if m:
a, b, rest = m.group(1, 2, 3)
if len(a) > len(b):
seq = "{0}{1}{2}".format(a, a[0] * len(b), rest)
else:
seq = "{0}{1}{2}".format(b[0] * len(a), b, rest)
return seq
return None
def smooth_end(seq):
for e in self.end_exprs:
m = e.match(seq)
if m:
rest, a, b = m.group(1, 2, 3)
if len(a) > len(b):
seq = "{0}{1}{2}".format(rest, a, a[0] * len(b))
else:
seq = "{0}{1}{2}".format(rest, b[0] * len(a), b)
return seq
return None
def smooth_mid(seq):
for e in self.mid_exprs:
m = e.match(seq)
if m:
beg, mid, end = m.group(1, 2, 3)
return "{0}{1}{2}".format(beg, end[0] * len(mid), end)
return None
for func in (smooth_beg, smooth_end, smooth_mid):
for i in xrange(0, self.max_iter):
newseq = func(seq)
if newseq:
seq = newseq
else:
break
return seq
def current_rates(self, time_period_minutes):
assert time_period_minutes > 0
with self._lock:
now = time.time()
urls = self.urls
warc_bytes = self.warc_bytes
t = now - time_period_minutes * 60
if time_period_minutes <= 2:
start_snap = self._closest_ten_sec_snap(t)
else:
start_snap = self._closest_minute_snap(t)
elapsed = now - start_snap[0]
logging.trace(
'elapsed=%0.1fs urls=%s warc_bytes=%s', elapsed,
urls - start_snap[1], warc_bytes - start_snap[2])
return elapsed, (urls - start_snap[1]) / elapsed, (warc_bytes - start_snap[2]) / elapsed
def run( application, address, port ):
loop = asyncio.get_event_loop()
server = Server(application, address, port)
coro = asyncio.start_server(server.request, address, port, loop=loop)
server = loop.run_until_complete(coro)
socket = server.sockets[0].getsockname()
logging.info("Retro {font_server}asyncio{reset} server listening on {font_url}http://{host}:{port}{reset}".format(
host=socket[0], port=socket[1],
font_server=bold(255),
font_url=normal(51),
reset=RESET,
))
try:
loop.run_forever()
except KeyboardInterrupt:
pass
# Close the server
server.close()
loop.run_until_complete(server.wait_closed())
loop.close()
logging.trace("done")
def rebuild(self):
currSeq=self.getter()
# clear everything
rows=int(self.form.count()/2)
for row in range(rows):
logging.trace('counts',self.form.rowCount(),self.form.count())
for wi in self.form.itemAt(row,QFormLayout.FieldRole),self.form.itemAt(row,QFormLayout.LabelRole):
self.form.removeItem(wi)
logging.trace('deleting widget',wi.widget())
widget=wi.widget(); widget.hide(); del widget # for some reason, deleting does not make the thing disappear visually; hiding does, however
logging.trace('counts after ',self.form.rowCount(),self.form.count())
logging.debug('cleared')
# add everything
Klass=_fundamentalEditorMap.get(self.itemType,None)
if not Klass:
errMsg=QTextEdit(self)
errMsg.setReadOnly(True); errMsg.setText("Sorry, editing sequences of %s's is not (yet?) implemented."%(self.itemType.__name__))
self.form.insertRow(0,'<b>Error</b>',errMsg)
return
class ItemGetter(object):
def __init__(self,getter,index): self.getter,self.index=getter,index
def __call__(self): return self.getter()[self.index]
class ItemSetter(object):
def __init__(self,getter,setter,index): self.getter,self.setter,self.index=getter,setter,index
def __call__(self,val): seq=self.getter(); seq[self.index]=val; self.setter(seq)
for i,item in enumerate(currSeq):
widget=Klass(self,ItemGetter(self.getter,i),ItemSetter(self.getter,self.setter,i)) #proxy,'value')
self.form.insertRow(i,'%d. '%i,widget)
logging.debug('added item %d %s'%(i,str(widget)))
if len(currSeq)==0: self.form.insertRow(0,'<i>empty</i>',QLabel('<i>(right-click for menu)</i>'))
logging.debug('rebuilt, will refresh now')
self.refreshEvent(dontRebuild=True) # avoid infinite recursion it the length would change meanwhile
def _read(self):
d = self.socket.recv(65536)
if not d:
raise ConnectionClosed(self.socket_path)
b = self.read_buffer
b += d
while b.ready:
if self._decode:
packet = b.decode(self._decode)
self._decode = None
b.want(header.size)
name = packet.__class__.__name__
logging.trace("recv %r", packet)
try:
h = getattr(self, "handle_" + name)
except AttributeError:
h = getattr(self.handler, "babel_" + name)
h(*packet)
else:
packet_type, size = b.unpack_from(header)
self._decode = Packet.response_dict[packet_type]
b.want(size)
def _formatElement( self, element ):
"""Formats the given element and its content, by using the formatting
operations defined in this class."""
attributes = element._attributesAsHTML()
exceptions = HTML_EXCEPTIONS.get(element.name)
content = element.content
mode = element.mode.split("+")[0] if element.mode else None
# FIXME: Flags are not properly supported
if exceptions:
not_empty = exceptions.get("NOT_EMPTY")
if not_empty != None and not content:
element.content.append(Text(not_empty))
# Does this element has any content that needs to be pre-processed?
if mode == "sugar":
lines = element.contentAsLines()
import paml.web
source = u"".join(lines)
t = time.time()
res, _ = paml.web.processSugar(source, "", cache=self.useProcessCache, includeSource=element.mode.endswith("+source"))
logging.trace("Parsed Sugar: {0} lines in {1:0.2f}s".format(len(lines), time.time() - t))
element.content = [Text(res)]
elif mode in ("coffeescript", "coffee"):
lines = element.contentAsLines()
import paml.web
source = u"".join(lines)
t = time.time()
res, _ = paml.web.processCoffeeScript(source, "", cache=self.useProcessCacheFalse)
logging.trace("Parsed CoffeeScript: {0} lines in {1:0.2f}s".format(len(lines), time.time() - t))
element.content = [Text(res)]
elif mode in ("typescript", "ts"):
lines = element.contentAsLines()
import paml.web
source = u"".join(lines)
t = time.time()
res, _ = paml.web.processTypeScript(source, "", cache=self.useProcessCacheFalse)
logging.trace("Parsed TypeScript: {0} lines in {1:0.2f}s".format(len(lines), time.time() - t))
element.content = [Text(res)]
elif mode in ("clevercss", "ccss"):
lines = element.contentAsLines()
import paml.web
source = u"".join(lines)
t = time.time()
res, _ = paml.web.processCleverCSS(source, ".")
logging.trace("Parsed CleverCSS: {0} lines in {1:0.2f}s".format(len(lines), time.time() - t))
element.content = [Text(res)]
elif mode in ("pythoniccss", "pcss"):
lines = element.contentAsLines()
import paml.web
source = u"".join(lines)
t = time.time()
res, _ = paml.web.processPythonicCSS(source, ".")
logging.trace("Parsed PythonicCSS: {0} lines in {1:0.2f}s".format(len(lines), time.time() - t))
element.content = [Text(res)]
elif element.mode and element.mode.endswith("nobrackets"):
lines = element.contentAsLines()
import paml.web
source = u"".join(lines)
t = time.time()
prefix = element.mode[0:0-(len("nobrackets"))]
suffix = ".nb"
if prefix: suffix = "." + prefix + suffix
p = tempfile.mktemp(suffix=suffix)
with open(p, "w") as f: f.write(source)
res, _ = paml.web.processNobrackets(source, p)
if os.path.exists(p): os.unlink(p)
logging.trace("Parsed Nobrackets: {0} lines in {1:0.2f}s".format(len(lines), time.time() - t))
element.content = [Text(res)]
elif mode == "texto":
lines = element.contentAsLines()
import texto
source = u"".join(lines)
res = ensure_unicode(texto.toHTML(source))
element.content = [Text(res)]
elif mode == "hjson":
lines = element.contentAsLines()
import hjson
source = u"".join(lines)
res = ensure_unicode(hjson.dumpsJSON(hjson.loads(source)))
element.content = [Text(res)]
elif mode == "raw":
element.setFormat(FORMAT_PRESERVE)
# NOTE: This is a post-processor
if element.mode and (element.mode.endswith ("+escape") or "+escape+" in element.mode):
for text in element.content:
if isinstance(text, Text):
text.content = text.content.replace("<", "<").replace(">", ">")
# If the element has any content, then we apply it
if element.content:
flags = element.getFormatFlags() + list(self.getDefaults(element.name))
self.pushFlags(*flags)
if element.isPI:
assert not attributes, "Processing instruction cannot have attributes"
start = "<?%s " % (element.name)
end = " ?>"
else:
start = "<%s%s>" % (element.name, attributes)
end = "</%s>" % (element.name)
if self.hasFlag(FORMAT_INLINE):
if self._inlineCanSpanOneLine(element):
self.setFlag(FORMAT_SINGLE_LINE)
# If the element is an inline, we enter the SINGLE_LINE formatting
# mode, without adding an new line
# FIXME: isInline is always false
if element.isInline:
self.pushFlags(FORMAT_SINGLE_LINE)
self.writeTag(start)
self._formatContent(element)
self.writeTag(end)
self.popFlags()
# Or maybe the element has a SINGLE_LINE flag, in which case we add a
# newline inbetween
elif self.hasFlag(FORMAT_SINGLE_LINE) or element.isTextOnly():
self.newLine()
self.writeTag(start)
self._formatContent(element)
self.writeTag(end)
# Otherwise it's a normal open/closed element
else:
self.newLine()
self.writeTag(start)
self.newLine()
self.startIndent()
self._formatContent(element)
self.endIndent()
self.ensureNewLine()
self.writeTag(end)
self.popFlags()
# Otherwise it doesn't have any content
else:
if exceptions and exceptions.get("NO_CLOSING"):
text = "<%s%s>" % (element.name, attributes)
else:
text = "<%s%s />" % (element.name, attributes)
# And if it's an inline, we don't add a newline
if not element.isInline: self.newLine()
self.writeTag(text)
def get_data(self, pkt_dump_only=False, timeout=1):
""" read data from the socket """
# This method behaves slightly differently depending on whether it is
# called to read the response to a command (pkt_dump_only = 0) or if
# it is called specifically to read a packet dump (pkt_dump_only = 1).
#
# Packet dumps look like:
# pktdump,<port_id>,<data_len>\n
# <packet contents as byte array>\n
# This means the total packet dump message consists of 2 lines instead
# of 1 line.
#
# - Response for a command (pkt_dump_only = 0):
# 1) Read response from the socket until \n (end of message)
# 2a) If the response is a packet dump header (starts with "pktdump,"):
# - Read the packet payload and store the packet dump for later
# retrieval.
# - Reset the state and restart from 1). Eventually state 2b) will
# be reached and the function will return.
# 2b) If the response is not a packet dump:
# - Return the received message as a string
#
# - Explicit request to read a packet dump (pkt_dump_only = 1):
# - Read the dump header and payload
# - Store the packet dump for later retrieval
# - Return True to signify a packet dump was successfully read
ret_str = None
dat = ""
done = 0
while done == 0:
# recv() is blocking, so avoid calling it when no data is waiting.
ready = select.select([self._sock], [], [], timeout)
if ready[0]:
logging.debug("Reading from socket")
dat = self._sock.recv(256)
ret_str = ""
else:
logging.debug("No data waiting on socket")
done = 1
logging.trace("Iterating over input buffer (%d octets)", len(dat))
i = 0
while i < len(dat) and (done == 0):
if dat[i] == '\n':
# Terminating \n for a string reply encountered.
if ret_str.startswith('pktdump,'):
logging.trace("Packet dump header read: [%s]", ret_str)
# The line is a packet dump header. Parse it, read the
# packet payload, store the dump for later retrieval.
# Skip over the packet dump and continue processing: a
# 1-line response may follow the packet dump.
_, port_id, data_len = ret_str.split(',', 2)
port_id, data_len = int(port_id), int(data_len)
data_start = i + 1 # + 1 to skip over \n
data_end = data_start + data_len
pkt_payload = array.array('B', map(ord, dat[data_start:data_end]))
pkt_dump = PacketDump(port_id, data_len, pkt_payload)
self._pkt_dumps.append(pkt_dump)
# Reset state. Increment i with payload length and add
# 1 for the trailing \n.
ret_str = ""
i += data_len + 1
if pkt_dump_only:
# Return boolean instead of string to signal
# successful reception of the packet dump.
logging.trace("Packet dump stored, returning")
ret_str = True
done = 1
else:
# Regular 1-line message. Stop reading from the socket.
logging.trace("Regular response read")
done = 1
else:
ret_str += dat[i]
i = i + 1
logging.debug("Received data from socket: [%s]", ret_str)
return ret_str
def simple_table(array, has_hdr = True):
"""Generate a simple table with the contents of array.
Args:
array ([[hdr1,hdr2,...], [...],...]): A list of lists containing the
values to put in the table.
has_hdr (bool): True if the first list contains the header names of
the columns. False if the first list contains data.
Returns:
str. A simple table in RST format containing the array data.
"""
result = ''
logging.trace('Array passed in: %s', array)
# Calculate max. lengths of each column. RST tables require column
# alignment.
col_widths = [0] * len(sorted(array, cmp=lambda x,y: -cmp(len(x), len(y)))[0])
logging.trace('Initialized col_widths with 0: %s', col_widths)
for row in array:
logging.trace('- Adjusting column widths for row %s', row)
for index in range(len(row)):
col_widths[index] = max(col_widths[index], len(str(row[index])))
logging.trace(' New col_widths: %s', col_widths)
# Draw top border
for col_width in col_widths:
result += '=' * col_width
result += ' '
result += '\n'
# Draw table cells, pad every cell with appropriate amount of spaces for
# correct alignment.
hdr_border_drawn = False
for row in array:
if len(row) > 1:
for index in range(len(row)):
result += str(row[index]).ljust(col_widths[index])
result += ' '
result += '\n'
# Draw table header border if needed
if has_hdr and not hdr_border_drawn:
for col_width in col_widths:
result += '=' * col_width
result += ' '
result += '\n'
hdr_border_drawn = True
else:
result += row[0] + "\n"
for index in range(len(col_widths)):
result += '=' * col_widths[index]
result += '==' if index > 0 else ""
result += '\n'
hdr_border_drawn = True
# Draw bottom border
for col_width in col_widths:
result += '=' * col_width
result += ' '
result += '\n\n'
return result
def start_component(componentclass, interactive_callback=None, thread_policy=None, loggerName=None):
execparams, interactive = parseCommandLineArgs(componentclass)
setupSignalHandlers()
orb = None
try:
try:
orb = createOrb()
globals()['__orb__'] = orb
name_binding=""
component_identifier=""
# set up backwards-compatable logging
#configureLogging(execparams, loggerName, orb)
componentPOA = getPOA(orb, thread_policy, "componentPOA")
if not execparams.has_key("COMPONENT_IDENTIFIER"):
if not interactive:
logging.warning("No 'COMPONENT_IDENTIFIER' argument provided")
execparams["COMPONENT_IDENTIFIER"] = ""
if not execparams.has_key("NAME_BINDING"):
if not interactive:
logging.warning("No 'NAME_BINDING' argument provided")
execparams["NAME_BINDING"] = ""
if not execparams.has_key("PROFILE_NAME"):
if not interactive:
logging.warning("No 'PROFILE_NAME' argument provided")
execparams["PROFILE_NAME"] = ""
# Configure logging (defaulting to INFO level).
log_config_uri = execparams.get("LOGGING_CONFIG_URI", None)
debug_level = execparams.get("DEBUG_LEVEL", None)
if debug_level != None: debug_level = int(debug_level)
dpath=execparams.get("DOM_PATH", "")
component_identifier=execparams.get("COMPONENT_IDENTIFIER", "")
name_binding=execparams.get("NAME_BINDING", "")
## sets up logging during component startup
ctx = ossie.logger.ComponentCtx(
name = name_binding,
id = component_identifier,
dpath = dpath )
ossie.logger.Configure(
logcfgUri = log_config_uri,
logLevel = debug_level,
ctx = ctx)
# Create the component
component_Obj = componentclass(execparams["COMPONENT_IDENTIFIER"], execparams)
component_Obj.setAdditionalParameters(execparams["PROFILE_NAME"])
componentPOA.activate_object(component_Obj)
component_Var = component_Obj._this()
## sets up logging context for resource to support CF::Logging
component_Obj.saveLoggingContext( log_config_uri, debug_level, ctx )
# get the naming context and bind to it
if execparams.has_key("NAMING_CONTEXT_IOR"):
rootContext = orb.string_to_object(execparams['NAMING_CONTEXT_IOR'])
if rootContext == None:
logging.error("Failed to lookup naming context")
sys.exit(-1)
rootContext = rootContext._narrow(CosNaming.NamingContext)
name = URI.stringToName(execparams['NAME_BINDING'])
rootContext.rebind(name, component_Var)
else:
if not interactive:
logging.warning("Skipping name-binding because required execparams 'NAMING_CONTEXT_IOR' is missing")
if not interactive:
logging.trace("Starting ORB event loop")
orb.run()
else:
logging.trace("Entering interactive mode")
if callable(interactive_callback):
# Pass only the Var to prevent anybody from calling non-CORBA functions
interactive_callback(component_Obj)
else:
print orb.object_to_string(component_Obj._this())
orb.run()
try:
orb.shutdown(true)
except:
pass
signal.signal(signal.SIGINT, signal.SIG_IGN)
except SystemExit:
pass
except KeyboardInterrupt:
pass
except:
logging.exception("Unexpected Error")
finally:
if orb:
orb.destroy()
def start_component(componentclass, interactive_callback=None, thread_policy=None, loggerName=None):
execparams, interactive = parseCommandLineArgs(componentclass)
setupSignalHandlers()
try:
try:
orb = createOrb()
globals()['__orb__'] = orb
configureLogging(execparams, loggerName, orb)
componentPOA = getPOA(orb, thread_policy, "componentPOA")
if not execparams.has_key("COMPONENT_IDENTIFIER"):
if not interactive:
logging.warning("No 'COMPONENT_IDENTIFIER' argument provided")
execparams["COMPONENT_IDENTIFIER"] = ""
if not execparams.has_key("NAME_BINDING"):
if not interactive:
logging.warning("No 'NAME_BINDING' argument provided")
execparams["NAME_BINDING"] = ""
# Create the component
component_Obj = componentclass(execparams["COMPONENT_IDENTIFIER"], execparams)
componentPOA.activate_object(component_Obj)
component_Var = component_Obj._this()
# get the naming context and bind to it
if execparams.has_key("NAMING_CONTEXT_IOR"):
rootContext = orb.string_to_object(execparams['NAMING_CONTEXT_IOR'])
if rootContext == None:
logging.error("Failed to lookup naming context")
sys.exit(-1)
rootContext = rootContext._narrow(CosNaming.NamingContext)
name = URI.stringToName(execparams['NAME_BINDING'])
rootContext.rebind(name, component_Var)
else:
if not interactive:
logging.warning("Skipping name-binding because required execparams 'NAMING_CONTEXT_IOR' is missing")
if not interactive:
logging.trace("Starting ORB event loop")
orb.run()
else:
logging.trace("Entering interactive mode")
if callable(interactive_callback):
# Pass only the Var to prevent anybody from calling non-CORBA functions
interactive_callback(component_Obj)
else:
print orb.object_to_string(component_Obj._this())
orb.run()
except SystemExit:
pass
except KeyboardInterrupt:
pass
except:
logging.exception("Unexpected Error")
finally:
if orb:
orb.destroy()
objectPlacements = {
"Object 1": (2, 3),
"Object 2": (6, 2),
"Object 3": (3, 7),
"Object 4": (7, 6)
}
# Learn objects in egocentric space.
exp.learnObjects(objectPlacements)
# Infer the objects without any location input.
filename = "logs/infer-no-location.csv"
with open(filename, "w") as fileOut:
print "Logging to", filename
with trace(exp, csv.writer(fileOut)):
exp.inferObjectsWithRandomMovements(objectPlacements)
# Shuffle the objects. Infer them without any location input.
filename = "logs/infer-shuffled-location.csv"
with open(filename, "w") as fileOut:
print "Logging to", filename
with trace(exp, csv.writer(fileOut)):
exp.inferObjectsWithRandomMovements({
"Object 1": (7, 6),
"Object 2": (2, 7),
"Object 3": (7, 2),
"Object 4": (3, 3)
})
print "Visualize these CSV files at:"
def _forwardPacket(self, packet, next_hop):
#FIXME: implement
logging.trace("Forwarding packet %s at LSR %s to %s", packet, self, next_hop)
return None
def make_test(f):
logging.trace("Wrapping test %s with setup and teardown", f.__name__)
f.__dict__.update(is_test=True, setup=setup, teardown=teardown)
return f
def main():
# Get arguments
config = getConfig()
cert = x509.Cert(config.ca, config.key, config.cert)
config.openvpn_args += cert.openvpn_args
if config.test:
sys.exit(eval(config.test, None, config.__dict__))
# Set logging
utils.setupLog(config.verbose, os.path.join(config.log, "re6stnet.log"))
logging.trace("Environment: %r", os.environ)
logging.trace("Configuration: %r", config)
utils.makedirs(config.state)
db_path = os.path.join(config.state, "cache.db")
if config.ovpnlog:
plib.ovpn_log = config.log
exit.signal(0, signal.SIGINT, signal.SIGTERM)
exit.signal(-1, signal.SIGHUP, signal.SIGUSR2)
cache = Cache(db_path, config.registry, cert)
network = cert.network
if config.client_count is None:
config.client_count = cache.client_count
if config.max_clients is None:
config.max_clients = cache.max_clients
if config.table is not None:
logging.warning("--table option is deprecated: use --default instead")
config.default = True
if config.default and config.gateway:
sys.exit("error: conflicting options --default and --gateway")
if "none" in config.disable_proto:
config.disable_proto = ()
if config.default:
# Make sure we won't tunnel over re6st.
config.disable_proto = tuple(set(("tcp6", "udp6")).union(config.disable_proto))
address = ()
server_tunnels = {}
forwarder = None
if config.client:
config.babel_args.append("re6stnet")
elif config.max_clients:
if config.pp:
pp = [(int(port), proto) for port, proto in config.pp]
for port, proto in pp:
if proto in config.disable_proto:
sys.exit("error: conflicting options --disable-proto %s" " and --pp %u %s" % (proto, port, proto))
else:
pp = [x for x in ((1194, "udp"), (1194, "tcp")) if x[1] not in config.disable_proto]
def ip_changed(ip):
for family, proto_list in ((socket.AF_INET, ("tcp", "udp")), (socket.AF_INET6, ("tcp6", "udp6"))):
try:
socket.inet_pton(family, ip)
break
except socket.error:
pass
else:
family = None
return family, [(ip, str(port), proto) for port, proto in pp if not family or proto in proto_list]
if config.gw_list:
gw_list = deque(config.gw_list)
def remote_gateway(dest):
gw_list.rotate()
return gw_list[0]
else:
remote_gateway = None
if len(config.ip) > 1:
if "upnp" in config.ip or "any" in config.ip:
sys.exit("error: argument --ip can be given only once with" " 'any' or 'upnp' value")
logging.info(
"Multiple --ip passed: note that re6st does nothing to"
" make sure that incoming paquets are replied via the correct"
" gateway. So without manual network configuration, this can"
" not be used to accept server connections from multiple"
" gateways."
)
if "upnp" in config.ip or not config.ip:
logging.info("Attempting automatic configuration via UPnP...")
try:
from re6st.upnpigd import Forwarder
forwarder = Forwarder("re6stnet openvpn server")
except Exception, e:
if config.ip:
raise
logging.info("%s: assume we are not NATed", e)
else:
atexit.register(forwarder.clear)
for port, proto in pp:
forwarder.addRule(port, proto)
ip_changed = forwarder.checkExternalIp
address = (ip_changed(),)
elif "any" not in config.ip:
address = map(ip_changed, config.ip)
ip_changed = None
for x in pp:
server_tunnels.setdefault("re6stnet-" + x[1], x)
def _process_batch(self, batch):
batch_buckets = self._tally_batch(batch)
self._update_db(batch_buckets)
logging.trace('updated stats from batch of %s', len(batch))
def classify_windows(snps, genos, exclude, window_size, slide, min_hom, smoothing_size):
nsnp = len(snps)
if log.is_debug(): log.debug("{0} snps".format(nsnp))
nsam = len(genos)
start = 0
end = start + window_size - 1
excl = exclude.next()
regions = []
region = None
start_pos = snps[0].position
while end < nsnp:
pos = Range(snps[start].position, snps[end].position, True)
if log.is_trace(): log.trace("Current region: {pos.start}-{pos.end}".format(pos=pos))
if excl and excl.intersects(pos):
if log.is_trace(): log.trace("Current region overlaps exclude region: {0}".format(excl))
if region:
region.end_pos = excl.start - 1
if log.is_trace(): log.trace(
"Ending region {0} with {1} windows".format(region, len(region.windows)))
region = None
while end <= nsnp and excl >= snps[start].position:
start += 1
start_pos = excl.end + 1
end = start + window_size - 1
excl = exclude.next()
else:
if not region:
region = Region(nsam, start_pos)
regions.append(region)
window = region.add_window(pos)
if log.is_trace(): log.trace("Adding window for range {0}".format(pos))
for i,g in enumerate(genos):
hom = g.homozygosity(start, end)
call = hom >= min_hom
if log.is_trace(): log.trace(
"{0} markers are homozygous; call = {1}".format(hom, call))
window.append(hom)
region.append(i, call)
start += slide
end += slide
if region:
region.end_pos = snps[-1].position
if log.is_trace(): log.trace(
"Ending region {0} with {1} windows".format(region, len(region.windows)))
if log.is_debug(): log.debug("Regions: {0}".format(regions))
return regions
def main():
print "Dataplane Automated Testing System, version " + __version__
print "Copyright (c) 2015-2016, Intel Corporation. All rights reserved."
print "Copyright (c) 2016, Viosoft Corporation. All rights reserved."
print
args = parse_cmdline()
config.set_cmdline_args(args)
read_configfile(args)
setup_logging()
logging.debug("Command line arguments: %s", args)
all_tests = dats.test.get_tests(args.tests_dir)
if args.list:
print "Tests in directory " + args.tests_dir + ": " + ' '.join(sorted(all_tests.keys()))
sys.exit(0)
# SUT information
sut_information_hw = [["Hardware"]]
sut_inf_commands_hw = [
["sudo dmidecode --type system | grep 'Product Name' | cut -d: -f2 2> /dev/null", "Platform"],
["grep 'model name' /proc/cpuinfo | uniq | cut -d : -f 2 | cut -c 2- 2> /dev/null", "Processor"],
["cat /proc/cpuinfo | grep processor | wc -l", "# of cores"],
["printf '%d MB' \\$(free -m | grep Mem | tr -s ' ' | cut -d' ' -f2)", "RAM"],
[config.getOption("sutDpdkDir") + "/tools/dpdk_nic_bind.py --status | grep drv=igb_uio | cut -d\\' -f 2", "DPDK ports"],
]
sut_information_sw = [["Software"]]
sut_inf_commands_sw = [
["sudo dmidecode --type bios | grep 'Version' | cut -d: -f2 2> /dev/null", "BIOS version"],
["sudo dmidecode --type bios | grep 'Release Date' | cut -d: -f2 2> /dev/null", "BIOS release date"],
["sed '1!d' /etc/*-release", "OS"],
["uname -rm", "Kernel"],
["printf 'v%d.%d' "
+ "\\$(grep '#define VERSION_MAJOR' "
+ config.getOption('sutProxDir') + "/version.h "
+ "| sed 's/[^0-9]//g') "
+ "\\$(grep '#define VERSION_MINOR' "
+ config.getOption('sutProxDir') + "/version.h "
+ "| sed 's/[^0-9]//g')",
"PROX version"],
["printf 'v%d.%d.%d' "
+ "\\$(grep '#define RTE_VER_MAJOR' "
+ config.getOption('sutDpdkDir') + "/lib/librte_eal/common/include/rte_version.h "
+ "| sed 's/[^0-9]//g') "
+ "\\$(grep '#define RTE_VER_MINOR' "
+ config.getOption('sutDpdkDir') + "/lib/librte_eal/common/include/rte_version.h "
+ "| sed 's/[^0-9]//g') "
+ "\\$(grep '#define RTE_VER_PATCH_LEVEL' "
+ config.getOption('sutDpdkDir') + "/lib/librte_eal/common/include/rte_version.h "
+ "| sed 's/[^0-9]//g') ",
"DPDK version"],
["cat /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages 2>/dev/null",
"Hugepages - 2048kB"],
["cat /sys/devices/system/node/node0/hugepages/hugepages-1048576kB/nr_hugepages 2>/dev/null",
"Hugepages - 1GB"],
#["uname -a"],
#["cat /proc/cmdline"],
#["lspci | grep 82599 | cut -d ' ' -f1", "Niantic ports"],
#["lspci | grep X710 | cut -d ' ' -f1", "Fortville ports"],
]
logging.info("Retrieving SUT hardware description")
execute_inf_commands(sut_inf_commands_hw, sut_information_hw)
logging.info("Retrieving SUT software description")
execute_inf_commands(sut_inf_commands_sw, sut_information_sw)
### Main program
if not os.path.exists(args.report_dir):
os.makedirs(args.report_dir)
# update the parameters.lua file to use the correct CPU socket
os.system("sed -i 's/tester_socket_id=.*/tester_socket_id=\"" + str(config.getOption('testerSocketId')) + "\"/' " \
+ args.tests_dir + "/prox-configs/parameters.lua")
os.system("sed -i 's/sut_socket_id=.*/sut_socket_id=\"" + str(config.getOption('sutSocketId')) + "\"/' " \
+ args.tests_dir + "/prox-configs/parameters.lua")
# Determine which tests to run. These locations are checked in order, the
# first non-empty result is used:
# - command line: test names specified as parameters
# - config file: test names specified in the [general] section, key 'tests'
# - all tests in the directory specified on the command line with -d
# - all tests in directory tests/
tests_to_run = args.test
if tests_to_run is not None and len(tests_to_run) == 0:
logging.debug("No test specified on the command line. Checking config file")
tests_to_run = config.getOption('tests')
if tests_to_run is not None:
tests_to_run = tests_to_run.split(',')
if tests_to_run is None:
logging.debug("No test specified in the config file. Running all tests from " + args.tests_dir)
tests_to_run = sorted(all_tests.keys())
logging.debug("Tests to run: '%s'", "', '".join(tests_to_run))
test_summaries = []
for test in tests_to_run:
if test not in all_tests.keys():
logging.error("Test '%s' not found. Use the '-l' command line parameter, possibly with '-d' to see the list of available tests.", test)
continue
logging.info("Loading test suite %s", test)
# Load test script directly from disk
test_module = imp.load_source(test, all_tests[test])
# Get all classes defined in test_module. Filter out imported classes and
# classes that are not derived from DATSTest.
test_classes = [c[1] for c in inspect.getmembers(test_module, inspect.isclass)]
test_classes = [c for c in test_classes if c.__module__ == test_module.__name__]
test_classes = [c for c in test_classes if issubclass(c, TestBase)]
for test_class in test_classes:
test = test_class()
logging.info("Running test %s - %s",
test.__class__.__name__, test.short_descr())
test_results = None
try:
test.setup_class()
test_results = test.run_all_tests()
test.teardown_class()
logging.trace('Test results: %s', test_results)
test_summaries.append(dict(test=test, results=test_results))
except KeyboardInterrupt:
logging.error("Test run interrupted by keyboard. Generating partial report.")
test_summaries.append(dict(test=test, results=Exception('Test run interrupted by user')))
break
except IOError, ex:
logging.error("I/O error ({0}): {1}: {2}".format(ex.errno, ex.filename, ex.strerror))
test_results = ex
except Exception, ex:
logging.error(ex)
logging.debug("Exception: %s", traceback.format_exc())
