def reinject_decorated_tables():
"""
reinject the decorated tables (and columns)
"""
logger.info("reinject_decorated_tables")
# need to clear any non-decorated tables that were added during the previous run
orca._TABLES.clear()
orca._COLUMNS.clear()
orca._TABLE_CACHE.clear()
orca._COLUMN_CACHE.clear()
for name, func in iteritems(_DECORATED_TABLES):
logger.debug("reinject decorated table %s" % name)
orca.add_table(name, func)
for column_key, args in iteritems(_DECORATED_COLUMNS):
table_name, column_name = column_key
logger.debug("reinject decorated column %s.%s" % (table_name, column_name))
orca.add_column(table_name, column_name, args['func'], cache=args['cache'])
for name, args in iteritems(_DECORATED_INJECTABLES):
logger.debug("reinject decorated injectable %s" % name)
orca.add_injectable(name, args['func'], cache=args['cache'])
def __init__(self, connection=None, specification = None , base_url = None):
self._model_apis = {}
connection = connection if connection else config_connection()
# Set the default URL for the twitter API we're using.
self._base_url = Api.BASE_URL if base_url is None else base_url
if specification is None:
with closing(open_data("api.yaml")) as f:
specification = yaml.load(f)
self._connection = connection
self.limit_remaining = None
self.limit_reset = None
# Now look through all immediate attributes
for attribute, value in iteritems(specification):
if hasattr(models, attribute):
# The element matches a model object, so it will be used to bless objects of that type.
target_class = getattr(models, attribute)
self._model_apis[target_class] = dict()
# Instantiate the specificatons for all the relevant methods
for inner_attribute, inner_value in iteritems(value):
inner_value["method"] = inner_attribute
self._model_apis[target_class][inner_attribute] = _ApiMethodSpec(**inner_value)
else:
# Otherwise, add it to the tree of API calls.
value["method"] = attribute
setattr(self, attribute, ApiMethod(self._connection, self, _ApiMethodSpec(**value)))
def parse_params(indict, spec=NIF_PARAMS):
logger.debug("Parsing: {}\n{}".format(indict, spec))
outdict = indict.copy()
wrong_params = {}
for param, options in iteritems(spec):
if param[0] != "@": # Exclude json-ld properties
for alias in options.get("aliases", []):
if alias in indict:
outdict[param] = indict[alias]
if param not in outdict:
if options.get("required", False) and "default" not in options:
wrong_params[param] = spec[param]
else:
if "default" in options:
outdict[param] = options["default"]
else:
if "options" in spec[param] and \
outdict[param] not in spec[param]["options"]:
wrong_params[param] = spec[param]
if wrong_params:
logger.debug("Error parsing: %s", wrong_params)
message = Error(
status=400,
message="Missing or invalid parameters",
parameters=outdict,
errors={param: error
for param, error in iteritems(wrong_params)})
raise message
return outdict
def __str__(self):
lines = []
if self.total:
lines.append("Total: %(read)s, %(write)s" % self.total)
if self.default:
lines.append("Default: %(read)s, %(write)s" % self.default)
for tablename, table_limit in iteritems(self.tables):
lines.append(
"%s: %s, %s" % (tablename, table_limit["read"], table_limit["write"])
)
indexes = self.indexes.get(tablename, {})
for indexname, limit in iteritems(indexes):
lines.append(
"%s:%s: %s, %s"
% (tablename, indexname, limit["read"], limit["write"])
)
# Add all the throttled indexes that don't have their table throttled.
for tablename, data in iteritems(self.indexes):
if tablename in self.tables:
continue
for indexname, limit in iteritems(data):
lines.append(
"%s:%s: %s, %s"
% (tablename, indexname, limit["read"], limit["write"])
)
if lines:
return "\n".join(lines)
else:
return "No throttle"
def assertValidCheckFile(self, path):
"""Tests whether a check definition has a valid configuration."""
# Figure out the relative path of the check files.
prefix = os.path.commonprefix(config.CONFIG["Checks.config_dir"])
relpath = os.path.relpath(path, prefix)
# If the config can't load fail immediately.
try:
configs = checks.LoadConfigsFromFile(path)
except yaml.error.YAMLError as e:
self.fail("File %s could not be parsed: %s\n" % (relpath, e))
# Otherwise, check all the configs and pass/fail at the end.
errors = collections.OrderedDict()
for check_id, check_spec in iteritems(configs):
check_errors = self.GetCheckErrors(check_spec)
if check_errors:
msg = errors.setdefault(relpath, ["check_id: %s" % check_id])
msg.append(check_errors)
if errors:
message = ""
for k, v in iteritems(errors):
message += "File %s errors:\n" % k
message += " %s\n" % v[0]
for err in v[1]:
message += " %s\n" % err
self.fail(message)
def export(self, **options):
"""
Exports the data into a XML file-like stream.
Arguments:
options (dict): The exporting options
Returns:
io.BytesIO: A XML file-like stream
Raises:
ExportError: When data fails to export
"""
data = self._data.normalize(includeKey=True)
database = Element("database", name="dtb_{}".format(self._data._base.year))
for table_name, rows in iteritems(data):
if not options.get("minify"):
database.append(Comment(" Table {} ".format(table_name)))
table = SubElement(database, "table", name=table_name)
for row_data in itervalues(rows):
row = SubElement(table, "row")
for column_name, column_value in iteritems(row_data):
SubElement(row, "field", name=column_name).text = unicode(column_value)
xml_data = xml_str(database, pretty_print=not options.get("minify"), xml_declaration=True, encoding="utf-8")
return io.BytesIO(xml_data)
def sql_where_from_params(**kwargs):
"""
Utility function for converting a param dictionary into a where clause
Lists and tuples become in clauses
"""
clauses = [ 'true' ]
type_handler = collections.OrderedDict()
type_handler[type(None)] = "{0} is null"
type_handler[list] = "{0} in (%({0})s)"
type_handler[tuple] = "{0} in %({0})s"
for key, value in sorted(iteritems(kwargs)):
if isinstance(value, (tuple, list)):
if not value:
clauses = [ 'true = false' ]
break
for proposed_type, pattern in iteritems(type_handler):
if isinstance(value, proposed_type):
clauses.append(pattern.format(key))
break
else:
clauses.append("{0} = %({0})s".format(key))
return ' and '.join(clauses)
def testNoTraceOfDeletedHuntIsLeftInTheDataStore(self):
# This only works with the test data store (FakeDataStore).
if not isinstance(data_store.DB, fake_data_store.FakeDataStore):
self.skipTest("Only supported on FakeDataStore.")
with test_lib.ConfigOverrider(
{"DataRetention.hunts_ttl": rdfvalue.Duration("1s")}):
with test_lib.FakeTime(40 + 60 * self.NUM_HUNTS):
self._RunCleanup()
for hunt_urn in self.hunts_urns:
hunt_id = hunt_urn.Basename()
for subject, subject_data in iteritems(data_store.DB.subjects):
# Foreman rules are versioned, so hunt ids will be mentioned
# there. Ignoring audit events as well.
if subject == "aff4:/foreman" or subject.startswith("aff4:/audit"):
continue
self.assertNotIn(hunt_id, subject)
for column_name, values in iteritems(subject_data):
self.assertNotIn(hunt_id, column_name)
for value, _ in values:
self.assertNotIn(hunt_id, utils.SmartUnicode(value))
def _ParseInsserv(self, data):
"""/etc/insserv.conf* entries define system facilities.
Full format details are in man 8 insserv, but the basic structure is:
$variable facility1 facility2
$second_variable facility3 $variable
Any init script that specifies Required-Start: $second_variable needs to be
expanded to facility1 facility2 facility3.
Args:
data: A string of insserv definitions.
"""
p = config_file.FieldParser()
entries = p.ParseEntries(data)
raw = {e[0]: e[1:] for e in entries}
# Now expand out the facilities to services.
facilities = {}
for k, v in iteritems(raw):
# Remove interactive tags.
k = k.replace("<", "").replace(">", "")
facilities[k] = v
for k, vals in iteritems(facilities):
self.insserv[k] = []
for v in vals:
self.insserv[k].extend(self._InsservExpander(facilities, v))
def remote_getBuildRequests(self):
brids = dict()
for builderid, brid in iteritems(self.brids):
builderDict = yield self.master.data.get(('builders', builderid))
brids[builderDict['name']] = brid
defer.returnValue([(n, RemoteBuildRequest(self.master, n, brid))
for n, brid in iteritems(brids)])
def check(result):
# finish up the debounced updateSummary before checking
self.debounceClock.advance(1)
self.assertEqual(self.expected_remote_commands, [],
"assert all expected commands were run")
# in case of unexpected result, display logs in stdout for debugging failing tests
if result != self.exp_result:
for loog in itervalues(self.step.logs):
print(loog.stdout)
print(loog.stderr)
self.assertEqual(result, self.exp_result, "expected result")
if self.exp_state_string:
stepStateString = self.master.data.updates.stepStateString
stepids = list(stepStateString)
assert stepids, "no step state strings were set"
self.assertEqual(stepStateString[stepids[0]],
self.exp_state_string,
"expected step state strings")
for pn, (pv, ps) in iteritems(self.exp_properties):
self.assertTrue(self.properties.hasProperty(pn),
"missing property '%s'" % pn)
self.assertEqual(self.properties.getProperty(pn),
pv, "property '%s'" % pn)
if ps is not None:
self.assertEqual(
self.properties.getPropertySource(pn), ps, "property '%s' source" % pn)
for pn in self.exp_missing_properties:
self.assertFalse(self.properties.hasProperty(pn),
"unexpected property '%s'" % pn)
for l, contents in iteritems(self.exp_logfiles):
self.assertEqual(
self.step.logs[l].stdout, contents, "log '%s' contents" % l)
def connection_lost(self, reason):
self.service._OnDisconnect() # use RemoteObject(self) as argument?
if self.authed:
if reason.check(error.ConnectionDone):
logger.debug("Connection to %s closed", self.name)
else:
logger.info("Connection to %s lost: %s", self.name, reason.getErrorMessage())
self.authed = False
self.friends.reset_connection(self.name, RemoteObject(self))
self.service._delete_all_objects(self.connid) # makes use by other peers impossible
logger.debug("%s outstanding requests, %s streams", len(self._deferreds), len(self._multideferreds))
if reason.check(error.ConnectionDone):
for sequid, (deferred, __) in itertools.chain(iteritems(self._deferreds), iteritems(self._multideferreds)):
deferred.errback(ConnectionLost("Connection to {} closed".format(self.name)))
else:
for sequid, (deferred, __) in itertools.chain(iteritems(self._deferreds), iteritems(self._multideferreds)):
deferred.errback(ConnectionLost("Connection to {} lost: {}".format(self.name, reason.getErrorMessage())))
else:
if reason.check(error.ConnectionDone):
logger.debug("Connection to %s closed", self.addr)
else:
logger.info("Connection to %s lost: %s", self.addr, reason.getErrorMessage())
assert not self._deferreds and not self._multideferreds
self.closed = True
def _zerorpc_inspect(self):
methods = dict((m, f) for m, f in iteritems(self._methods) if not m.startswith("_"))
detailled_methods = dict(
(m, dict(args=self._format_args_spec(f._zerorpc_args()), doc=f._zerorpc_doc()))
for (m, f) in iteritems(methods)
)
return {"name": self._name, "methods": detailled_methods}
def __init__(self, json):
self.raw = json
super(JSON_Map, self).__init__(json)
cls = self.__class__
# Map the json structure to event dispatcher properties
# but only those attributes which do not already exist in the object
properties = JSON_Map.map_attributes(self, json)
self._python_properties = set()
for c in cls.__mro__:
for attr_name, attr in iteritems(c.__dict__):
if isinstance(attr, property):
self._python_properties.add(attr_name)
self._json_maps = {}
for attr_name, attr in iteritems(self.__dict__):
if isinstance(attr, JSON_Map) and attr_name in json:
self._json_maps[attr_name] = attr
with self.temp_unbind_all(*iterkeys(self.event_dispatcher_properties)):
for key in iterkeys(properties):
if key in json:
setattr(self, key, json[key])
self.bind(**{p: partial(self._update_raw , p) for p in properties})
def MultiResolvePrefix(self,
subjects,
attribute_prefix,
timestamp=None,
limit=None):
unicode_to_orig = {utils.SmartUnicode(s): s for s in subjects}
result = {}
for unicode_subject, orig_subject in iteritems(unicode_to_orig):
values = self.ResolvePrefix(
unicode_subject, attribute_prefix, timestamp=timestamp, limit=limit)
if not values:
continue
if limit:
if limit < len(values):
values = values[:limit]
result[orig_subject] = values
limit -= len(values)
if limit <= 0:
return iteritems(result)
else:
result[orig_subject] = values
return iteritems(result)
def Handle(self, args, token=None):
"""Renders list of descriptors for all the flows."""
if data_store.RelationalDBEnabled():
flow_iterator = iteritems(registry.FlowRegistry.FLOW_REGISTRY)
else:
flow_iterator = iteritems(registry.AFF4FlowRegistry.FLOW_REGISTRY)
result = []
for name, cls in sorted(flow_iterator):
# Flows without a category do not show up in the GUI.
if not getattr(cls, "category", None):
continue
# Only show flows that the user is allowed to start.
try:
if self.access_check_fn:
self.access_check_fn(token.username, name)
except access_control.UnauthorizedAccess:
continue
result.append(ApiFlowDescriptor().InitFromFlowClass(cls, token=token))
return ApiListFlowDescriptorsResult(items=result)
def CreateServiceProto(job):
"""Create the Service protobuf.
Args:
job: Launchdjobdict from servicemanagement framework.
Returns:
sysinfo_pb2.OSXServiceInformation proto
"""
service = rdf_client.OSXServiceInformation(
label=job.get("Label"),
program=job.get("Program"),
sessiontype=job.get("LimitLoadToSessionType"),
lastexitstatus=int(job["LastExitStatus"]),
timeout=int(job["TimeOut"]),
ondemand=bool(job["OnDemand"]))
for arg in job.get("ProgramArguments", "", stringify=False):
# Returns CFArray of CFStrings
service.args.Append(str(arg))
mach_dict = job.get("MachServices", {}, stringify=False)
for key, value in iteritems(mach_dict):
service.machservice.Append("%s:%s" % (key, value))
job_mach_dict = job.get("PerJobMachServices", {}, stringify=False)
for key, value in iteritems(job_mach_dict):
service.perjobmachservice.Append("%s:%s" % (key, value))
if "PID" in job:
service.pid = job["PID"].value
return service
def fill_template_service(item):
"""
Prepare fields of service with fields of service templates
:param item: field name / values of the service
:type item: dict
:return: None
"""
service = current_app.data.driver.db['service']
ignore_fields = ['_id', '_etag', '_updated', '_created', '_template_fields', '_templates',
'_is_template', '_realm', 'host', '_templates_from_host_template']
fields_not_update = []
for (field_name, field_value) in iteritems(item):
fields_not_update.append(field_name)
item['_template_fields'] = {}
if ('_is_template' not in item or not item['_is_template']) \
and '_templates' in item and item['_templates'] != []:
for service_template in item['_templates']:
services = service.find_one({'_id': ObjectId(service_template)})
if services is not None:
for (field_name, field_value) in iteritems(services):
if field_name not in fields_not_update \
and field_name not in ignore_fields:
item[field_name] = field_value
item['_template_fields'][field_name] = service_template
schema = service_schema()
ignore_schema_fields = ['_realm', '_template_fields', '_templates', '_is_template',
'_templates_from_host_template']
for key in schema['schema']:
if key not in ignore_schema_fields:
if key not in item:
item['_template_fields'][key] = 0
def check_metrics(self, results):
for intent_name, intent_metrics in iteritems(results["metrics"]):
if intent_name is None or intent_name == "null":
continue
classification_precision = intent_metrics["intent"]["precision"]
classification_recall = intent_metrics["intent"]["recall"]
self.assertGreaterEqual(
classification_precision, INTENT_CLASSIFICATION_THRESHOLD,
"Intent classification precision is too low (%.3f) for intent "
"'%s'" % (classification_precision, intent_name))
self.assertGreaterEqual(
classification_recall, INTENT_CLASSIFICATION_THRESHOLD,
"Intent classification recall is too low (%.3f) for intent "
"'%s'" % (classification_recall, intent_name))
for slot_name, slot_metrics in iteritems(intent_metrics["slots"]):
precision = slot_metrics["precision"]
recall = slot_metrics["recall"]
self.assertGreaterEqual(
precision, SLOT_FILLING_THRESHOLD,
"Slot precision is too low (%.3f) for slot '%s' of intent "
"'%s'" % (precision, slot_name, intent_name))
self.assertGreaterEqual(
recall, SLOT_FILLING_THRESHOLD,
"Slot recall is too low (%.3f) for slot '%s' of intent "
"'%s'" % (recall, slot_name, intent_name))
def json(self):
intent_datasets_json = {d.intent_name: d.json
for d in self.intent_datasets}
intents = {
intent_name: {
"utterances": dataset_json["utterances"]
}
for intent_name, dataset_json in iteritems(intent_datasets_json)
}
ents = deepcopy(self.entities)
ents_values = dict()
for entity_name, entity in iteritems(self.entities):
ents_values[entity_name] = set(a.value for a in entity.utterances)
if entity.use_synonyms:
ents_values[entity_name].update(
set(t for s in entity.utterances for t in s.synonyms))
for dataset in self.intent_datasets:
for ent_name, ent in iteritems(dataset.entities):
if ent_name not in ents:
ents[ent_name] = ent
elif not is_builtin_entity(ent_name):
for u in ent.utterances:
if u.value not in ents_values:
ents[ent_name].utterances.append(u)
ents = {
entity_name: entity.json
for entity_name, entity in iteritems(ents)
}
return dict(language=self.language, intents=intents, entities=ents)
def fill_template_host(item):
"""
Prepare fields of host with fields of host templates
:param item: field name / values of the host
:type item: dict
:return: None
"""
host = current_app.data.driver.db['host']
ignore_fields = ['_id', '_etag', '_updated', '_created', '_template_fields', '_templates',
'_is_template', 'realm', '_templates_with_services']
fields_not_update = []
for (field_name, field_value) in iteritems(item):
fields_not_update.append(field_name)
item['_template_fields'] = {}
if ('_is_template' not in item or not item['_is_template']) \
and '_templates' in item and item['_templates'] != []:
for host_template in item['_templates']:
hosts = host.find_one({'_id': ObjectId(host_template)})
if hosts is not None:
for (field_name, field_value) in iteritems(hosts):
if field_name not in fields_not_update \
and field_name not in ignore_fields:
item[field_name] = field_value
item['_template_fields'][field_name] = host_template
schema = host_schema()
ignore_schema_fields = ['realm', '_template_fields', '_templates', '_is_template',
'_templates_with_services']
for key in schema['schema']:
if key not in ignore_schema_fields:
if key not in item:
item['_template_fields'][key] = 0
def parse_axes(A_shape, B_shape, conv_axes, dot_axes, mode):
A_ndim, B_ndim = len(A_shape), len(B_shape)
if conv_axes is None:
conv_axes = [list(range(A_ndim)), list(range(A_ndim))]
axes = {'A' : {'conv' : list(conv_axes[0]),
'dot' : list(dot_axes[0]),
'ignore' : [i for i in range(A_ndim)
if i not in conv_axes[0] and i not in dot_axes[0]]},
'B' : {'conv' : list(conv_axes[1]),
'dot' : list(dot_axes[1]),
'ignore' : [i for i in range(B_ndim)
if i not in conv_axes[1] and i not in dot_axes[1]]}}
assert len(axes['A']['dot']) == len(axes['B']['dot'])
assert len(axes['A']['conv']) == len(axes['B']['conv'])
i1 = len(axes['A']['ignore'])
i2 = i1 + len(axes['B']['ignore'])
i3 = i2 + len(axes['A']['conv'])
axes['out'] = {'ignore_A' : list(range(i1)),
'ignore_B' : list(range(i1, i2)),
'conv' : list(range(i2, i3))}
conv_shape = [compute_conv_size(A_shape[i], B_shape[j], mode)
for i, j in zip(axes['A']['conv'], axes['B']['conv'])]
shapes = {'A' : {s : [A_shape[i] for i in ax] for s, ax in iteritems(axes['A'])},
'B' : {s : [B_shape[i] for i in ax] for s, ax in iteritems(axes['B'])}}
shapes['out'] = {'ignore_A' : shapes['A']['ignore'],
'ignore_B' : shapes['B']['ignore'],
'conv' : conv_shape}
return axes, shapes
def _sync_metadata(self, kp):
"""
Syncs all of the metadata keyspaces and their underlying tables and columns. Sets keyspace to be a dict
of all MetaKeyspace in the connection by name:MetaKeyspace
:return:
"""
self.keyspaces = {}
#TODO: Turn off warnings when this occurs
self.session.row_factory = dict_factory
#gets all of the column data for all tables/keyspaces
result = self.session.execute("""SELECT keyspace_name, columnfamily_name, column_name, component_index, index_name,
index_options, index_type, type as cql_type, validator FROM system.schema_columns""")
cols = [ColumnMeta(**row) for row in result]
for i in cols:
#create keyspace if not already exists
if self.keyspaces.get(i.keyspace) is None:
self.keyspaces.update({i.keyspace:KeyspaceMeta(i.keyspace)})
#add table if not already exists
kp = self.keyspaces.get(i.keyspace)
if kp.tables.get(i.table) is None:
kp.tables.update({i.table:TableMeta(i.keyspace, i.table)})
#finally add/overwrite column into table
tb = kp.tables.get(i.table)
tb.columns[i.name] = i
for kp_nm, kp in iteritems(self.keyspaces):
for tbl_nm, tbl in iteritems(kp.tables):
tbl.categorize_columns()
self.session.row_factory = self.panda_factory
def to_dict(self):
"""Returns a json-serializable dict"""
if hasattr(self.tfidf_vectorizer, "vocabulary_"):
# pylint: # pylint: disable=W0212
vocab = {k: int(v) for k, v in
iteritems(self.tfidf_vectorizer.vocabulary_)}
idf_diag = self.tfidf_vectorizer._tfidf._idf_diag.data.tolist()
# pylint: enable=W0212
entity_utterances_to_entity_names = {
k: list(v)
for k, v in iteritems(self.entity_utterances_to_feature_names)
}
else:
vocab = None
idf_diag = None
entity_utterances_to_entity_names = dict()
tfidf_vectorizer = {
'vocab': vocab,
'idf_diag': idf_diag
}
return {
'language_code': self.language,
'tfidf_vectorizer': tfidf_vectorizer,
'best_features': self.best_features,
'entity_utterances_to_feature_names':
entity_utterances_to_entity_names,
'config': self.config.to_dict(),
'unknown_words_replacement_string':
self.unknown_words_replacement_string
}
def __init__(self, d):
for k, v in iteritems(d):
setattr(self, k, v)
self.properties = properties.Properties()
for k, v in iteritems(d['properties']):
self.properties.setProperty(k, v[0], v[1])
self.who = d['author']
def doRequest(self):
# create a new session if it doesn't exist
self.session = getSession()
requestkwargs = {
'method': self.method,
'url': self.url
}
for param in self.requestsParams:
value = getattr(self, param, None)
if value is not None:
requestkwargs[param] = value
log = self.addLog('log')
# known methods already tested in __init__
log.addHeader('Performing %s request to %s\n' %
(self.method, self.url))
if self.params:
log.addHeader('Parameters:\n')
params = requestkwargs.get("params", {})
if params:
params = sorted(iteritems(params), key=lambda x: x[0])
requestkwargs['params'] = params
for k, v in params:
log.addHeader('\t%s: %s\n' % (k, v))
data = requestkwargs.get("data", None)
if data:
log.addHeader('Data:\n')
if isinstance(data, dict):
for k, v in iteritems(data):
log.addHeader('\t%s: %s\n' % (k, v))
else:
log.addHeader('\t%s\n' % data)
try:
r = yield self.session.request(**requestkwargs)
except requests.exceptions.ConnectionError as e:
log.addStderr(
'An exception occurred while performing the request: %s' % e)
self.finished(FAILURE)
return
if r.history:
log.addStdout('\nRedirected %d times:\n\n' % len(r.history))
for rr in r.history:
self.log_response(rr)
log.addStdout('=' * 60 + '\n')
self.log_response(r)
log.finish()
self.descriptionDone = ["Status code: %d" % r.status_code]
if (r.status_code < 400):
self.finished(SUCCESS)
else:
self.finished(FAILURE)
def print_model_params(self):
self.pub("###")
self.pub("####### RecNet - Recurrent Neural Network Framework ########")
self.pub("###")
date_time = str(datetime.datetime.today())
self.pub("# Start Datetime: "+ date_time )
self.pub("###")
self.pub("# Basic Informations")
for kk, pp in iteritems(self.prm.basic):
str_obj = str(kk) + ": "+ str(pp)
self.pub(str_obj)
self.pub("###")
self.pub("# Data Information")
for kk, pp in iteritems(self.prm.data):
str_obj = str(kk) + ": " + str(pp)
self.pub(str_obj)
self.pub("###")
self.pub("# Network Structure")
for kk, pp in iteritems(self.prm.struct):
str_obj = str(kk) + ": "+ str(pp)
self.pub(str_obj)
self.pub("###")
self.pub("# Optimization Parameters")
for kk, pp in iteritems(self.prm.optimize):
str_obj = str(kk) + ": "+ str(pp)
self.pub(str_obj)
self.pub("###")
def __init__(self, private={}, public={}, private_seeds={}):
# It is possible to distinguish between private and public seeds
# based on the string content. Consider modifying this function
# to take merely one dict of seeds. Trees should still be stored
# separately.
self.trees = {}
self.private_trees = {}
self.public_trees = {}
def treegen(value, entropy=False):
if entropy:
# this method also takes a netcode parameter, but we don't care
# what network pycoin thinks this node is, because we only use it
# for key derivation.
return BIP32Node.from_master_secret(unhexlify(value))
else:
# this method will infer a network from the header bytes. We
# don't care right now for the same reason as above, but we will
# if Gem's API stops returning 'xpub' as the pubkey header bytes
# because if pycoin doesn't recognize a header it will error.
return BIP32Node.from_hwif(value)
for name, seed in iteritems(private):
tree = treegen(seed)
self.private_trees[name] = self.trees[name] = tree
for name, seed in iteritems(private_seeds):
tree = treegen(seed, True)
self.private_trees[name] = self.trees[name] = tree
for name, seed in iteritems(public):
tree = BIP32Node.from_hwif(seed)
self.public_trees[name] = self.trees[name] = tree
def total_issues(self):
"""Number of issue entries for this arch."""
total = 0
for issue_status, issue_types in iteritems(self.issues):
for issue_type, ies in iteritems(issue_types):
total += len(ies)
return total
def __init__(self,
fhandle=None,
action=None,
initial_profiles=None,
**session_args):
super(GrrRekallSession, self).__init__(
cache_dir=config.CONFIG["Client.rekall_profile_cache_path"])
self.action = action
# Just hard code the initial repository manager. Note this can be
# overwritten later if needed.
self._repository_managers = [(None,
RekallCachingIOManager(
initial_profiles=initial_profiles,
session=self))]
# Apply default configuration options to the session state, unless
# explicitly overridden by the session_args.
with self.state:
for k, v in iteritems(session_args):
self.state.Set(k, v)
for name, options in iteritems(rekall_config.OPTIONS.args):
# We don't want to override configuration options passed via
# **session_args.
if name not in session_args:
self.state.Set(name, options.get("default"))
# Ensure the action's Progress() method is called when Rekall reports
# progress.
self.proc = psutil.Process()
self.memory_quota = config.CONFIG["Client.rss_max"] * 1024 * 1024
self.progress.Register(id(self), lambda *_, **__: self._CheckQuota())
def _get_outgoing_weights(self, tag):
return [((first, second), w) for (first, second), w
in iteritems(self.crf_model.transition_features_)
if first == tag]
score = 0
for i in range(len(sentence)):
if i == 0:
# beginning word
score += np.log(bigram_probs[start_idx, sentence[i]])
else:
# middle word
score += np.log(bigram_probs[sentence[i - 1], sentence[i]])
# final word
score += np.log(bigram_probs[sentence[-1], end_idx])
# normalize the score
return score / (len(sentence) + 1)
# a function to map word indexes back to real words
idx2word = dict((v, k) for k, v in iteritems(word2idx))
def get_words(sentence):
return ' '.join(idx2word[i] for i in sentence)
# when we sample a fake sentence, we want to ensure not to sample
# start token or end token
sample_probs = np.ones(V)
sample_probs[start_idx] = 0
sample_probs[end_idx] = 0
sample_probs /= sample_probs.sum()
# test our model on real and fake sentences
while True:
# real sentence
real_idx = np.random.choice(len(sentences))
def db_model_factory(Base, model, all_models):
def get_or_create_association_table(model1_name, model2_name):
_association_table_name = calculate_association_table_name(
model1_name, model2_name
)
logger.debug(
"Creating/getting ManyToMany relationship table: %s",
_association_table_name,
)
if _association_table_name in globals():
return globals()[_association_table_name]
# create an association table
_association_table = Table(
_association_table_name,
Base.metadata,
Column(
"%s_pk" % model1_name.lower(), String, ForeignKey("%s.pk" % model1_name)
),
Column(
"%s_pk" % model2_name.lower(), String, ForeignKey("%s.pk" % model2_name)
),
)
# track it in our globals
set_global(_association_table_name, _association_table)
return _association_table
logger.debug("-----")
logger.debug("Generating model: %s", model.name)
model_fields = {"__tablename__": model.name, "pk": Column(String, primary_key=True)}
# populate all of the relevant additional relationships for this model
for field_name, rel in iteritems(model.additional_rels):
kwargs = {}
if rel.get("back_populates", None) is not None:
kwargs["back_populates"] = rel["back_populates"]
if rel.get("secondary", None) is not None:
kwargs["secondary"] = get_or_create_association_table(*rel["secondary"])
logger.debug(
"Creating additional relationship %s.%s -> %s (%s)",
model.name,
field_name,
rel["to_model"],
kwargs,
)
model_fields[field_name] = relationship(rel["to_model"], **kwargs)
# now populate all of the standard fields
for field_name in model.field_names:
field = model.fields[field_name]
if field.field_type in SQLALCHEMY_FIELD_MAPPER:
# if it's a simple field
model_fields[field.name] = Column(
field.name, SQLALCHEMY_FIELD_MAPPER[field.field_type]
)
elif field.field_type in all_models:
# if it's a foreign key reference
if isinstance(field, StatikForeignKeyField):
model_fields["%s_id" % field.name] = Column(
"%s_id" % field.name, ForeignKey("%s.pk" % field.field_type)
)
# if it's a self-referencing foreign key
if field.field_type == model.name:
back_populates = field.back_populates or "children"
model_fields[back_populates] = relationship(
field.field_type,
backref=backref(field_name, remote_side=[model_fields["pk"]]),
)
else:
kwargs = {}
if field.back_populates is not None:
kwargs["back_populates"] = field.back_populates
logger.debug(
"Field %s.%s has back-populates field name: %s",
model.name,
field_name,
field.back_populates,
)
else:
logger.debug(
"No back-populates field name for %s.%s",
model.name,
field_name,
)
foreign_key = model_fields.get("%s_id" % field.name, None)
model_fields[field.name] = relationship(
field.field_type, foreign_keys=[foreign_key], **kwargs
)
elif isinstance(field, StatikManyToManyField):
association_table = get_or_create_association_table(
model.name, field.field_type
)
kwargs = {"secondary": association_table}
if field.back_populates is not None:
kwargs["back_populates"] = field.back_populates
logger.debug(
"Creating model ManyToMany field %s.%s -> %s (%s)",
model.name,
field.name,
field.field_type,
kwargs,
)
model_fields[field.name] = relationship(field.field_type, **kwargs)
else:
raise InvalidFieldTypeError(model.name, field.name)
Model = type(str(model.name), (Base,), model_fields)
logger.debug("Model %s fields = %s", model.name, model_fields)
# add the model class reference to the global scope
set_global(model.name, Model)
return Model
def test_taker_init(createcmtdata, schedule, highfee, toomuchcoins, minmakers,
notauthed, ignored, nocommit):
#these tests do not trigger utxo_retries
oldtakerutxoretries = jm_single().config.get("POLICY", "taker_utxo_retries")
oldtakerutxoamtpercent = jm_single().config.get("POLICY", "taker_utxo_amtpercent")
jm_single().config.set("POLICY", "taker_utxo_retries", "20")
def clean_up():
jm_single().config.set("POLICY", "minimum_makers", oldminmakers)
jm_single().config.set("POLICY", "taker_utxo_retries", oldtakerutxoretries)
jm_single().config.set("POLICY", "taker_utxo_amtpercent", oldtakerutxoamtpercent)
oldminmakers = jm_single().config.get("POLICY", "minimum_makers")
jm_single().config.set("POLICY", "minimum_makers", str(minmakers))
taker = get_taker(schedule)
orderbook = copy.deepcopy(t_orderbook)
if highfee:
for o in orderbook:
#trigger high-fee warning; but reset in next step
o['cjfee'] = '1.0'
if ignored:
taker.ignored_makers = ignored
if nocommit:
jm_single().config.set("POLICY", "taker_utxo_amtpercent", nocommit)
if schedule[0][1] == 0.2:
#triggers calc-ing amount based on a fraction
jm_single().mincjamount = 50000000 #bigger than 40m = 0.2 * 200m
res = taker.initialize(orderbook)
assert res[0]
assert res[1] == jm_single().mincjamount
return clean_up()
res = taker.initialize(orderbook)
if toomuchcoins or ignored:
assert not res[0]
return clean_up()
if nocommit:
print(str(res))
assert res[0] == "commitment-failure"
return clean_up()
taker.orderbook = copy.deepcopy(t_chosen_orders) #total_cjfee unaffected, all same
maker_response = copy.deepcopy(t_maker_response)
if notauthed:
#Doctor one of the maker response data fields
maker_response["J659UPUSLLjHJpaB"][1] = "xx" #the auth pub
if schedule[0][1] == 199850000:
#triggers negative change
#makers offer 3000 txfee; we estimate ~ 147*10 + 2*34 + 10=1548 bytes
#times 30k = 46440, so we pay 43440, plus maker fees = 3*0.0002*200000000
#roughly, gives required selected = amt + 163k, hence the above =
#2btc - 150k sats = 199850000 (tweaked because of aggressive coin selection)
#simulate the effect of a maker giving us a lot more utxos
taker.utxos["dummy_for_negative_change"] = ["a", "b", "c", "d", "e"]
with pytest.raises(ValueError) as e_info:
res = taker.receive_utxos(maker_response)
return clean_up()
if schedule[0][1] == 199850001:
#our own change is greater than zero but less than dust
#use the same edge case as for negative change, don't add dummy inputs
#(because we need tx creation to complete), but trigger case by
#bumping dust threshold
jm_single().BITCOIN_DUST_THRESHOLD = 14000
res = taker.receive_utxos(maker_response)
#should have succeeded to build tx
assert res[0]
#change should be none
assert not taker.my_change_addr
return clean_up()
if schedule[0][1] == 199599800:
#need to force negative fees to make this feasible
for k, v in iteritems(taker.orderbook):
v['cjfee'] = '-0.002'
# change_amount = (total_input - self.cjamount -
# self.orderbook[nick]['txfee'] + real_cjfee)
#suppose change amount is 1000 (sub dust), then solve for x;
#given that real_cjfee = -0.002*x
#change = 200000000 - x - 1000 - 0.002*x
#x*1.002 = 1999999000; x = 199599800
res = taker.receive_utxos(maker_response)
assert not res[0]
assert res[1] == "Not enough counterparties responded to fill, giving up"
return clean_up()
if schedule[0][3] == "mxeLuX8PP7qLkcM8uarHmdZyvP1b5e1Ynf":
#to trigger rounding error for sweep (change non-zero),
#modify the total_input via the values in self.input_utxos;
#the amount to trigger a 2 satoshi change is found by trial-error.
#TODO note this test is not adequate, because the code is not;
#the code does not *DO* anything if a condition is unexpected.
taker.input_utxos = copy.deepcopy(t_utxos_by_mixdepth)[0]
for k,v in iteritems(taker.input_utxos):
v["value"] = int(0.999805228 * v["value"])
res = taker.receive_utxos(maker_response)
assert res[0]
return clean_up()
res = taker.receive_utxos(maker_response)
if minmakers != 2:
assert not res[0]
assert res[1] == "Not enough counterparties responded to fill, giving up"
return clean_up()
assert res[0]
#re-calling will trigger "finished" code, since schedule is "complete".
res = taker.initialize(orderbook)
assert not res[0]
#some exception cases: no coinjoin address, no change address:
#donations not yet implemented:
taker.my_cj_addr = None
with pytest.raises(NotImplementedError) as e_info:
taker.prepare_my_bitcoin_data()
with pytest.raises(NotImplementedError) as e_info:
a = taker.coinjoin_address()
taker.wallet.inject_addr_get_failure = True
taker.my_cj_addr = "dummy"
assert not taker.prepare_my_bitcoin_data()
#clean up
return clean_up()
def mut_add(self, xs, ys):
return {k : v.mut_add(xs[k], ys[k])
for k, v in iteritems(self.shape)}
def zeros(self):
return {k : v.zeros() for k, v in iteritems(self.shape)}
def __init__(self, value):
self.shape = {k : vspace(v) for k, v in iteritems(value)}
self.size = sum(s.size for s in self.shape.values())
def get_list_queryset(self, queryset):
lookup_params = dict([
(smart_str(k)[len(FILTER_PREFIX):], v)
for k, v in self.admin_view.params.items()
if smart_str(k).startswith(FILTER_PREFIX) and v != ''
])
for p_key, p_val in iteritems(lookup_params):
if p_val == "False":
lookup_params[p_key] = False
use_distinct = False
# for clean filters
self.admin_view.has_query_param = bool(lookup_params)
self.admin_view.clean_query_url = self.admin_view.get_query_string(
remove=[
k for k in self.request.GET.keys()
if k.startswith(FILTER_PREFIX)
])
# Normalize the types of keys
if not self.free_query_filter:
for key, value in lookup_params.items():
if not self.lookup_allowed(key, value):
raise SuspiciousOperation("Filtering by %s not allowed" %
key)
self.filter_specs = []
if self.list_filter:
for list_filter in self.list_filter:
if callable(list_filter):
# This is simply a custom list filter class.
spec = list_filter(self.request, lookup_params, self.model,
self)
else:
field_path = None
field_parts = []
if isinstance(list_filter, (tuple, list)):
# This is a custom FieldListFilter class for a given field.
field, field_list_filter_class = list_filter
else:
# This is simply a field name, so use the default
# FieldListFilter class that has been registered for
# the type of the given field.
field, field_list_filter_class = list_filter, filter_manager.create
if not isinstance(field, models.Field):
field_path = field
field_parts = get_fields_from_path(
self.model, field_path)
field = field_parts[-1]
spec = field_list_filter_class(field,
self.request,
lookup_params,
self.model,
self.admin_view,
field_path=field_path)
if len(field_parts) > 1:
# Add related model name to title
spec.title = "%s %s" % (field_parts[-2].name,
spec.title)
# Check if we need to use distinct()
use_distinct = (use_distinct or lookup_needs_distinct(
self.opts, field_path))
if spec and spec.has_output():
try:
new_qs = spec.do_filte(queryset)
except ValidationError as e:
new_qs = None
self.admin_view.message_user(
_("<b>Filtering error:</b> %s") % e.messages[0],
'error')
if new_qs is not None:
queryset = new_qs
self.filter_specs.append(spec)
self.has_filters = bool(self.filter_specs)
self.admin_view.filter_specs = self.filter_specs
obj = filter(lambda f: f.is_used, self.filter_specs)
if six.PY3:
obj = list(obj)
self.admin_view.used_filter_num = len(obj)
try:
for key, value in lookup_params.items():
use_distinct = (use_distinct
or lookup_needs_distinct(self.opts, key))
except FieldDoesNotExist as e:
raise IncorrectLookupParameters(e)
try:
# fix a bug by david: In demo, quick filter by IDC Name() cannot be used.
if isinstance(queryset, models.query.QuerySet) and lookup_params:
new_lookup_parames = dict()
for k, v in lookup_params.iteritems():
list_v = v.split(',')
if len(list_v) > 0:
new_lookup_parames.update({k: list_v})
else:
new_lookup_parames.update({k: v})
queryset = queryset.filter(**new_lookup_parames)
except (SuspiciousOperation, ImproperlyConfigured):
raise
except Exception as e:
raise IncorrectLookupParameters(e)
else:
if not isinstance(queryset, models.query.QuerySet):
pass
query = self.request.GET.get(SEARCH_VAR, '')
# Apply keyword searches.
def construct_search(field_name):
if field_name.startswith('^'):
return "%s__istartswith" % field_name[1:]
elif field_name.startswith('='):
return "%s__iexact" % field_name[1:]
elif field_name.startswith('@'):
return "%s__search" % field_name[1:]
else:
return "%s__icontains" % field_name
if self.search_fields and query:
orm_lookups = [
construct_search(str(search_field))
for search_field in self.search_fields
]
for bit in query.split():
or_queries = [
models.Q(**{orm_lookup: bit}) for orm_lookup in orm_lookups
]
queryset = queryset.filter(reduce(operator.or_, or_queries))
if not use_distinct:
for search_spec in orm_lookups:
if lookup_needs_distinct(self.opts, search_spec):
use_distinct = True
break
self.admin_view.search_query = query
if use_distinct:
return queryset.distinct()
else:
return queryset
def update_attributes(self, data):
for k, v in iteritems(data):
setattr(self, k, v)
def load_data(self):
mainLog = _make_logger(method_name='QueueConfigMapper.load_data')
with self.lock:
# check if to update
timeNow_timestamp = time.time()
if self.lastUpdate is not None:
last_reload_timestamp = self._get_last_reload_time()
if (last_reload_timestamp is not None
and self.lastUpdate is not None
and datetime.datetime.utcfromtimestamp(
last_reload_timestamp) < self.lastUpdate
and timeNow_timestamp - last_reload_timestamp <
self.updateInterval):
return
# start
with self.lock:
# update timesatmp of last reload, lock with check interval
got_timesatmp_update_lock = self.dbProxy.get_process_lock(
'qconf_reload', 'qconf_universal', self.updateInterval)
if got_timesatmp_update_lock:
retVal = self._update_last_reload_time()
if retVal:
mainLog.debug('updated last reload timestamp')
else:
mainLog.warning(
'failed to update last reload timestamp. Skipped')
else:
mainLog.debug(
'did not get qconf_reload timestamp lock. Skipped to update last reload timestamp'
)
# init
newQueueConfig = dict()
localTemplatesDict = dict()
remoteTemplatesDict = dict()
finalTemplatesDict = dict()
localQueuesDict = dict()
remoteQueuesDict = dict()
dynamicQueuesDict = dict()
allQueuesNameList = set()
getQueuesDynamic = False
invalidQueueList = set()
pandaQueueDict = PandaQueuesDict()
# get resolver
resolver = self._get_resolver()
if resolver is None:
mainLog.debug('No resolver is configured')
# load config json from cacher (RT & RQ)
queueConfigJson_cacher = self._load_config_from_cache()
if queueConfigJson_cacher is not None:
for queueName, queueDict in iteritems(queueConfigJson_cacher):
if queueDict.get('isTemplateQueue') is True \
or queueName.endswith('_TEMPLATE'):
# is RT
queueDict['isTemplateQueue'] = True
queueDict.pop('templateQueueName', None)
remoteTemplatesDict[queueName] = queueDict
else:
# is RQ
queueDict['isTemplateQueue'] = False
remoteQueuesDict[queueName] = queueDict
# load config from local json file (LT & LQ)
queueConfigJson_local = self._load_config_from_file()
if queueConfigJson_local is not None:
for queueName, queueDict in iteritems(queueConfigJson_local):
if queueDict.get('isTemplateQueue') is True \
or queueName.endswith('_TEMPLATE'):
# is LT
queueDict['isTemplateQueue'] = True
queueDict.pop('templateQueueName', None)
localTemplatesDict[queueName] = queueDict
else:
# is LQ
queueDict['isTemplateQueue'] = False
localQueuesDict[queueName] = queueDict
else:
mainLog.warning(
'Failed to load config from local json file. Skipped')
# fill in final template (FT)
finalTemplatesDict.update(remoteTemplatesDict)
finalTemplatesDict.update(localTemplatesDict)
finalTemplatesDict.pop(None, None)
# remove queues with invalid templateQueueName
for acr, queuesDict in [('RQ', remoteQueuesDict),
('LQ', localQueuesDict)]:
for queueName, queueDict in iteritems(queuesDict.copy()):
templateQueueName = queueDict.get('templateQueueName')
if templateQueueName is not None \
and templateQueueName not in finalTemplatesDict:
del queuesDict[queueName]
mainLog.warning(
'Invalid templateQueueName "{0}" for {1} ({2}). Skipped'
.format(templateQueueName, queueName, acr))
# get queue names from resolver and fill in dynamic queue (DQ)
if resolver is not None \
and 'DYNAMIC' in harvester_config.qconf.queueList:
getQueuesDynamic = True
dynamicQueuesNameList = resolver.get_all_queue_names()
for queueName in dynamicQueuesNameList.copy():
queueDict = dict()
# template and default template via workflow
templateQueueName = None
resolver_harvester_template = None
if resolver is not None:
resolver_harvester_template = resolver.get_harvester_template(
queueName)
resolver_type, resolver_workflow = resolver.get_type_workflow(
queueName)
if resolver_harvester_template:
templateQueueName = resolver_harvester_template
elif not (resolver_type is None
or resolver_workflow is None):
templateQueueName = '{pq_type}.{workflow}'.format(
pq_type=resolver_type, workflow=resolver_workflow)
else:
templateQueueName = harvester_config.qconf.defaultTemplateQueueName
if templateQueueName not in finalTemplatesDict:
# remove queues with invalid templateQueueName
dynamicQueuesNameList.discard(queueName)
mainLog.warning(
'Invalid templateQueueName "{0}" for {1} (DQ). Skipped'
.format(templateQueueName, queueName))
continue
# parameters
resolver_harvester_params = resolver.get_harvester_params(
queueName)
for key, val in iteritems(resolver_harvester_params):
if key in self.dynamic_queue_generic_attrs:
queueDict[key] = val
# fill in dynamic queue configs
queueDict['templateQueueName'] = templateQueueName
queueDict['isTemplateQueue'] = False
dynamicQueuesDict[queueName] = queueDict
# fill in all queue name list (names of RQ + DQ + LQ)
allQueuesNameList |= set(remoteQueuesDict)
allQueuesNameList |= set(dynamicQueuesDict)
allQueuesNameList |= set(localQueuesDict)
allQueuesNameList.discard(None)
# set attributes
for queueName in allQueuesNameList:
# sources or queues and templates
queueSourceList = []
templateSourceList = []
# prepare templateQueueName
templateQueueName = None
for queuesDict in [
remoteQueuesDict, dynamicQueuesDict, localQueuesDict
]:
if queueName not in queuesDict:
continue
tmp_queueDict = queuesDict[queueName]
tmp_templateQueueName = tmp_queueDict.get(
'templateQueueName')
if tmp_templateQueueName is not None:
templateQueueName = tmp_templateQueueName
# prepare queueDict
queueDict = dict()
if templateQueueName in finalTemplatesDict:
queueDict.update(
copy.deepcopy(finalTemplatesDict[templateQueueName]))
for acr, templatesDict in [('RT', remoteTemplatesDict),
('LT', localTemplatesDict)]:
if templateQueueName in templatesDict:
templateSourceList.append(acr)
# update queueDict
for acr, queuesDict in [('RQ', remoteQueuesDict),
('DQ', dynamicQueuesDict),
('LQ', localQueuesDict)]:
if queueName not in queuesDict:
continue
queueSourceList.append(acr)
tmp_queueDict = queuesDict[queueName]
for key, val in iteritems(tmp_queueDict):
val = copy.deepcopy(val)
if key in self.updatable_plugin_attrs \
and isinstance(queueDict.get(key), dict) \
and isinstance(val, dict):
# update plugin parameters instead of overwriting whole plugin section
queueDict[key].update(val)
else:
queueDict[key] = val
# record sources of the queue config and its templates in log
if templateQueueName:
mainLog.debug(
('queue {queueName} comes from {queueSource} '
'(with template {templateName} '
'from {templateSource})').format(
queueName=queueName,
templateName=templateQueueName,
queueSource=','.join(queueSourceList),
templateSource=','.join(templateSourceList)))
else:
mainLog.debug(
'queue {queueName} comes from {queueSource}'.format(
queueName=queueName,
queueSource=','.join(queueSourceList)))
# prepare queueConfig
if queueName in newQueueConfig:
queueConfig = newQueueConfig[queueName]
else:
queueConfig = QueueConfig(queueName)
# queueName = siteName/resourceType
queueConfig.siteName = queueConfig.queueName.split('/')[0]
if queueConfig.siteName != queueConfig.queueName:
queueConfig.resourceType = queueConfig.queueName.split(
'/')[-1]
# get common attributes
commonAttrDict = dict()
if isinstance(queueDict.get('common'), dict):
commonAttrDict = queueDict.get('common')
# according to queueDict
for key, val in iteritems(queueDict):
if isinstance(val,
dict) and 'module' in val and 'name' in val:
# plugin attributes
val = copy.deepcopy(val)
# fill in common attributes for all plugins
for c_key, c_val in iteritems(commonAttrDict):
if c_key not in val and c_key not in ('module',
'name'):
val[c_key] = c_val
# check module and class name
try:
_t3mP_1Mp0R7_mO6U1e__ = importlib.import_module(
val['module'])
_t3mP_1Mp0R7_N4m3__ = getattr(
_t3mP_1Mp0R7_mO6U1e__, val['name'])
except Exception as _e:
invalidQueueList.add(queueConfig.queueName)
mainLog.error(
'Module or class not found. Omitted {0} in queue config ({1})'
.format(queueConfig.queueName, _e))
continue
else:
del _t3mP_1Mp0R7_mO6U1e__
del _t3mP_1Mp0R7_N4m3__
# fill in siteName and queueName
if 'siteName' not in val:
val['siteName'] = queueConfig.siteName
if 'queueName' not in val:
val['queueName'] = queueConfig.queueName
# middleware
if 'middleware' in val and val[
'middleware'] in queueDict:
# keep original config
val['original_config'] = copy.deepcopy(val)
# overwrite with middleware config
for m_key, m_val in iteritems(
queueDict[val['middleware']]):
val[m_key] = m_val
setattr(queueConfig, key, val)
# delete isTemplateQueue attribute
try:
if getattr(queueConfig, 'isTemplateQueue'):
mainLog.error(
'Internal error: isTemplateQueue is True. Omitted {0} in queue config'
.format(queueConfig.queueName))
invalidQueueList.add(queueConfig.queueName)
else:
delattr(queueConfig, 'isTemplateQueue')
except AttributeError as _e:
mainLog.error(
'Internal error with attr "isTemplateQueue". Omitted {0} in queue config ({1})'
.format(queueConfig.queueName, _e))
invalidQueueList.add(queueConfig.queueName)
# get Panda Queue Name
if resolver is not None:
queueConfig.pandaQueueName = resolver.get_panda_queue_name(
queueConfig.siteName)
# additional criteria for getJob
if queueConfig.getJobCriteria is not None:
tmpCriteria = dict()
for tmpItem in queueConfig.getJobCriteria.split(','):
tmpKey, tmpVal = tmpItem.split('=')
tmpCriteria[tmpKey] = tmpVal
if len(tmpCriteria) == 0:
queueConfig.getJobCriteria = None
else:
queueConfig.getJobCriteria = tmpCriteria
# nullify job attributes if NoJob mapType
if queueConfig.mapType == WorkSpec.MT_NoJob:
for attName in [
'nQueueLimitJob', 'nQueueLimitJobRatio',
'nQueueLimitJobMax', 'nQueueLimitJobMin'
]:
setattr(queueConfig, attName, None)
# heartbeat suppression
if queueConfig.truePilot and queueConfig.noHeartbeat == '':
queueConfig.noHeartbeat = 'running,transferring,finished,failed'
# set unique name
queueConfig.set_unique_name()
# put into new queue configs
newQueueConfig[queueName] = queueConfig
# Check existence of mandatory attributes
if queueName in newQueueConfig:
queueConfig = newQueueConfig[queueName]
missing_attr_list = []
for _attr in self.mandatory_attrs:
if not hasattr(queueConfig, _attr):
invalidQueueList.add(queueConfig.queueName)
missing_attr_list.append(_attr)
if missing_attr_list:
mainLog.error(
'Missing mandatory attributes {0} . Omitted {1} in queue config'
.format(','.join(missing_attr_list),
queueConfig.queueName))
# delete invalid queues
for invalidQueueName in invalidQueueList:
if invalidQueueName in newQueueConfig:
del newQueueConfig[invalidQueueName]
# auto blacklisting
autoBlacklist = False
if resolver is not None and hasattr(harvester_config.qconf, 'autoBlacklist') and \
harvester_config.qconf.autoBlacklist:
autoBlacklist = True
# get queue dumps
queueConfigDumps = self.dbProxy.get_queue_config_dumps()
# get active queues
activeQueues = dict()
for queueName, queueConfig in iteritems(newQueueConfig):
# get status
if queueConfig.queueStatus is None and autoBlacklist:
queueConfig.queueStatus = resolver.get_queue_status(
queueName)
# get dynamic information
if 'DYNAMIC' in harvester_config.qconf.queueList:
# UPS queue
if resolver is not None and resolver.is_ups_queue(
queueName):
queueConfig.runMode = 'slave'
queueConfig.mapType = 'NoJob'
# set online if undefined
if queueConfig.queueStatus is None:
queueConfig.queueStatus = 'online'
queueConfig.queueStatus = queueConfig.queueStatus.lower()
# look for configID
dumpSpec = QueueConfigDumpSpec()
dumpSpec.queueName = queueName
dumpSpec.set_data(vars(queueConfig))
if dumpSpec.dumpUniqueName in queueConfigDumps:
dumpSpec = queueConfigDumps[dumpSpec.dumpUniqueName]
else:
# add dump
dumpSpec.creationTime = datetime.datetime.utcnow()
dumpSpec.configID = self.dbProxy.get_next_seq_number(
'SEQ_configID')
tmpStat = self.dbProxy.add_queue_config_dump(dumpSpec)
if not tmpStat:
dumpSpec.configID = self.dbProxy.get_config_id_dump(
dumpSpec)
if dumpSpec.configID is None:
mainLog.error(
'failed to get configID for {0}'.format(
dumpSpec.dumpUniqueName))
continue
queueConfigDumps[dumpSpec.dumpUniqueName] = dumpSpec
queueConfig.configID = dumpSpec.configID
# ignore offline
if queueConfig.queueStatus == 'offline':
continue
# filter for pilot version
if hasattr(harvester_config.qconf, 'pilotVersion') and \
pandaQueueDict[queueConfig.siteName].get('pilot_version') != str(harvester_config.qconf.pilotVersion):
continue
if 'ALL' not in harvester_config.qconf.queueList and \
'DYNAMIC' not in harvester_config.qconf.queueList and \
queueName not in harvester_config.qconf.queueList:
continue
activeQueues[queueName] = queueConfig
self.queueConfig = newQueueConfig
self.activeQueues = activeQueues
newQueueConfigWithID = dict()
for dumpSpec in queueConfigDumps.values():
queueConfig = QueueConfig(dumpSpec.queueName)
queueConfig.update_attributes(dumpSpec.data)
queueConfig.configID = dumpSpec.configID
newQueueConfigWithID[dumpSpec.configID] = queueConfig
self.queueConfigWithID = newQueueConfigWithID
self.lastUpdate = datetime.datetime.utcnow()
# update database
if self.toUpdateDB:
self.dbProxy.fill_panda_queue_table(self.activeQueues.keys(), self)
mainLog.debug('updated to DB')
# done
mainLog.debug('done')
def assertInstanceEqual(self, expected, inst):
for field_name, field_value in iteritems(expected):
self.assertEqual(field_value, getattr(inst, field_name))
def set_def_id(self, def_ids):
# Sort AMQP definitions by their names.
def_ids = sorted(iteritems(def_ids), key=itemgetter(1))
for id, name in def_ids:
self.fields['def_id'].choices.append([id, name])
def __init__(self, **kwargs):
self.currencies = []
for currency, fee in iteritems(kwargs):
self.currencies.append(currency)
setattr(self, currency, Decimal(fee))
def main():
parser = create_argument_parser()
args = parser.parse_args()
if not args.config:
args.print_usage()
return
# Configuration
with open(args.config) as config:
options = config_to_options(config.read())
# Override settings with command line options
if args.max_results:
options['settings']['max_results'] = args.max_results
if args.quantiles:
try:
quantiles = [float(s.strip()) for s in quantiles_raw.split(',')]
options['settings']['quantiles'] = quantiles
except (
AttributeError,
ValueError,
):
print("Invalid value for --quantiles: " + args.quantiles)
args.print_usage()
return
quantiles = options['settings']['quantiles']
if args.charts_from:
options['settings']['charts_from'] = args.charts_from
if args.charts_to:
options['settings']['charts_to'] = args.charts_to
output_format = args.format.lower() if args.format else 'csv'
throughput_window_end = parse_relative_date(
args.throughput_window_end
) if args.throughput_window_end else datetime.date.today()
throughput_window_days = args.throughput_window
# Query JIRA
try:
jira = get_jira_client(options['connection'])
q = CycleTimeQueries(jira, **options['settings'])
print("Fetching issues (this could take some time)")
cycle_data = pd.DataFrame()
size_data = pd.DataFrame()
cycle_data, size_data = q.cycle_data(verbose=args.verbose,
result_cycle=cycle_data,
result_size=size_data)
if args.points:
print("Working out size changes of issues over time")
df_size_history = q.size_history(size_data)
df_size_history.to_csv(r'size_history.csv',
sep='\t') # Save to file.
else:
df_size_history = None
except JIRAError as e:
eprint(e)
return 1
#cfd_data = q.cfd(cycle_data)
cfd_data = q.cfd(cycle_data,
size_history=df_size_history,
pointscolumn=args.points,
stacked=False)
cfd_data_stackable = q.cfd(cycle_data,
size_history=df_size_history,
pointscolumn=args.points,
stacked=True)
scatter_data = q.scatterplot(cycle_data)
histogram_data = q.histogram(cycle_data)
percentile_data = q.percentiles(cycle_data, percentiles=quantiles)
#daily_throughput_data = q.throughput_data(
# cycle_data[cycle_data['completed_timestamp'] >= (throughput_window_end - datetime.timedelta(days=throughput_window_days))],
#)
if args.points:
daily_throughput_data = q.throughput_data(
cycle_data[cycle_data['completed_timestamp'] >= (
throughput_window_end -
datetime.timedelta(days=throughput_window_days))],
pointscolumn=args.points)
else:
daily_throughput_data = q.throughput_data(
cycle_data[cycle_data['completed_timestamp'] >= (
throughput_window_end -
datetime.timedelta(days=throughput_window_days))], )
if options['settings']['statusmapping']:
for key, state in iteritems(
options['settings']['statusmapping']
): # # use .items() for python 3 and iteritems() for python 2
if state == 'complete':
done_column = key
if state == 'final':
final_column = key
if state == 'committed':
committed_column = key
if state == 'backlog':
backlog_column = key
else:
backlog_column = args.backlog_column or cfd_data.columns[0]
committed_column = args.committed_column or cfd_data.columns[1]
final_column = args.final_column or cfd_data.columns[-2]
done_column = args.done_column or cfd_data.columns[-1]
cycle_names = [s['name'] for s in q.settings['cycle']]
field_names = sorted(options['settings']['fields'].keys())
query_attribute_names = [q.settings['query_attribute']
] if q.settings['query_attribute'] else []
# Burnup forecast
target = args.charts_burnup_forecast_target or None
trials = args.charts_burnup_forecast_trials or 1000
# TODO - parameterise historical throughput
#try:
if args.points:
burnup_forecast_data = q.burnup_forecast(cfd_data,
daily_throughput_data,
trials=trials,
target=target,
backlog_column=backlog_column,
done_column=done_column,
percentiles=quantiles,
sized='Sized')
else:
burnup_forecast_data = q.burnup_forecast(cfd_data,
daily_throughput_data,
trials=trials,
target=target,
backlog_column=backlog_column,
done_column=done_column,
percentiles=quantiles,
sized='')
#except Exception as e:
# print("Warning: Failed to calculate burnup forecast")
# burnup_forecast_data = None
# Write files
if args.output:
print("Writing cycle data to", args.output)
header = ['ID', 'Link', 'Name'] + cycle_names + [
'Type', 'Status', 'Resolution'
] + field_names + query_attribute_names
columns = ['key', 'url', 'summary'] + cycle_names + [
'issue_type', 'status', 'resolution'
] + field_names + query_attribute_names
if output_format == 'json':
values = [header] + [
map(to_json_string, row)
for row in cycle_data[columns].values.tolist()
]
with open(args.output, 'w') as out:
out.write(json.dumps(values))
elif output_format == 'xlsx':
cycle_data.to_excel(args.output,
'Cycle data',
columns=columns,
header=header,
index=False)
else:
cycle_data.to_csv(args.output,
columns=columns,
header=header,
date_format='%Y-%m-%d',
index=False,
sep='\t')
if args.records:
if output_format == 'json':
print("Writing cycle data as JSON records")
cycle_data.to_json(args.records,
date_format='iso',
orient='records')
else:
print(
"Warning: Ignoring cycle data as JSON records. Use --format json"
)
if args.size_history:
print("Writing issue size history data to", args.size_history)
if output_format == 'json':
size_data.to_json(args.size_history, date_format='iso')
elif output_format == 'xlsx':
size_data.to_excel(args.size_history, 'SIZES')
else:
size_data.to_csv(args.size_history,
columns=['key', 'fromDate', 'toDate', 'size'],
sep='\t',
date_format='%Y-%m-%d')
if args.cfd:
print("Writing Cumulative Flow Diagram data to", args.cfd)
if output_format == 'json':
cfd_data.to_json(args.cfd, date_format='iso')
elif output_format == 'xlsx':
cfd_data.to_excel(args.cfd, 'CFD')
else:
cfd_data.to_csv(args.cfd, sep='\t')
if args.scatterplot:
print("Writing cycle time scatter plot data to", args.scatterplot)
if output_format == 'json':
scatter_data.to_json(args.scatterplot, date_format='iso')
elif output_format == 'xlsx':
scatter_data.to_excel(args.scatterplot, 'Scatter', index=False)
else:
scatter_data.to_csv(args.scatterplot, index=False, sep='\t')
if args.percentiles:
print("Writing cycle time percentiles", args.percentiles)
if output_format == 'json':
percentile_data.to_json(args.percentiles, date_format='iso')
elif output_format == 'xlsx':
percentile_data.to_frame(name='percentiles').to_excel(
args.percentiles, 'Percentiles', header=True)
else:
percentile_data.to_csv(args.percentiles, header=True, sep='\t')
if args.histogram:
print("Writing cycle time histogram data to", args.histogram)
if output_format == 'json':
histogram_data.to_json(args.histogram, date_format='iso')
elif output_format == 'xlsx':
histogram_data.to_frame(name='histogram').to_excel(args.histogram,
'Histogram',
header=True)
else:
histogram_data.to_csv(args.histogram, header=True, sep='\t')
if args.throughput:
print("Writing throughput data to", args.throughput)
if output_format == 'json':
daily_throughput_data.to_json(args.throughput, date_format='iso')
elif output_format == 'xlsx':
daily_throughput_data.to_excel(args.throughput,
'Throughput',
header=True)
else:
daily_throughput_data.to_csv(args.throughput,
header=True,
sep='\t')
if args.burnup_forecast and burnup_forecast_data is not None:
print("Writing burnup forecast data to", args.burnup_forecast)
if output_format == 'json':
burnup_forecast_data.to_json(args.burnup_forecast,
date_format='iso')
elif output_format == 'xlsx':
burnup_forecast_data.to_excel(args.burnup_forecast_data,
'Forecast',
header=True)
else:
burnup_forecast_data.to_csv(args.burnup_forecast_data,
header=True,
sep='\t')
# Output charts (if we have the right things installed)
if charting.HAVE_CHARTING:
charts_from = parse_relative_date(
options['settings']['charts_from']
) if options['settings']['charts_from'] is not None else None
charts_to = parse_relative_date(
options['settings']['charts_to']
) if options['settings']['charts_to'] is not None else None
cycle_data_sliced = cycle_data
if charts_from is not None:
cycle_data_sliced = cycle_data[
cycle_data['completed_timestamp'] >= charts_from]
if charts_to is not None:
cycle_data_sliced = cycle_data[
cycle_data['completed_timestamp'] <= charts_to]
cfd_data_sliced = cfd_data[slice(charts_from, charts_to)]
cfd_data_stackable_sliced = cfd_data_stackable[slice(
charts_from, charts_to)]
charting.set_context()
if args.charts_scatterplot:
print("Drawing scatterplot in", args.charts_scatterplot)
charting.set_style('darkgrid')
try:
ax = charting.cycle_time_scatterplot(
cycle_data_sliced,
percentiles=quantiles,
title=args.charts_scatterplot_title)
except charting.UnchartableData as e:
print("** WARNING: Did not draw chart:", e)
else:
fig = ax.get_figure()
fig.savefig(args.charts_scatterplot,
bbox_inches='tight',
dpi=300)
if args.charts_histogram:
print("Drawing histogram in", args.charts_histogram)
charting.set_style('darkgrid')
try:
ax = charting.cycle_time_histogram(
cycle_data_sliced,
percentiles=quantiles,
title=args.charts_histogram_title)
except charting.UnchartableData as e:
print("** WARNING: Did not draw chart:", e)
else:
fig = ax.get_figure()
fig.savefig(args.charts_histogram,
bbox_inches='tight',
dpi=300)
if args.charts_cfd:
print("Drawing CFD in", args.charts_cfd)
charting.set_style('whitegrid')
try:
if args.points:
ax = charting.cfd(cfd_data_stackable_sliced,
title=args.charts_cfd_title,
pointscolumn=args.points)
else:
ax = charting.cfd(cfd_data_sliced,
title=args.charts_cfd_title)
except charting.UnchartableData as e:
print("** WARNING: Did not draw chart:", e)
else:
fig = ax.get_figure()
fig.savefig(args.charts_cfd, bbox_inches='tight', dpi=300)
if args.charts_throughput:
print("Drawing throughput chart in", args.charts_throughput)
charting.set_style('darkgrid')
try:
ax = charting.throughput_trend_chart(
daily_throughput_data, title=args.charts_throughput_title)
except charting.UnchartableData as e:
print("** WARNING: Did not draw chart:", e)
else:
fig = ax.get_figure()
fig.savefig(args.charts_throughput,
bbox_inches='tight',
dpi=300)
if args.charts_burnup:
print("Drawing burnup chart in", args.charts_burnup)
charting.set_style('whitegrid')
try:
if args.points:
ax = charting.burnup(cfd_data_sliced,
backlog_column=backlog_column,
done_column=done_column,
title=args.charts_burnup_title,
sized='Sized')
else:
ax = charting.burnup(cfd_data_sliced,
backlog_column=backlog_column,
done_column=done_column,
title=args.charts_burnup_title,
sized='')
except charting.UnchartableData as e:
print("** WARNING: Did not draw chart:", e)
else:
fig = ax.get_figure()
fig.savefig(args.charts_burnup, bbox_inches='tight', dpi=300)
if args.charts_burnup_forecast:
target = args.charts_burnup_forecast_target or None
trials = args.charts_burnup_forecast_trials or 100
deadline = parse_relative_date(
args.charts_burnup_forecast_deadline
) if args.charts_burnup_forecast_deadline else None
deadline_confidence = args.charts_burnup_forecast_deadline_confidence
print("Drawing burnup foreacst chart in",
args.charts_burnup_forecast)
charting.set_style('whitegrid')
try:
if args.points:
ax = charting.burnup_forecast(
cfd_data_sliced,
daily_throughput_data,
trials=trials,
target=target,
backlog_column=backlog_column,
done_column=done_column,
percentiles=quantiles,
deadline=deadline,
deadline_confidence=deadline_confidence,
title=args.charts_burnup_forecast_title,
sized='Sized')
else:
ax = charting.burnup_forecast(
cfd_data_sliced,
daily_throughput_data,
trials=trials,
target=target,
backlog_column=backlog_column,
done_column=done_column,
percentiles=quantiles,
deadline=deadline,
deadline_confidence=deadline_confidence,
title=args.charts_burnup_forecast_title,
sized='')
except charting.UnchartableData as e:
print("** WARNING: Did not draw chart:", e)
else:
fig = ax.get_figure()
fig.savefig(args.charts_burnup_forecast,
bbox_inches='tight',
dpi=300)
if args.charts_wip:
print("Drawing WIP chart in", args.charts_wip)
charting.set_style('darkgrid')
try:
ax = charting.wip_chart(q.cfd(
cycle_data[cycle_data[backlog_column] >= (
datetime.date.today() - datetime.timedelta(
weeks=(args.charts_wip_window or 6)))]),
start_column=committed_column,
end_column=final_column,
title=args.charts_wip_title)
except charting.UnchartableData as e:
print("** WARNING: Did not draw chart:", e)
else:
fig = ax.get_figure()
fig.savefig(args.charts_wip, bbox_inches='tight', dpi=300)
if args.charts_ageing_wip:
print("Drawing ageing WIP chart in", args.charts_ageing_wip)
charting.set_style('whitegrid')
try:
ax = charting.ageing_wip_chart(
cycle_data,
start_column=committed_column,
end_column=final_column,
done_column=done_column,
title=args.charts_ageing_wip_title)
except charting.UnchartableData as e:
print("** WARNING: Did not draw chart:", e)
else:
fig = ax.get_figure()
fig.savefig(args.charts_ageing_wip,
bbox_inches='tight',
dpi=300)
if args.charts_net_flow:
print("Drawing net flow chart in", args.charts_net_flow)
charting.set_style('darkgrid')
try:
ax = charting.net_flow_chart(q.cfd(
cycle_data[cycle_data[backlog_column] >= (
datetime.date.today() - datetime.timedelta(
weeks=(args.charts_net_flow_window or 6)))]),
start_column=committed_column,
end_column=done_column,
title=args.charts_net_flow_title)
except charting.UnchartableData as e:
print("** WARNING: Did not draw chart:", e)
else:
fig = ax.get_figure()
fig.savefig(args.charts_net_flow, bbox_inches='tight', dpi=300)
print("Done")
def __init__(self,
xsizes,
intersects,
outf='cairotest.pdf',
width=1024,
height=1024,
bgcol='embl_gray125',
cols=None,
interscols=None,
ysizes=None,
ycols=None,
skip=3.5,
margin=24,
mincircle=5,
cellpadding=4):
for key, val in iteritems(locals()):
setattr(self, key, val)
self.colors = {
'embl_green': (115, 179, 96, 255),
'embl_blue': (0, 102, 102, 255),
'embl_yellow': (250, 183, 0, 255),
'embl_red': (227, 62, 62, 255),
'embl_black': (0, 0, 0, 255),
'embl_gray875': (32, 32, 32, 255),
'embl_gray75': (64, 64, 64, 255),
'embl_gray625': (96, 96, 96, 255),
'embl_gray50': (128, 128, 128, 255),
'embl_gray25': (192, 192, 192, 255),
'embl_gray125': (224, 224, 224, 255),
'white': (255, 255, 255, 255)
}
# positions of circle labels:
self.clabels = [(0.5, math.sqrt(3) / -2.0), (math.sqrt(3) / 2.0, -0.5),
(math.sqrt(2) / 2.0, math.sqrt(2) / -2.0)]
self.palette = [
self.colors[x]
for x in ['embl_green', 'embl_blue', 'embl_yellow']
]
self.fontpal = [
self.colors[x] for x in ['embl_gray875', 'white', 'embl_gray875']
]
self.font = 'HelveticaNeueLT Std Lt'
self.bgcol = self.bgcol if type(self.bgcol) is tuple else self.colors[
self.bgcol]
# set parameters for x:
# list of column labels (first elements of tuples in x list)
self.xlabs = [x[0] for x in self.xsizes]
# colors of sets in column headers
self.xcols = self.get_colors(self.xsizes)
# set sizes:
self.xsizes = self.get_sizes(self.xsizes)
# same for y:
self.ylabs = [y[0] for y in self.ysizes
] if self.ysizes is not None else self.xlabs
self.ycols = self.get_colors(self.ysizes) \
if self.ysizes is not None else self.xcols
self.ysizes = self.get_sizes(self.ysizes) \
if self.ysizes is not None else self.xsizes
# margin:
# margin is either a single integer, or a tuple of 4 integers:
self.margin = self.margin if type(self.margin) is tuple else (
self.margin, ) * 4
# table:
# proportions of cell sizes:
self.xcellsize = [3, 1] + [3] * len(self.xsizes)
self.ycellsize = [3, 1] + [3] * len(self.ysizes)
# sizes of table cells:
self.xcoo = self.cells(self.xcellsize, self.margin[0],
self.width - self.margin[1])
self.ycoo = self.cells(self.ycellsize, self.margin[2],
self.height - self.margin[3])
# width and height of diagram cells:
self.cellw = self.xcoo[0]
self.cellh = self.ycoo[0]
# largest circle fit in the cells:
self.maxcircle = min(self.cellw, self.cellh) / 2 - 2 * self.cellpadding
self.maxarea = pow(self.maxcircle, 2) * math.pi
self.minarea = pow(self.mincircle, 2) * math.pi
# scaling circle sizes between min and max circle size
self.xcircsize = self.scale_sizes(self.xsizes)
self.ycircsize = self.scale_sizes(self.ysizes)
ssize = self.scale_sizes([x['size'] for x in self.intersects.values()])
for i, k in enumerate(self.intersects.keys()):
self.intersects[k]['ssize'] = ssize[i]
if 'color' not in self.intersects[k]:
self.intersects[k]['color'] = self.palette[0:len(ssize[i])]
def __init__(self,
graph=None,
filename=None,
graphix_dir="pdf",
graphix_format="pdf",
name=None,
title_text=None,
title_font_family=None,
title_font_size=None,
title_color='#646567',
size=None,
layout="fruchterman_reingold",
layout_param=None,
vertex_label=None,
vertex_size=None,
vertex_label_size='degree_label_size',
edge_width=None,
vertex_color='#6EA945',
vertex_label_color='#007B7F',
vertex_alpha='AA',
vertex_frame_color='#FFFFFF00',
vertex_frame_width=0,
edge_label=None,
edge_label_size=None,
edge_label_color='#007B7F',
edge_curved=None,
edge_color='#818284',
edge_alpha='AA',
autocurve=None,
vertex_label_font="sans-serif",
edge_label_font="sans-serif",
edge_arrow_size=1.0,
edge_arrow_width=1.0,
palettes={},
bbox=None,
margin=10,
small=None,
dimensions=(1280, 1280),
grouping=None,
**kwargs):
# setting parameters:
for key, val in iteritems(locals()):
setattr(self, key, val)
self.default_alpha = {
'vertex_color': 'AA',
'edge_color': 'AA',
'vertex_label_color': 'FF',
'vertex_frame_color': '00',
'edge_label_color': 'FF'
}
self.default_vertex_label_size = 6.0
self.plots = []
self.session = gen_session_id()
self.loglevel = 'INFO'
self.ownlog = logn.logw(self.session, self.loglevel)
self.name = self.name if self.name is not None else self.session
self.label_sizes = {
'small': (15.0, 13.7),
'medium': (13.0, 10.0),
'large': (9.0, 6.0)
}
self.palettes = {
'vertex': ['#6EA945', '#007B7F', '#FCCC06', '#DA0025', '#000000'],
'edge': ['#007B7F', '#6EA945', '#DA0025'],
'vertex_label': ['#454447'],
'edge_label': ['#454447']
}
self.small_param = {
'vertex_size': 21,
'edge_width': 0.051,
'autocurve': True,
'vertex_label_dist': 1.5
}
self.medium_param = {
'vertex_size': 7,
'edge_width': 0.051,
'autocurve': True,
'vertex_label_dist': 1.33,
'edge_label_size': 1.0
}
self.large_param = {
'vertex_size': 2,
'edge_width': 0.051,
'vertex_label_dist': 1.0,
'edge_label_size': 1.0
}
self.layout_defaults = {
'fruchterman_reingold': {
'repulserad': self.graph.vcount()**2.8,
'maxiter': 1000,
'area': self.graph.vcount()**2.3
}
}
self.update_page()
self.update_graph(graph)
def iteritems(self):
return iteritems(self.dictionary)
def set_defaults(self, preset):
if hasattr(self, preset):
for k, v in iteritems(getattr(self, preset)):
self.set_param(k, v)
def _attributes(self):
return {
k: v
for k, v in iteritems(vars(self))
if not (k.startswith('_') or k == 'id')
}
def __reduce__(self):
return dict, tuple(), None, None, iter(iteritems(self.dictionary))
def asDict(self):
retval = {}
for interface, handler in iteritems(self.handlers):
retval.update(handler.asDict())
return retval
def mergePackageDependenciesList(package, isDependencies=True):
packageDepDict = dict()
if isDependencies:
routineDeps = package.getPackageRoutineDependencies()
globalDeps = package.getPackageGlobalDependencies()
globalRtnDeps = package.getPackageGlobalRoutineDependencies()
globalGblDeps = package.getPackageGlobalGlobalDependencies()
fileManDeps = package.getPackageFileManFileDependencies()
dbCallDeps = package.getPackageFileManDbCallDependencies()
optionDeps = package.getPackageComponentDependencies()
else:
routineDeps = package.getPackageRoutineDependents()
globalDeps = package.getPackageGlobalDependents()
globalRtnDeps = package.getPackageGlobalRoutineDependendents()
globalGblDeps = package.getPackageGlobalGlobalDependents()
fileManDeps = package.getPackageFileManFileDependents()
dbCallDeps = package.getPackageFileManDbCallDependents()
optionDeps = {}
for (package, depTuple) in iteritems(routineDeps):
if package not in packageDepDict:
packageDepDict[package] = [
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
]
packageDepDict[package][0] = len(depTuple[0])
packageDepDict[package][1] = len(depTuple[1])
for (package, depTuple) in iteritems(globalDeps):
if package not in packageDepDict:
packageDepDict[package] = [
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
]
packageDepDict[package][2] = len(depTuple[0])
packageDepDict[package][3] = len(depTuple[1])
for (package, depTuple) in iteritems(fileManDeps):
if package not in packageDepDict:
packageDepDict[package] = [
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
]
packageDepDict[package][4] = len(depTuple[0])
packageDepDict[package][5] = len(depTuple[1])
for (package, depTuple) in iteritems(dbCallDeps):
if package not in packageDepDict:
packageDepDict[package] = [
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
]
packageDepDict[package][6] = len(depTuple[0])
packageDepDict[package][7] = len(depTuple[1])
for (package, depTuple) in iteritems(optionDeps):
if package not in packageDepDict:
packageDepDict[package] = [
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
]
packageDepDict[package][8] = len(depTuple[0])
packageDepDict[package][9] = len(depTuple[1])
for (package, depTuple) in iteritems(globalRtnDeps):
if package not in packageDepDict:
packageDepDict[package] = [
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
]
packageDepDict[package][10] = len(depTuple[0])
packageDepDict[package][11] = len(depTuple[1])
for (package, depTuple) in iteritems(globalGblDeps):
if package not in packageDepDict:
packageDepDict[package] = [
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
]
packageDepDict[package][12] = len(depTuple[0])
packageDepDict[package][13] = len(depTuple[1])
return packageDepDict
def _convert_to_variant_record(self, record, infos, formats):
"""Converts the PyVCF record to a :class:`Variant` object.
Args:
record (:class:`~vcf.model._Record`): An object containing info about a
variant.
infos (dict): The PyVCF dict storing INFO extracted from the VCF header.
The key is the info key and the value is :class:`~vcf.parser._Info`.
formats (dict): The PyVCF dict storing FORMAT extracted from the VCF
header. The key is the FORMAT key and the value is
:class:`~vcf.parser._Format`.
Returns:
A :class:`Variant` object from the given record.
"""
variant = Variant()
variant.reference_name = record.CHROM
variant.start = record.start
variant.end = record.end
variant.reference_bases = (
record.REF if record.REF != MISSING_FIELD_VALUE else None)
# ALT fields are classes in PyVCF (e.g. Substitution), so need convert
# them to their string representations.
variant.alternate_bases.extend([str(r) for r in record.ALT
if r] if record.ALT else [])
variant.names.extend(record.ID.split(';') if record.ID else [])
variant.quality = record.QUAL
# PyVCF uses None for '.' and an empty list for 'PASS'.
if record.FILTER is not None:
variant.filters.extend(
record.FILTER if record.FILTER else [PASS_FILTER])
for k, v in iteritems(record.INFO):
# Special case: END info value specifies end of the record, so adjust
# variant.end and do not include it as part of variant.info.
if k == END_INFO_KEY:
variant.end = v
continue
field_count = None
if k in infos:
field_count = self._get_field_count_as_string(infos[k].num)
variant.info[k] = VariantInfo(data=v, field_count=field_count)
for sample in record.samples:
call = VariantCall()
call.name = sample.sample
for allele in sample.gt_alleles or [MISSING_GENOTYPE_VALUE]:
if allele is None:
allele = MISSING_GENOTYPE_VALUE
call.genotype.append(int(allele))
phaseset_from_format = (getattr(
sample.data, PHASESET_FORMAT_KEY) if PHASESET_FORMAT_KEY
in sample.data._fields else None)
# Note: Call is considered phased if it contains the 'PS' key regardless
# of whether it uses '|'.
if phaseset_from_format or sample.phased:
call.phaseset = (str(phaseset_from_format)
if phaseset_from_format else
DEFAULT_PHASESET_VALUE)
for field in sample.data._fields:
# Genotype and phaseset (if present) are already included.
if field in (GENOTYPE_FORMAT_KEY, PHASESET_FORMAT_KEY):
continue
data = getattr(sample.data, field)
# Convert single values to a list for cases where the number of fields
# is unknown. This is to ensure consistent types across all records.
# Note: this is already done for INFO fields in PyVCF.
if (field in formats
and formats[field].num is None and isinstance(
data, (int, float, long, str, unicode, bool))):
data = [data]
call.info[field] = data
variant.calls.append(call)
return variant
raise ValueError("Unable to parse client id from session_id: %s" %
session_id)
status_map = {
rdf_flows.GrrStatus.ReturnedStatus.OK: FlowStatus.Status.OK,
rdf_flows.GrrStatus.ReturnedStatus.IOERROR: FlowStatus.Status.IOERROR,
rdf_flows.GrrStatus.ReturnedStatus.CLIENT_KILLED:
FlowStatus.Status.CLIENT_KILLED,
rdf_flows.GrrStatus.ReturnedStatus.NETWORK_LIMIT_EXCEEDED:
FlowStatus.Status.NETWORK_LIMIT_EXCEEDED,
rdf_flows.GrrStatus.ReturnedStatus.GENERIC_ERROR: FlowStatus.Status.ERROR,
}
inv_status_map = {v: k for k, v in iteritems(status_map)}
def FlowResponseForLegacyResponse(legacy_msg):
"""Helper function to convert legacy client replies to flow responses."""
if legacy_msg.type == legacy_msg.Type.MESSAGE:
response = FlowResponse(client_id=_ClientIDFromSessionID(
legacy_msg.session_id),
flow_id=legacy_msg.session_id.Basename(),
request_id=legacy_msg.request_id,
response_id=legacy_msg.response_id,
payload=legacy_msg.payload)
elif legacy_msg.type == legacy_msg.Type.STATUS:
legacy_status = legacy_msg.payload
if legacy_status.status not in status_map:
raise ValueError("Unable to convert returned status: %s" %
def _list(self, dir_or_prefix):
for path, size in iteritems(self._files):
if path.startswith(dir_or_prefix):
yield FileMetadata(path, size)
def init_flat_sio(self,
cid,
sio,
data_format,
transport,
wsgi_environ,
required_list,
_sio_container=(tuple, list)):
""" Initializes flat SIO requests, i.e. not list ones.
"""
self.is_xml = data_format == SIMPLE_IO.FORMAT.XML
self.data_format = data_format
self.transport = transport
self._wsgi_environ = wsgi_environ
optional_list = getattr(sio, 'input_optional', [])
optional_list = optional_list if isinstance(
optional_list, _sio_container) else [optional_list]
path_prefix = getattr(sio, 'request_elem', 'request')
default_value = getattr(sio, 'default_value', NO_DEFAULT_VALUE)
use_text = getattr(sio, 'use_text', True)
use_channel_params_only = getattr(sio, 'use_channel_params_only',
False)
self.encrypt_secrets = getattr(sio, 'encrypt_secrets', True)
if self.simple_io_config:
self.has_simple_io_config = True
self.bool_parameter_prefixes = self.simple_io_config.get(
'bool_parameter_prefixes', [])
self.int_parameters = self.simple_io_config.get(
'int_parameters', [])
self.int_parameter_suffixes = self.simple_io_config.get(
'int_parameter_suffixes', [])
self.bytes_to_str_encoding = self.simple_io_config['bytes_to_str'][
'encoding']
else:
self.payload = self.raw_request
required_params = {}
if required_list:
required_list = required_list if isinstance(
required_list, _sio_container) else [required_list]
# Needs to check for this exact default value to prevent a FutureWarning in 'if not self.payload'
if self.payload == '' and not self.channel_params:
raise ZatoException(cid, 'Missing input')
required_params.update(
self.get_params(required_list, use_channel_params_only,
path_prefix, default_value, use_text))
if optional_list:
optional_params = self.get_params(optional_list,
use_channel_params_only,
path_prefix, default_value,
use_text, False)
else:
optional_params = {}
self.input.update(required_params)
self.input.update(optional_params)
for param, value in iteritems(self.channel_params):
if param not in self.input:
self.input[param] = value
def clean_up(self, servermap):
for ip, port in iteritems(servermap):
result = ClientGen.call(ip, port, 'STOP')
log.info('stop server @{}:{}, {}'.format(ip, port, result))
def dictionaries_eq(a, b):
return set(iteritems(a)) == set(iteritems(b))
def build_shoe_diagram(config):
builder = DiagramBuilder()
station = builder.AddSystem(ManipulationDiagram(config))
station.add_rope_and_ground(include_ground=False)
if 'arms' in config['env']:
station.add_arms_from_config(config)
parser = Parser(station.mbp, station.sg)
shoe_dir = os.path.dirname(os.path.abspath(__file__))
model_file = os.path.join(shoe_dir, "model/shoe.sdf")
shoe_model = parser.AddModelFromFile(model_file, "shoe")
if config["env"]["visualization"]:
station.connect_to_drake_visualizer()
visualizer = None
if "meshcat" in config["env"] and config["env"]["meshcat"]:
visualizer = station.connect_to_meshcat()
if config["env"]["parameterization"] == "closed":
left_rope_point = station.add_vis_object("left_rope", [1, 0, 0, 1])
right_rope_point = station.add_vis_object("right_rope", [0, 1, 0, 1])
left_target_point = station.add_vis_object("left_target_point",
[1, 0, 0, 1])
right_target_point = station.add_vis_object("right_target_point",
[0, 1, 0, 1])
if config["env"]["rgbd_sensors"]["enabled"]:
station.add_rgbd_sensors_from_config(config)
station.finalize()
post_finalize_rope_settings(config, station.mbp, station.sg)
targets = {}
if 'arms' in config['env']:
gripper_info = {}
for arm_name, arm_config in iteritems(config['env']['arms']):
# Add PID Control
gripper = station.mbp.GetBodyByName("body",
station.model_ids[arm_name])
gripper_info[arm_name] = gripper.index()
# Initialize targets from file
init = config["env"]["arms"][arm_name]["rpy"][:]
init.extend(config["env"]["arms"][arm_name]["pos"])
targets[arm_name] = builder.AddSystem(ConstantVectorSource(init))
width_init = config["env"]["arms"][arm_name]["grip"]
targets[f"{arm_name}_width"] = builder.AddSystem(
ConstantVectorSource([width_init]))
pid = builder.AddSystem(SpatialHandController(gripper_info))
builder.Connect(station.GetOutputPort(f"body_poses"),
pid.GetInputPort("body_positions"))
sp_control = builder.AddSystem(
SetpointController(
gripper_info, {
"position": [0.005, 0.005, 0.005, 0.0003, 0.0003, 0.0003],
"width": 0.001
}))
for arm_name, arm_config in iteritems(config['env']['arms']):
builder.Connect(targets[arm_name].get_output_port(0),
sp_control.GetInputPort(f"{arm_name}_target"))
builder.Connect(sp_control.GetOutputPort(f"{arm_name}_setpoint"),
pid.GetInputPort(f"{arm_name}_desired"))
builder.Connect(
targets[f"{arm_name}_width"].get_output_port(0),
sp_control.GetInputPort(f"{arm_name}_width_target"))
builder.Connect(
sp_control.GetOutputPort(f"{arm_name}_width_setpoint"),
station.GetInputPort(f"{arm_name}_position"))
builder.Connect(pid.GetOutputPort("spatial_forces_vector"),
station.GetInputPort("spatial_input"))
diagram = builder.Build()
simulator = Simulator(diagram)
sim_context = simulator.get_mutable_context()
station_context = diagram.GetMutableSubsystemContext(station, sim_context)
systems = {
"station": station,
"targets": targets,
"sp_control": sp_control,
"pid": pid
}
if config["env"]["parameterization"] == "closed":
systems["left_rope"] = left_rope_point
systems["right_rope"] = right_rope_point
systems["left_target_point"] = left_target_point
systems["right_target_point"] = right_target_point
if 'arms' in config['env']:
values = {}
for arm_name, arm_config in iteritems(config['env']['arms']):
station.GetInputPort(f"{arm_name}_force_limit").FixValue(
station_context, 40.)
simulator.set_target_realtime_rate(config['env']['target_realtime_rate'])
reset_simulator_from_config(config, simulator, diagram, systems)
return simulator, diagram, systems, visualizer
def get_datasources(self):
"""
Get datasource or create it if it does not exist
:return: None
"""
headers = {"Authorization": "Bearer " + self.api_key}
try:
url = "%s://%s:%s/api/datasources" % (self.scheme, self.host,
self.port)
response = requests.get(url, headers=headers, timeout=10)
except requests.exceptions.SSLError as e:
print("[cron_grafana] SSL connection error to grafana %s: %s" %
(self.name, e))
current_app.logger.error(
"[cron_grafana] SSL connection error to grafana %s "
"for dashboard creation: %s", self.name, e)
return
except requests.exceptions.RequestException as e:
print("[cron_grafana] Connection error to grafana %s: %s" %
(self.name, e))
current_app.logger.error(
"[cron_grafana] Connection error to grafana %s "
"for dashboard creation: %s", self.name, e)
self.connection = False
return
resp = response.json()
if 'message' in resp:
print("----------")
print("Grafana message: %s" % resp['message'])
print("----------")
return
# get existing datasource in grafana
self.datasources = {}
for datasource in iter(resp):
self.datasources[datasource['name']] = {
'id': datasource['id'],
'ts_id': None
}
# associate the datasource to timeseries data
for _, timeserie in iteritems(self.timeseries):
ds_name = 'alignak-' + timeserie['type'] + '-' + timeserie['name']
if ds_name in self.datasources.keys():
self.datasources[ds_name]['ts_id'] = str(timeserie['_id'])
continue
# Missing datasource, create it
# Note that no created datasource is the default one
if timeserie['type'] == 'influxdb':
data = {
"name":
ds_name,
"type":
"influxdb",
"typeLogoUrl":
"",
"access":
"proxy",
"url":
"http://" + timeserie['address'] + ":" +
str(timeserie['port']),
"password":
timeserie['password'],
"user":
timeserie['login'],
"database":
timeserie['database'],
"basicAuth":
False,
"basicAuthUser":
"",
"basicAuthPassword":
"",
"withCredentials":
False,
"isDefault":
False,
"jsonData": {}
}
elif timeserie['type'] == 'graphite':
data = {
"name":
ds_name,
"type":
"graphite",
"access":
"proxy",
"url":
"http://%s:%s" % (timeserie['graphite_address'],
timeserie['graphite_port']),
"basicAuth":
False,
"basicAuthUser":
"",
"basicAuthPassword":
"",
"withCredentials":
False,
"isDefault":
False,
"jsonData": {}
}
# Request datasource creation
response = requests.post(self.scheme + '://' + self.host + ':' +
self.port + '/api/datasources',
json=data,
headers=headers)
resp = response.json()
# resp is as: {u'message': u'Datasource added', u'id': 4}
if 'id' not in resp and 'message' in resp:
current_app.logger.info("Grafana message: %s", resp['message'])
return
current_app.logger.info(
"[grafana-%s] datasource created: '%s': id = %s", self.name,
ds_name, resp['id'])
self.datasources[ds_name] = {
'id': resp['id'],
'ts_id': str(timeserie['_id'])
}
current_app.logger.info("[grafana-%s] available datasources:",
self.name)
for ds_name, datasource in iteritems(self.datasources):
current_app.logger.info("- %s: %s", ds_name, datasource)
