def get_content_snippet2(content, keyword, max_lines=10):
max_lines = int(max_lines)
p = re.compile(
r'(?P<before>.*)%s(?P<after>.*)' %
re.escape(keyword),
re.MULTILINE | re.IGNORECASE | re.DOTALL)
m = p.search(content)
html = ""
if m:
words = list(filter(
lambda x: x != "",
striptags(
m.group("before")).split("\n")))
before_lines = words[-max_lines // 2:]
words = list(filter(
lambda x: x != "",
striptags(
m.group("after")).split("\n")))
after = "<br/>".join(words[:max_lines - len(before_lines)])
before = "<br/>".join(before_lines)
html = "%s %s %s" % (before, striptags(keyword), after)
kw_p = re.compile(r'(%s)' % keyword, re.IGNORECASE)
html = kw_p.sub(r"<strong>\1</strong>", html)
html = mark_safe(html)
else:
html = " ".join(
list(filter(
lambda x: x != "",
striptags(content).replace(
"\n",
" ").split(" ")))[
:max_lines])
return html
def check(process_output, judge_output, split_on='lines', **kwargs):
split_pattern = {
'lines': b'[\r\n]',
'whitespace': b'[\s]',
}.get(split_on)
if not split_pattern:
raise InternalError('invalid `split_on` mode')
process_lines = list(filter(None, resplit(split_pattern, utf8bytes(process_output))))
judge_lines = list(filter(None, resplit(split_pattern, utf8bytes(judge_output))))
if len(process_lines) != len(judge_lines):
return False
if split_on == 'lines':
process_lines = list(map(six.binary_type.split, process_lines))
judge_lines = list(map(six.binary_type.split, judge_lines))
process_lines.sort()
judge_lines.sort()
for process_line, judge_line in zip(process_lines, judge_lines):
if process_line != judge_line:
return False
return True
def startup(self, group):
""" Prepare for a new run.
Args
----
group : `Group`
Group that owns this recorder.
"""
myparams = myunknowns = myresids = set()
if MPI:
rank = group.comm.rank
owned = group._owning_ranks
# Compute the inclusion lists for recording
if self.options['record_params']:
myparams = set(filter(self._check_path, group.params))
if self.options['record_unknowns']:
myunknowns = set(filter(self._check_path, group.unknowns))
if self.options['record_resids']:
myresids = set(filter(self._check_path, group.resids))
self._filtered[group.pathname] = {
'p': myparams,
'u': myunknowns,
'r': myresids
}
def label_and_sentence(line, clean_fn):
label_text = re.split(TSVSeqLabelReader.SPLIT_ON, line)
label = label_text[0]
text = label_text[1:]
text = ' '.join(list(filter(lambda s: len(s) != 0, [clean_fn(w) for w in text])))
text = list(filter(lambda s: len(s) != 0, re.split('\s+', text)))
return label, text
def parse_celery_workers(celery_workers):
"""
Parses the response from the flower get workers api into a list of hosts
we expect to be running and a list of hosts we expect to be stopped
"""
expect_stopped = []
expect_running = list(filter(
lambda hostname: not hostname.endswith('_timestamp'),
celery_workers,
))
timestamped_workers = list(filter(
lambda hostname: hostname.endswith('_timestamp'),
celery_workers,
))
def _strip_timestamp(hostname):
return '.'.join(hostname.split('.')[:-1])
timestamped_workers = sorted(timestamped_workers, key=_strip_timestamp)
for hostname, group in groupby(timestamped_workers, _strip_timestamp):
sorted_workers = sorted(list(group), reverse=True)
expect_running.append(sorted_workers.pop(0))
expect_stopped.extend(sorted_workers)
return expect_running, expect_stopped
def process_trade(self, trade_event):
if trade_event.sid not in self.open_orders:
return
if trade_event.volume < 1:
# there are zero volume trade_events bc some stocks trade
# less frequently than once per minute.
return
orders = self.open_orders[trade_event.sid]
orders.sort(key=lambda o: o.dt)
# Only use orders for the current day or before
current_orders = filter(
lambda o: o.dt <= trade_event.dt,
orders)
processed_orders = []
for txn, order in self.process_transactions(trade_event,
current_orders):
processed_orders.append(order)
yield txn, order
# remove closed orders. we should only have to check
# processed orders
def not_open(order):
return not order.open
closed_orders = filter(not_open, processed_orders)
for order in closed_orders:
orders.remove(order)
if len(orders) == 0:
del self.open_orders[trade_event.sid]
def process(self):
"""
Process the file upload and add products to the range
"""
all_ids = set(self.extract_ids())
products = self.range.included_products.all()
existing_skus = products.values_list('stockrecord__partner_sku',
flat=True)
existing_skus = set(filter(bool, existing_skus))
existing_upcs = products.values_list('upc', flat=True)
existing_upcs = set(filter(bool, existing_upcs))
existing_ids = existing_skus.union(existing_upcs)
new_ids = all_ids - existing_ids
products = Product._default_manager.filter(
models.Q(stockrecord__partner_sku__in=new_ids) |
models.Q(upc__in=new_ids))
for product in products:
self.range.add_product(product)
# Processing stats
found_skus = products.values_list('stockrecord__partner_sku',
flat=True)
found_skus = set(filter(bool, found_skus))
found_upcs = set(filter(bool, products.values_list('upc', flat=True)))
found_ids = found_skus.union(found_upcs)
missing_ids = new_ids - found_ids
dupes = set(all_ids).intersection(existing_ids)
self.mark_as_processed(products.count(), len(missing_ids), len(dupes))
def _get_eligible_broker_pair(self, under_loaded_rg, eligible_partition):
"""Evaluate and return source and destination broker-pair from over-loaded
and under-loaded replication-group if possible, return None otherwise.
Return source broker with maximum partitions and destination broker with
minimum partitions based on following conditions:-
1) At-least one broker in under-loaded group which does not have
victim-partition. This is because a broker cannot have duplicate replica.
2) At-least one broker in over-loaded group which has victim-partition
"""
under_brokers = list(filter(
lambda b: eligible_partition not in b.partitions,
under_loaded_rg.brokers,
))
over_brokers = list(filter(
lambda b: eligible_partition in b.partitions,
self.brokers,
))
# Get source and destination broker
source_broker, dest_broker = None, None
if over_brokers:
source_broker = max(
over_brokers,
key=lambda broker: len(broker.partitions),
)
if under_brokers:
dest_broker = min(
under_brokers,
key=lambda broker: len(broker.partitions),
)
return (source_broker, dest_broker)
def test_get_questions_with_repeats(self):
"""
This test ensures that questions that start with the repeat group id
do not get marked as repeats. For example:
/data/repeat_name <-- repeat group path
/data/repeat_name_count <-- question path
Before /data/repeat_name_count would be tagged as a repeat incorrectly.
See http://manage.dimagi.com/default.asp?234108 for context
"""
form = self.app.get_form(self.form_with_repeats_unique_id)
questions = form.wrapped_xform().get_questions(
['en'],
include_groups=True,
)
repeat_name_count = list(filter(
lambda question: question['value'] == '/data/repeat_name_count',
questions,
))[0]
self.assertIsNone(repeat_name_count['repeat'])
repeat_question = list(filter(
lambda question: question['value'] == '/data/repeat_name/question5',
questions,
))[0]
self.assertEqual(repeat_question['repeat'], '/data/repeat_name')
def _leaf_versions(tree, rc):
'''
Recursively traverse the versions tree in a depth-first fashion,
and collect the last node of each branch, i.e. leaf versions.
'''
versions = []
if _is_iterable(tree):
for subtree in tree:
versions.extend(_leaf_versions(subtree, rc))
if not versions:
if rc:
last_rc = next(filter(lambda v: v.is_rc, reversed(tree)), None)
last_prod = next(
filter(lambda v: not v.is_rc, reversed(tree)), None)
if last_rc and last_prod and (last_prod < last_rc):
versions.extend([last_prod, last_rc])
elif not last_prod:
versions.append(last_rc)
else:
# Either there is no RC, or we ignore the RC as older than
# the latest production version:
versions.append(last_prod)
else:
versions.append(tree[-1])
return versions
def removePyc(folder, only_excess=True, show_logs=True):
folder = sp(folder)
for root, dirs, files in os.walk(folder):
pyc_files = filter(lambda filename: filename.endswith(".pyc"), files)
py_files = set(filter(lambda filename: filename.endswith(".py"), files))
excess_pyc_files = (
filter(lambda pyc_filename: pyc_filename[:-1] not in py_files, pyc_files) if only_excess else pyc_files
)
for excess_pyc_file in excess_pyc_files:
full_path = os.path.join(root, excess_pyc_file)
if show_logs:
log.debug("Removing old PYC file: %s", full_path)
try:
os.remove(full_path)
except:
log.error("Couldn't remove %s: %s", (full_path, traceback.format_exc()))
for dir_name in dirs:
full_path = os.path.join(root, dir_name)
if len(os.listdir(full_path)) == 0:
try:
os.rmdir(full_path)
except:
log.error("Couldn't remove empty directory %s: %s", (full_path, traceback.format_exc()))
def test_simple_plan_add_on_creation(self):
# add a sample plan to the plans backend
mocurly.backend.plans_backend.add_object(self.base_backed_plan_data['plan_code'], self.base_backed_plan_data)
self.assertEqual(len(mocurly.backend.plan_add_ons_backend.datastore), 0)
# now create some addons
plan = recurly.Plan.get(self.base_backed_plan_data['plan_code'])
for add_on in self.base_add_on_data:
add_on['name'] = add_on['add_on_code'].upper()
add_on['unit_amount_in_cents'] = recurly.Money(**add_on['unit_amount_in_cents'])
plan.create_add_on(recurly.AddOn(**add_on))
self.assertEqual(len(mocurly.backend.plan_add_ons_backend.datastore), 2)
foo_add_on_backed = mocurly.backend.plan_add_ons_backend.get_object(self.base_backed_plan_data['plan_code'] + '__foo')
add_ons = filter(lambda add_on: add_on['add_on_code'] == 'foo', self.base_add_on_data)
foo_add_on = next(add_ons)
for k, v in foo_add_on.items():
if k == 'unit_amount_in_cents':
self.assertEqual(foo_add_on_backed[k], dict((curr, str(amt)) for curr, amt in v.currencies.items()))
else:
self.assertEqual(foo_add_on_backed[k], v)
bar_add_on_backed = mocurly.backend.plan_add_ons_backend.get_object(self.base_backed_plan_data['plan_code'] + '__bar')
add_ons = filter(lambda add_on: add_on['add_on_code'] == 'bar', self.base_add_on_data)
bar_add_on = next(add_ons)
for k, v in bar_add_on.items():
if k == 'unit_amount_in_cents':
self.assertEqual(bar_add_on_backed[k], dict((curr, str(amt)) for curr, amt in v.currencies.items()))
else:
self.assertEqual(bar_add_on_backed[k], v)
# make sure foreign keys are linked properly
self.assertEqual(len(plan.add_ons()), 2)
def test_in(self):
values = ['a', 'b', 'c']
filter = self.get_filter('in', values)
for value in values:
self.assertTrue(filter({'foo': value}))
for value in ['d', 'e', 'f']:
self.assertFalse(filter({'foo': value}))
def retrieve_keys(bucket, key, prefix='', postfix='', delim='/',
directories=False, recursive=False):
"""
Retrieve keys from a bucket
"""
if key and prefix:
assert key.endswith(delim)
key += prefix
# check whether key is a directory
if not key.endswith(delim) and key:
# check for matching prefix
if BotoClient.check_prefix(bucket, key + delim, delim=delim):
# found a directory
key += delim
listdelim = delim if not recursive else None
results = bucket.list(prefix=key, delimiter=listdelim)
if postfix:
func = lambda k_: BotoClient.filter_predicate(k_, postfix, inclusive=True)
return filter(func, results)
elif not directories:
func = lambda k_: BotoClient.filter_predicate(k_, delim, inclusive=False)
return filter(func, results)
else:
return results
def get_revert_migrations(self, current_migrations, backup_migrations):
current_migrations, all_migrations = itertools.tee(reversed(list(map(
Migration,
filter(None, current_migrations.splitlines()),
))))
all_migrations = utils.OrderedSet(all_migrations)
backup_migrations = reversed(list(map(
Migration,
filter(None, backup_migrations.splitlines()),
)))
revert_migrations = collections.OrderedDict()
while True:
while True:
backup_migration = next(backup_migrations, None)
if not backup_migration or backup_migration in all_migrations:
break
for current_migration in current_migrations:
if current_migration == backup_migration:
break
revert_migration = self._get_parent_migration(
current_migration,
migrations=all_migrations,
)
revert_migrations[current_migration.app] = revert_migration
if backup_migration is None:
return revert_migrations.values()
def DelayedFcnCall(*args, **kwargs):
# Check to see if any args or kw are Later. If not, return normal fcn.
checkIfLater = lambda x: type(x) == Later
if (len(list(filter(checkIfLater, args))) == 0 and
len(list(filter(checkIfLater, list(kwargs.values())))) == 0):
return fcn(*args, **kwargs)
else:
return CreateLaterFunction(fcn, *args, **kwargs)
def startup(self, group):
""" Prepare for new run. """
# Compute the inclusion lists for recording
params = list(filter(self._check_path, group.params))
unknowns = list(filter(self._check_path, group.unknowns))
resids = list(filter(self._check_path, group.resids))
self._filtered[group.pathname] = (params, unknowns, resids)
def __init__(self, namespaces=None, pollster_list=None):
namespaces = namespaces or ['compute', 'central']
pollster_list = pollster_list or []
group_prefix = cfg.CONF.polling.partitioning_group_prefix
# features of using coordination and pollster-list are exclusive, and
# cannot be used at one moment to avoid both samples duplication and
# samples being lost
if pollster_list and cfg.CONF.coordination.backend_url:
raise PollsterListForbidden()
super(AgentManager, self).__init__()
def _match(pollster):
"""Find out if pollster name matches to one of the list."""
return any(utils.match(pollster.name, pattern) for
pattern in pollster_list)
if type(namespaces) is not list:
namespaces = [namespaces]
# we'll have default ['compute', 'central'] here if no namespaces will
# be passed
extensions = (self._extensions('poll', namespace).extensions
for namespace in namespaces)
# get the extensions from pollster builder
extensions_fb = (self._extensions_from_builder('poll', namespace)
for namespace in namespaces)
if pollster_list:
extensions = (moves.filter(_match, exts)
for exts in extensions)
extensions_fb = (moves.filter(_match, exts)
for exts in extensions_fb)
self.extensions = list(itertools.chain(*list(extensions))) + list(
itertools.chain(*list(extensions_fb)))
if self.extensions == []:
raise EmptyPollstersList()
self.discovery_manager = self._extensions('discover')
self.context = context.RequestContext('admin', 'admin', is_admin=True)
self.partition_coordinator = coordination.PartitionCoordinator()
# Compose coordination group prefix.
# We'll use namespaces as the basement for this partitioning.
namespace_prefix = '-'.join(sorted(namespaces))
self.group_prefix = ('%s-%s' % (namespace_prefix, group_prefix)
if group_prefix else namespace_prefix)
self.notifier = oslo_messaging.Notifier(
messaging.get_transport(),
driver=cfg.CONF.publisher_notifier.telemetry_driver,
publisher_id="ceilometer.polling")
self._keystone = None
self._keystone_last_exception = None
def purge_old(self):
cur_ts = time.time()
prev_ts = cur_ts - self.purge_elapsed
self.failed_log = list(filter(lambda x: x > prev_ts, self.failed_log))
self.succeeded_log = list(filter(lambda x: x > prev_ts,
self.succeeded_log))
self.failed_cnt = len(self.failed_log)
self.succeed_cnt = len(self.succeeded_log)
self.start_point = self._begin_log_ts()
def qsort(l, cmp_fn=cmp_num):
i = len(l)
if i == 0:
return l
pivot = i // 2
elt = l[pivot]
lt = list(filter(lambda x: cmp_fn(x, elt) == -1, l))
eq = list(filter(lambda x: cmp_fn(x, elt) == 0, l))
gt = list(filter(lambda x: cmp_fn(x, elt) == 1, l))
return qsort(lt, cmp_fn=cmp_fn) + eq + qsort(gt, cmp_fn=cmp_fn)
def load_numpy(max_pos=-1, max_neg=-1, *, unpack = False, scale_to_largest_image=False, scale_to_size=None):
# If you specify the maximum number of positive examples but not the maximum
# number of negative examples, it tries to return a 50/50 split. That may not
# actually make sense
if max_neg==-1 and max_pos>-1:
max_neg = max_pos
if unpack:
raise Exception('Unpacking to local FS is not supported yet')
if scale_to_size is not None and scale_to_largest_image:
raise Exception('Specify scaling to the largest image OR a size to scale to.')
# Debatable whether this is actually needed.
if scale_to_size is None and scale_to_largest_image:
with zipfile.ZipFile(train_zip_path) as z:
filelist = filter(jpg_re.match, z.namelist())
scale_to_size = util.find_largest_image_in_zip(z, filelist)
if scale_to_size is None and not scale_to_largest_image:
# silently changing to a hardcoded value, beware!
scale_to_size = [768,1024]
n_pos = 0
n_neg = 0
X = list()
y = list()
# Actually do the load
with zipfile.ZipFile(train_zip_path) as z:
filelist = filter(jpg_re.match, z.namelist())
for f in filelist:
if cat_re.match(f) and n_pos != max_pos:
y.append(1)
n_pos += 1
X.append(
util.process_img_from_file(
BytesIO(z.read(f)),
resize_dims = scale_to_size
)
)
elif (not cat_re.match(f)) and n_neg != max_neg:
y.append(0)
n_neg += 1
X.append(
util.process_img_from_file(
BytesIO(z.read(f)),
resize_dims = scale_to_size
)
)
if n_pos == max_pos and n_neg == max_neg:
break
return (np.vstack(X), np.array(y
))
def quickinstall(plone, products):
logger.info("Quick installing: %s", products)
qit = plone.portal_quickinstaller
not_installed_ids = [
x['id'] for x in qit.listInstallableProducts(skipInstalled=1)]
installed_ids = [x['id'] for x in qit.listInstalledProducts()]
installed_products = list(filter(installed_ids.count, products))
not_installed = list(filter(not_installed_ids.count, products))
if installed_products:
qit.reinstallProducts(installed_products)
if not_installed_ids:
qit.installProducts(not_installed)
def _get_nodes(self, account=None, type=None):
"""Return affected nodes matching the filters."""
result = self.affected_nodes
if account:
# Allow to filter by multiple accounts
if not isinstance(account, list):
account = [account]
result = filter(
lambda n: all([n.affects_account(a) for a in account]), result)
if type:
result = filter(lambda n: n.type == type, result)
return list(result)
def getAndAssertTableRowAction(self, response, table_name, action_name, row_id):
table = response.context[table_name + "_table"]
rows = list(moves.filter(lambda x: x.id == row_id, table.data))
self.assertEqual(1, len(rows), "Did not find a row matching id '%s'" % row_id)
row_actions = table.get_row_actions(rows[0])
actions = list(moves.filter(lambda x: x.name == action_name, row_actions))
msg_args = (action_name, table_name, row_id)
self.assertTrue(len(actions) > 0, "No action named '%s' found in '%s' table for id '%s'" % msg_args)
self.assertEqual(1, len(actions), "Multiple actions named '%s' found in '%s' table for id '%s'" % msg_args)
return actions[0]
def count_instances(inst=None, group=None, referrers=False, referents=False, prev=0):
gc.collect()
if group:
# def t(x):
# try:
# return group in x.__class__.__name__
# except:
# pass
t = lambda x: group in str(type(x))
n = group
objs = list(filter(t, gc.get_objects()))
s = sum(sys.getsizeof(o) for o in objs) * 1024 ** -2
nn = len(objs)
print('{:<50s}:{} {} {} {}'.format(n, nn, s, prev, nn - prev))
return nn
elif inst:
t = lambda x: isinstance(x, inst)
n = str(inst)
objs = list(filter(t, gc.get_objects()))
s = sum(sys.getsizeof(o) for o in objs) * 1024 ** -2
print('{:<50s}:{} {}'.format(n, len(objs), s))
else:
objs = gc.get_objects()
key = lambda x: type(x)
objs = sorted(objs, key=key)
# for g, aa in groupby(objs, key=key):
# d = [sys.getsizeof(ai) for ai in aa]
# s = sum(d)
# print '{} {} {}'.format(g, len(d), s)
xx = [(g, [ai for ai in aa])
for g, aa in groupby(objs, key=key)]
xx = [(g, sum([sys.getsizeof(ai) for ai in aa]), len(aa))
for g, aa in xx]
for g, s, n in sorted(xx, key=lambda x: x[1]):
if s > 1000:
print('{:<50s} {} {}'.format(g, s, n))
if referrers:
for obj in objs:
show_referrers(obj)
if referents:
for obj in objs:
show_referents(obj)
def getImages(self, files):
def test(s):
return getExt(s.lower()) in ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'tbn']
files = set(filter(test, files))
images = {
'backdrop': set(filter(lambda s: re.search('(^|[\W_])fanart|backdrop\d*[\W_]', s.lower()) and self.filesizeBetween(s, self.file_sizes['backdrop']), files))
}
# Rest
images['rest'] = files - images['backdrop']
return images
def group_structures(self, s_list, anonymous=False):
"""
Given a list of structures, use fit to group
them by structural equality.
Args:
s_list ([Structure]): List of structures to be grouped
anonymous (bool): Wheher to use anonymous mode.
Returns:
A list of lists of matched structures
Assumption: if s1 == s2 but s1 != s3, than s2 and s3 will be put
in different groups without comparison.
"""
if self._subset:
raise ValueError("allow_subset cannot be used with"
" group_structures")
original_s_list = list(s_list)
s_list = self._process_species(s_list)
# Use structure hash to pre-group structures
if anonymous:
c_hash = lambda c: c.anonymized_formula
else:
c_hash = self._comparator.get_hash
s_hash = lambda s: c_hash(s[1].composition)
sorted_s_list = sorted(enumerate(s_list), key=s_hash)
all_groups = []
# For each pre-grouped list of structures, perform actual matching.
for k, g in itertools.groupby(sorted_s_list, key=s_hash):
unmatched = list(g)
while len(unmatched) > 0:
i, refs = unmatched.pop(0)
matches = [i]
if anonymous:
inds = filter(lambda i: self.fit_anonymous(refs,
unmatched[i][1]), list(range(len(unmatched))))
else:
inds = filter(lambda i: self.fit(refs, unmatched[i][1]),
list(range(len(unmatched))))
inds = list(inds)
matches.extend([unmatched[i][0] for i in inds])
unmatched = [unmatched[i] for i in range(len(unmatched))
if i not in inds]
all_groups.append([original_s_list[i] for i in matches])
return all_groups
def get_recognizer(args):
"""Get the file recognizer object from the configured options."""
# Make sure we have empty sets when we have empty strings.
skip_dirs = set(filter(None, args.skip_dirs.split(",")))
skip_exts = set(filter(None, args.skip_exts.split(",")))
fr = FileRecognizer(
skip_hidden_files=args.skip_hidden_files,
skip_backup_files=args.skip_backup_files,
skip_hidden_dirs=args.skip_hidden_dirs,
skip_dirs=skip_dirs,
skip_exts=skip_exts,
skip_symlink_files=not args.follow_symlinks,
skip_symlink_dirs=not args.follow_symlinks,
)
return fr
def inner(function):
if isinstance(ambiguous_keyname, list):
keynames = map(self._get_precise_keyname, filter(bool, ambiguous_keyname))
for keyname in keynames:
self._statusbar_dispatch_table[keyname] = function
elif isinstance(ambiguous_keyname, binary_type):
keynames = map(self._get_precise_keyname, filter(bool, BLANK.split(ambiguous_keyname)))
for keyname in keynames:
self._statusbar_dispatch_table[keyname] = function
elif isinstance(ambiguous_keyname, text_type):
keynames = map(self._get_precise_keyname, filter(bool, BLANK.split(b(ambiguous_keyname))))
for keyname in keynames:
self._statusbar_dispatch_table[keyname] = function
else:
raise(Exception("bad argument error"))
def _merge_options_wrapper(merge_options, first, second):
"""Send the vectors to merge_options with the right order.
Assuming that merge_options will apply second options on first, send the
less significant vector first. First checking the depths where the vectors
were found. If equal, fallback to check which vector has more categories,
and eventually check who has the most significant category configured.
Most significant category is considered the right-most category.
:param func merge_options: Function that merges options.
:param tuple first: A vector to merge.
:param tuple second: A vector to merge.
:returns: The output from merge_options.
"""
first_opts = first[-1]
second_opts = second[-1]
first_depth = first_opts[DepthLevel]
second_depth = second_opts[DepthLevel]
logger.debug(
'Vector depths (first, second) = (%s, %s)',
first_depth, second_depth
)
if first_depth > second_depth:
return merge_options(second_opts, first_opts)
elif second_depth > first_depth:
return merge_options(first_opts, second_opts)
# Depth level is equal
# Check who have more categories configured
first_weight = len(tuple(filter(None, first[:-1])))
second_weight = len(tuple(filter(None, second[:-1])))
logger.debug(
'Vector weights (first, second) = (%s, %s)',
first_weight, second_weight
)
if first_weight > second_weight:
return merge_options(second_opts, first_opts)
elif second_weight > first_weight:
return merge_options(first_opts, second_opts)
# Both have same amount of categories configured
# Check who has most significant category
for first_c, second_c in reversed(list(zip(first[:-1], second[:-1]))):
if first_c and not second_c:
logger.debug('First has most significant category: %s', first_c)
return merge_options(second_opts, first_opts)
if second_c and not first_c:
logger.debug('Second has most significant category: %s', second_c)
return merge_options(first_opts, second_opts)
return merge_options(first_opts, second_opts)
def test_greater_than_equal(self):
filter = self.get_filter('gte', 3)
for match in (3, 11, '2'):
self.assertTrue(filter({'foo': match}))
for non_match in (-10, 0, 2):
self.assertFalse(filter({'foo': non_match}))
def _parse_varnishadm(self, output, tags):
""" Parse out service checks from varnishadm.
Example output:
Backend b0 is Sick
Current states good: 2 threshold: 3 window: 5
Average responsetime of good probes: 0.000000
Oldest Newest
================================================================
-------------------------------------------------------------444 Good IPv4
-------------------------------------------------------------XXX Good Xmit
-------------------------------------------------------------RRR Good Recv
----------------------------------------------------------HHH--- Happy
Backend b1 is Sick
Current states good: 2 threshold: 3 window: 5
Average responsetime of good probes: 0.000000
Oldest Newest
================================================================
----------------------------------------------------------HHH--- Happy
Example output (new output format):
Backend name Admin Probe
boot.default probe Healthy (no probe)
boot.backend2 probe Healthy 4/4
Current states good: 4 threshold: 3 window: 4
Average response time of good probes: 0.002504
Oldest ================================================== Newest
--------------------------------------------------------------44 Good IPv4
--------------------------------------------------------------XX Good Xmit
--------------------------------------------------------------RR Good Recv
------------------------------------------------------------HHHH Happy
"""
# Process status by backend.
backends_by_status = defaultdict(list)
for line in output.split("\n"):
backend, status, message = None, None, None
# split string and remove all empty fields
tokens = filter(None, line.strip().split(' '))
tokens = [t for t in tokens]
if len(tokens):
if tokens == ['Backend', 'name', 'Admin', 'Probe']:
# skip the column headers that exist in new output format
continue
# parse new output format
# the backend name will include the vcl name
# so split on first . to remove prefix
elif len(tokens) >= 4 and tokens[1] in ['healthy', 'sick']:
# If the backend health was overriden, lets grab the
# overriden value instead of the probed health
backend = tokens[0].split('.', 1)[-1]
status = tokens[1].lower()
elif len(tokens) >= 4 and tokens[1] == 'probe':
backend = tokens[0].split('.', 1)[-1]
status = tokens[2].lower()
# Parse older Varnish backend output
elif tokens[0] == 'Backend':
backend = tokens[1]
status = tokens[-1].lower()
if tokens[0] == 'Current' and backend is not None:
try:
message = ' '.join(tokens[2:]).strip()
except Exception:
# If we can't parse a message still send a status.
self.log.exception(
'Error when parsing message from varnishadm')
message = ''
if backend is not None:
backends_by_status[status].append((backend, message))
for status, backends in iteritems(backends_by_status):
check_status = BackendStatus.to_check_status(status)
for backend, message in backends:
service_checks_tags = ['backend:%s' % backend] + tags
self.service_check(self.SERVICE_CHECK_NAME,
check_status,
tags=service_checks_tags,
message=message)
def test_equal(self):
match = 'match'
filter = self.get_filter('eq', match)
self.assertTrue(filter({'foo': match}))
self.assertFalse(filter({'foo': 'non-match'}))
self.assertFalse(filter({'foo': None}))
def __get_best_for_criteria(journeys, criteria):
return min_from_criteria(
filter(has_pt, journeys),
[criteria, duration_crit, transfers_crit, nonTC_crit])
def curl(i_cube, j_cube, k_cube=None, ignore=None):
r'''
Calculate the 3d curl of the given vector of cubes.
Args:
* i_cube
The i cube of the vector to operate on
* j_cube
The j cube of the vector to operate on
Kwargs:
* k_cube
The k cube of the vector to operate on
* ignore
This argument is not used.
.. deprecated:: 0.8
The coordinates to ignore are determined automatically.
Return (i_cmpt_curl_cube, j_cmpt_curl_cube, k_cmpt_curl_cube)
The calculation of curl is dependent on the type of
:func:`iris.coord_systems.CoordSystem` in the cube:
Cartesian curl
The Cartesian curl is defined as:
.. math::
\nabla\times \vec u =
(\frac{\delta w}{\delta y} - \frac{\delta v}{\delta z})\vec a_i
-
(\frac{\delta w}{\delta x} - \frac{\delta u}{\delta z})\vec a_j
+
(\frac{\delta v}{\delta x} - \frac{\delta u}{\delta y})\vec a_k
Spherical curl
When spherical calculus is used, i_cube is the phi vector
component (e.g. eastward), j_cube is the theta component
(e.g. northward) and k_cube is the radial component.
The spherical curl is defined as:
.. math::
\nabla\times \vec A = \frac{1}{r cos \theta}
(\frac{\delta}{\delta \theta}
(\vec A_\phi cos \theta) -
\frac{\delta \vec A_\theta}{\delta \phi}) \vec r +
\frac{1}{r}(\frac{1}{cos \theta}
\frac{\delta \vec A_r}{\delta \phi} -
\frac{\delta}{\delta r} (r \vec A_\phi))\vec \theta +
\frac{1}{r}
(\frac{\delta}{\delta r}(r \vec A_\theta) -
\frac{\delta \vec A_r}{\delta \theta}) \vec \phi
where phi is longitude, theta is latitude.
'''
if ignore is not None:
ignore = None
warnings.warn('The ignore keyword to iris.analysis.calculus.curl '
'is deprecated, ignoring is now done automatically.')
# Get the vector quantity names.
# (i.e. ['easterly', 'northerly', 'vertical'])
vector_quantity_names, phenomenon_name = \
spatial_vectors_with_phenom_name(i_cube, j_cube, k_cube)
cubes = filter(None, [i_cube, j_cube, k_cube])
# get the names of all coords binned into useful comparison groups
coord_comparison = iris.analysis.coord_comparison(*cubes)
bad_coords = coord_comparison['ungroupable_and_dimensioned']
if bad_coords:
raise ValueError("Coordinates found in one cube that describe "
"a data dimension which weren't in the other "
"cube ({}), try removing this coordinate.".format(
', '.join(group.name() for group in bad_coords)))
bad_coords = coord_comparison['resamplable']
if bad_coords:
raise ValueError('Some coordinates are different ({}), consider '
'resampling.'.format(
', '.join(group.name() for group in bad_coords)))
ignore_string = ''
if coord_comparison['ignorable']:
ignore_string = ' (ignoring {})'.format(
', '.join(group.name() for group in bad_coords))
# Get the dim_coord, or None if none exist, for the xyz dimensions
x_coord = i_cube.coord(axis='X')
y_coord = i_cube.coord(axis='Y')
z_coord = i_cube.coord(axis='Z')
y_dim = i_cube.coord_dims(y_coord)[0]
horiz_cs = i_cube.coord_system('CoordSystem')
# Planar (non spherical) coords?
ellipsoidal = isinstance(horiz_cs, (iris.coord_systems.GeogCS,
iris.coord_systems.RotatedGeogCS))
if not ellipsoidal:
# TODO Implement some mechanism for conforming to a common grid
dj_dx = _curl_differentiate(j_cube, x_coord)
prototype_diff = dj_dx
# i curl component (dk_dy - dj_dz)
dk_dy = _curl_differentiate(k_cube, y_coord)
dk_dy = _curl_regrid(dk_dy, prototype_diff)
dj_dz = _curl_differentiate(j_cube, z_coord)
dj_dz = _curl_regrid(dj_dz, prototype_diff)
# TODO Implement resampling in the vertical (which regridding
# does not support).
if dj_dz is not None and dj_dz.data.shape != prototype_diff.data.shape:
dj_dz = _curl_change_z(dj_dz, z_coord, prototype_diff)
i_cmpt = _curl_subtract(dk_dy, dj_dz)
dj_dz = dk_dy = None
# j curl component (di_dz - dk_dx)
di_dz = _curl_differentiate(i_cube, z_coord)
di_dz = _curl_regrid(di_dz, prototype_diff)
# TODO Implement resampling in the vertical (which regridding
# does not support).
if di_dz is not None and di_dz.data.shape != prototype_diff.data.shape:
di_dz = _curl_change_z(di_dz, z_coord, prototype_diff)
dk_dx = _curl_differentiate(k_cube, x_coord)
dk_dx = _curl_regrid(dk_dx, prototype_diff)
j_cmpt = _curl_subtract(di_dz, dk_dx)
di_dz = dk_dx = None
# k curl component ( dj_dx - di_dy)
di_dy = _curl_differentiate(i_cube, y_coord)
di_dy = _curl_regrid(di_dy, prototype_diff)
# Since prototype_diff == dj_dx we don't need to recalculate dj_dx
# dj_dx = _curl_differentiate(j_cube, x_coord)
# dj_dx = _curl_regrid(dj_dx, prototype_diff)
k_cmpt = _curl_subtract(dj_dx, di_dy)
di_dy = dj_dx = None
result = [i_cmpt, j_cmpt, k_cmpt]
# Spherical coords (GeogCS or RotatedGeogCS).
else:
# A_\phi = i ; A_\theta = j ; A_\r = k
# theta = lat ; phi = long ;
# r_cmpt = 1 / (r * cos(lat)) *
# (d/dtheta (i_cube * sin(lat)) - d_j_cube_dphi)
# phi_cmpt = 1/r * ( d/dr (r * j_cube) - d_k_cube_dtheta)
# theta_cmpt = 1/r * ( 1/cos(lat) * d_k_cube_dphi - d/dr (r * i_cube)
if y_coord.name() not in ['latitude', 'grid_latitude'] \
or x_coord.name() not in ['longitude', 'grid_longitude']:
raise ValueError('Expecting latitude as the y coord and '
'longitude as the x coord for spherical curl.')
# Get the radius of the earth - and check for sphericity
ellipsoid = horiz_cs
if isinstance(horiz_cs, iris.coord_systems.RotatedGeogCS):
ellipsoid = horiz_cs.ellipsoid
if ellipsoid:
# TODO: Add a test for this
r = ellipsoid.semi_major_axis
r_unit = cf_units.Unit("m")
spherical = (ellipsoid.inverse_flattening == 0.0)
else:
r = DEFAULT_SPHERICAL_EARTH_RADIUS
r_unit = DEFAULT_SPHERICAL_EARTH_RADIUS_UNIT
spherical = True
if not spherical:
raise ValueError('Cannot take the curl over a non-spherical '
'ellipsoid.')
lon_coord = x_coord.copy()
lat_coord = y_coord.copy()
lon_coord.convert_units('radians')
lat_coord.convert_units('radians')
lat_cos_coord = _coord_cos(lat_coord)
# TODO Implement some mechanism for conforming to a common grid
temp = iris.analysis.maths.multiply(i_cube, lat_cos_coord, y_dim)
dicos_dtheta = _curl_differentiate(temp, lat_coord)
prototype_diff = dicos_dtheta
# r curl component: 1 / (r * cos(lat)) * (d_j_cube_dphi - dicos_dtheta)
# Since prototype_diff == dicos_dtheta we don't need to
# recalculate dicos_dtheta.
d_j_cube_dphi = _curl_differentiate(j_cube, lon_coord)
d_j_cube_dphi = _curl_regrid(d_j_cube_dphi, prototype_diff)
new_lat_coord = d_j_cube_dphi.coord(axis='Y')
new_lat_cos_coord = _coord_cos(new_lat_coord)
lat_dim = d_j_cube_dphi.coord_dims(new_lat_coord)[0]
r_cmpt = iris.analysis.maths.divide(_curl_subtract(d_j_cube_dphi,
dicos_dtheta),
r * new_lat_cos_coord, dim=lat_dim)
r_cmpt.units = r_cmpt.units / r_unit
d_j_cube_dphi = dicos_dtheta = None
# phi curl component: 1/r * ( drj_dr - d_k_cube_dtheta)
drj_dr = _curl_differentiate(r * j_cube, z_coord)
if drj_dr is not None:
drj_dr.units = drj_dr.units * r_unit
drj_dr = _curl_regrid(drj_dr, prototype_diff)
d_k_cube_dtheta = _curl_differentiate(k_cube, lat_coord)
d_k_cube_dtheta = _curl_regrid(d_k_cube_dtheta, prototype_diff)
if drj_dr is None and d_k_cube_dtheta is None:
phi_cmpt = None
else:
phi_cmpt = 1/r * _curl_subtract(drj_dr, d_k_cube_dtheta)
phi_cmpt.units = phi_cmpt.units / r_unit
drj_dr = d_k_cube_dtheta = None
# theta curl component: 1/r * ( 1/cos(lat) * d_k_cube_dphi - dri_dr )
d_k_cube_dphi = _curl_differentiate(k_cube, lon_coord)
d_k_cube_dphi = _curl_regrid(d_k_cube_dphi, prototype_diff)
if d_k_cube_dphi is not None:
d_k_cube_dphi = iris.analysis.maths.divide(d_k_cube_dphi,
lat_cos_coord)
dri_dr = _curl_differentiate(r * i_cube, z_coord)
if dri_dr is not None:
dri_dr.units = dri_dr.units * r_unit
dri_dr = _curl_regrid(dri_dr, prototype_diff)
if d_k_cube_dphi is None and dri_dr is None:
theta_cmpt = None
else:
theta_cmpt = 1/r * _curl_subtract(d_k_cube_dphi, dri_dr)
theta_cmpt.units = theta_cmpt.units / r_unit
d_k_cube_dphi = dri_dr = None
result = [phi_cmpt, theta_cmpt, r_cmpt]
for direction, cube in zip(vector_quantity_names, result):
if cube is not None:
cube.rename('%s curl of %s' % (direction, phenomenon_name))
return result
def parent_module(item):
return next(filter(lambda x: isinstance(x, Module), parents_of(item)))
def wrapper(request, *args, **kwargs):
from . import models
path = kwargs.pop('path', None)
article_id = kwargs.pop('article_id', None)
urlpath = None
# fetch by urlpath.path
if not path is None:
try:
urlpath = models.URLPath.get_by_path(path, select_related=True)
except NoRootURL:
return redirect('wiki:root_create')
except models.URLPath.DoesNotExist:
try:
pathlist = list(
filter(
lambda x: x != "",
path.split("/"),
))
path = "/".join(pathlist[:-1])
parent = models.URLPath.get_by_path(path)
return HttpResponseRedirect(
reverse("wiki:create", kwargs={
'path': parent.path,
}) + "?slug=%s" % pathlist[-1])
except models.URLPath.DoesNotExist:
return HttpResponseNotFound(
render_to_string(
"wiki/error.html",
context={'error_type': 'ancestors_missing'},
request=request))
if urlpath.article:
# urlpath is already smart about prefetching items on article
# (like current_revision), so we don't have to
article = urlpath.article
else:
# Be robust: Somehow article is gone but urlpath exists...
# clean up
return_url = reverse('wiki:get',
kwargs={'path': urlpath.parent.path})
urlpath.delete()
return HttpResponseRedirect(return_url)
# fetch by article.id
elif article_id:
# TODO We should try to grab the article form URLPath so the
# caching is good, and fall back to grabbing it from
# Article.objects if not
articles = models.Article.objects
article = get_object_or_404(articles, id=article_id)
try:
urlpath = models.URLPath.objects.get(articles__article=article)
except models.URLPath.DoesNotExist as noarticle:
models.URLPath.MultipleObjectsReturned = noarticle
urlpath = None
else:
raise TypeError('You should specify either article_id or path')
if not deleted_contents:
# If the article has been deleted, show a special page.
if urlpath:
if urlpath.is_deleted(): # This also checks all ancestors
return redirect('wiki:deleted', path=urlpath.path)
else:
if article.current_revision and article.current_revision.deleted:
return redirect('wiki:deleted', article_id=article.id)
if article.current_revision.locked and not_locked:
return response_forbidden(request, article, urlpath)
if can_read and not article.can_read(request.user):
return response_forbidden(request, article, urlpath)
if (can_write or can_create) and not article.can_write(request.user):
return response_forbidden(request, article, urlpath)
if can_create and not (request.user.is_authenticated()
or settings.ANONYMOUS_CREATE):
return response_forbidden(request, article, urlpath)
if can_delete and not article.can_delete(request.user):
return response_forbidden(request, article, urlpath)
if can_moderate and not article.can_moderate(request.user):
return response_forbidden(request, article, urlpath)
kwargs['urlpath'] = urlpath
return func(request, article, *args, **kwargs)
def backward_function_tester(rng, func, inputs=None,
func_args=[], func_kwargs={},
rtol_accum=1e-5,
atol_f=1e-4, atol_b=1e-3, atol_accum=5e-2,
dstep=1e-3, backward=None, backward_b=None,
ctx=None, non_accum_check=False, skip_backward_check=False, insert_identity=[],
auto_forward=False):
""" Automatic testing of backward function and backward pass of `func` by comparing it.
The backward pass of `func` is the reference; therefore,
the backward pass of `func` must be tested first!
Syntax of `ref_func`: inputs, parameters
"""
if ctx is None:
ctx = nn.Context()
if backward is None:
backward = [True for _ in inputs]
if pf.machine() == "aarch64":
rtol_accum = 1e-3
def create_variables(inputs, backward):
vinputs = []
for i, b in zip(inputs, backward):
if i is None:
vinputs += [None]
continue
vinp = nn.Variable(i.shape, need_grad=b)
vinp.grad.zero() # grads always not accumulation
vinputs += [vinp]
vinputs[-1].data.cast(i.dtype)[...] = i
return vinputs
vinputs = create_variables(inputs, backward)
vinputs_for_clear_buffer = create_variables(inputs, backward)
vinputs_for_nn_grad = create_variables(inputs, backward)
vinputs_identity = []
vinputs_identity_for_clear_buffer = []
vinputs_identity_for_nn_grad = []
if not insert_identity:
insert_identity = [True] * len(vinputs)
for idx, i in enumerate(zip(vinputs, vinputs_for_clear_buffer, vinputs_for_nn_grad)):
with nn.auto_forward(auto_forward):
i0, i1, i2 = i
if i0 is None:
vinputs_identity += [None]
vinputs_identity_for_clear_buffer += [None]
vinputs_identity_for_nn_grad += [None]
elif insert_identity[idx]:
vinputs_identity += [F.identity(i0)]
vinputs_identity_for_clear_buffer += [F.identity(i1)]
vinputs_identity_for_nn_grad += [F.identity(i2)]
else:
vinputs_identity += [i0]
vinputs_identity_for_clear_buffer += [i1]
vinputs_identity_for_nn_grad += [i2]
# Forward and backward of the forward function with no buffer clear
with nn.context_scope(ctx), nn.auto_forward(auto_forward):
outputs0 = func(*(vinputs_identity + func_args), **func_kwargs)
outputs0 = force_list(outputs0)
F.sink(*outputs0).forward(clear_no_need_grad=False)
grad_voutputs = []
for output in outputs0:
ograd = rng.randn(*output.shape)
grad_voutputs.append(nn.Variable.from_numpy_array(
ograd).apply(need_grad=True))
output.g = ograd
F.sink(*outputs0, one_input_grad=False).backward()
vinputs = list(filter(lambda x: x is not None, vinputs))
vinputs_identity = list(filter(lambda x: x is not None, vinputs_identity))
vinputs_for_clear_buffer = list(
filter(lambda x: x is not None, vinputs_for_clear_buffer))
grad_inputs0 = [inp.g.copy() for inp in vinputs]
# Forward and backward of the forward function with clear redundant buffer
with nn.context_scope(ctx), nn.auto_forward(auto_forward):
outputs_for_clear_buffer = func(
*(vinputs_identity_for_clear_buffer + func_args), **func_kwargs)
outputs_for_clear_buffer = force_list(outputs_for_clear_buffer)
outputs_for_clear_buffer = list(map(lambda x: F.identity(
x) if x is not None else None, outputs_for_clear_buffer))
F.sink(*outputs_for_clear_buffer).forward(clear_no_need_grad=True)
for o, ref_o in zip(outputs_for_clear_buffer, outputs0):
o.g = ref_o.g
# Check backward
F.sink(*outputs_for_clear_buffer,
one_input_grad=False).backward(clear_buffer=True)
grad_inputs_for_clear_buffer = [inp.g.copy()
for inp in vinputs_for_clear_buffer]
for grad_ref, grad_res in zip(grad_inputs0, grad_inputs_for_clear_buffer):
if grad_ref is None or grad_res is None:
continue
assert_allclose(grad_ref, grad_res, atol=atol_f,
err_msg="backward(clear_buffer=True) and backward(clear_buffer=False) results differ.")
# Forward of the backward function
from nnabla.backward_functions import registry
func_name = output.parent.info.type_name
func_backward = registry[func_name]
grad_vinputs = grad_voutputs + vinputs
grad_vinputs_identity = grad_voutputs + vinputs_identity
func_info_args = output.parent.info.args
with nn.context_scope(ctx), nn.auto_forward(auto_forward):
ograds0 = func_backward(grad_vinputs_identity, **func_info_args)
ograds0 = force_list(ograds0)
ograds0_ = list(filter(lambda o: o is not None, ograds0))
F.sink(*ograds0_).forward(clear_no_need_grad=True)
outputs1 = []
for i, ograd in enumerate(ograds0):
outputs1.append(ograd.d.copy()) if ograd is not None else \
outputs1.append(None)
# Check num of returned elements
assert_allclose(len(vinputs), len(outputs1),
err_msg="Length of the outputs ({}) does not match "
"the length of the inputs ({}) to the backward function".format(len(outputs1), len(vinputs)))
# Check forward
for i, elm in enumerate(zip(grad_inputs0, outputs1)):
grad_ref, grad_res = elm
if grad_ref is None or grad_res is None:
continue
assert_allclose(grad_ref, grad_res, atol=atol_f,
err_msg="Forward of the backward function ({}) fails at {}-th output.".format(
func_backward.__name__, i))
# Check the same results between backward_function and nn.grad
vinputs = [v for b, v in zip(backward, vinputs) if b]
vinputs = list(filter(lambda x: x is not None, vinputs))
with nn.context_scope(ctx), nn.auto_forward(auto_forward):
outputs0_for_nn_grad = func(
*(vinputs_identity_for_nn_grad + func_args), **func_kwargs)
outputs0_for_nn_grad = force_list(outputs0_for_nn_grad)
vinputs_identity_for_nn_grad = [v for b, v in zip(
backward, vinputs_identity_for_nn_grad) if b]
vinputs_identity_for_nn_grad = list(
filter(lambda x: x is not None, vinputs_identity_for_nn_grad))
ograds1 = nn.grad(outputs0_for_nn_grad, vinputs_identity_for_nn_grad,
grad_outputs=[g.d.copy() for g in grad_voutputs])
F.sink(*ograds1).forward(clear_no_need_grad=True)
ograds0 = list(filter(lambda o: o is not None, ograds0))
ograds1 = list(filter(lambda o: o is not None, ograds1))
for i in range(len(ograds0)):
if ograds0[i].parent is None:
continue
assert_allclose(ograds0[i].d, ograds1[i].d, atol=atol_f,
err_msg="nn.grad and backward_functon results differ.")
# Check backward
# needed since we sometimes do need_grad=False for optimization, e.g., mask.
def set_inputs(inputs0, vinputs):
begin = 0
for i in vinputs:
end = begin + i.size
i.d = inputs0[begin:end].reshape(i.shape)
begin = end
def obj_func(inputs0, voutput, vinputs):
set_inputs(inputs0, vinputs)
voutput.forward()
y = voutput.d.copy()
return y
initial_grads = []
for grad_vinput in grad_vinputs:
if grad_vinput is None:
continue
g = np.asarray(rng.randn(*grad_vinput.shape))
initial_grads.append(g)
grad_inputs1 = np.concatenate([v.d.flatten()
for v in grad_vinputs if v is not None])
for i, ograd in enumerate(ograds0):
# We can skip if the backward is the functions composite.
# If the backward is of functions composite,
# the numerical difference is really different from the analytical one for some functions.
if skip_backward_check:
continue
if ograd is None or not backward[i]:
continue
for ig, v in zip(initial_grads, grad_vinputs):
v.g = ig
# This must be first since approx_fprime destroys the input values
# analytical grad.
rgrad = rng.randn()
with nn.auto_forward(auto_forward):
sum_ograd = F.sum(ograd) * rgrad
sum_ograd.forward(clear_no_need_grad=True)
sum_ograd.backward()
analytical_grads = np.concatenate(
[v.g.flatten() for v in grad_vinputs])
analytical_grads -= np.concatenate([g.flatten()
for g in initial_grads])
# numerical grad
from scipy.optimize import approx_fprime
numerical_grads = approx_fprime(
grad_inputs1, obj_func, dstep, sum_ograd, grad_vinputs)
# grad_vinputs: dy_1, ..., dy_n, x_1, ..., x_n
# grad_voutputs: dy_1, ..., dy_n
seps = [0] + np.cumsum([int(np.prod(v.shape))
for v in grad_vinputs]).tolist()
ngrads = len(grad_voutputs)
ninputs = len(grad_vinputs)
backward_b = [True] * ninputs if backward_b is None else backward_b
for k, sep in enumerate(zip(seps[:-1], seps[1:])):
if k >= ngrads and not backward[k - ngrads] or not backward_b[k]:
continue
s0, s1 = sep
analytical_grad = analytical_grads[s0:s1]
numerical_grad = numerical_grads[s0:s1]
assert_allclose(analytical_grad, numerical_grad, rtol=rtol_accum, atol=atol_accum,
err_msg="Backward (accum) of the backward function ({}) wrt {}-th / {} input fails.".format(
func_backward.__name__, k, ninputs))
# Some functions backward like AffineDataGrad and AffineFilterGrad does not check non-accum anywhere
# so check those non-accum backward method here.
if non_accum_check:
# for any outputs, parents are the same function.
parent = outputs0[0].parent
inputs = parent.inputs
# Accum
initial_grads = np.concatenate(
[inp.g.flatten() for inp, b in zip(inputs, backward) if b])
accum = [True] * len(inputs)
parent.backward(inputs, outputs0, accum=accum)
accum_grads = np.concatenate([inp.g.flatten()
for inp, b in zip(inputs, backward) if b])
non_accum_grads0 = accum_grads - initial_grads
# Non-accum
accum = [False] * len(inputs)
parent.backward(inputs, outputs0, accum=accum)
non_accum_grads1 = np.concatenate(
[inp.g.flatten() for inp, b in zip(inputs, backward) if b])
# Check
assert_allclose(non_accum_grads0, non_accum_grads1, atol=atol_b,
err_msg="Backward (non-accum) of the backward function ({}) fails.".format(
func_backward.__name__))
def function_tester(rng, func, ref_func, inputs,
func_args=[], func_kwargs={},
atol_f=1e-6, atol_b=1e-3, atol_accum=1e-6, dstep=1e-3, backward=None,
ctx=None, func_name=None, ref_grad=None, disable_half_test=False, atol_half=1e-1,
insert_identity=[], disable_clear_no_need_grad_test=False, auto_forward=False):
""" Automatic testing of forward/backward pass of `func` by comparing it
to the reference implementation in `ref_func`.
Syntax of `ref_func`: inputs, parameters
Syntax of `ref_grad`: inputs, output grads, parameters
"""
if ctx is None:
ctx = nn.Context()
if backward is None:
backward = [True for _ in inputs]
# Create Variables
# print('create_variable')
def create_variables(inputs, backward):
vinputs = []
for i, b in zip(inputs, backward):
if i is None:
vinputs += [None]
continue
vinputs += [nn.Variable(i.shape, need_grad=b)]
vinputs[-1].data.cast(i.dtype)[...] = i
return vinputs
# Half test
if not disable_half_test:
finputs = create_variables(inputs, backward)
hinputs = create_variables(inputs, backward)
half_test(rng, func, finputs, hinputs, func_args,
func_kwargs, backward, ctx, func_name, atol=atol_half)
vinputs = create_variables(inputs, backward)
# Checking forward
# print('checking forward')
with nn.context_scope(ctx), nn.auto_forward():
o = func(*(vinputs + func_args), **func_kwargs)
rinputs = copy.deepcopy(inputs) # inputs for ref_func
refs = ref_func(*(rinputs + func_args), **func_kwargs)
refs = force_tuple(refs)
o = force_tuple(o)
assert len(o) == len(refs)
for i, ref in enumerate(refs):
res = o[i].d
assert_allclose(ref, res, atol=atol_f,
err_msg="{} forward test fails".format(func_name))
# Checking recomputation
vinputs = create_variables(inputs, backward)
recomputation_test(rng, func, vinputs, func_args,
func_kwargs, ctx)
# Checking forward(clear_no_need_grad=True)
if not disable_clear_no_need_grad_test:
clear_no_need_grad_tester(rng, func, inputs, func_args, func_kwargs,
backward, atol_f, ctx, func_name, insert_identity,
auto_forward)
# Checking function name
try:
import function_test_callback
result = create_function_nnp(
vinputs, o, func_name, func_args, func_kwargs)
if result is not None:
function_test_callback.callback(func_name, *result)
except UnboundLocalError:
pass
except IndexError:
pass
except ImportError:
pass
# print('checking function name')
if func_name is not None:
assert o[0].parent.name == func_name
# Checking backward
# print('checking backward')
if not True in backward:
return
# NNabla backward
for v in vinputs:
if v is None:
continue
if len(v.shape) == 0:
v.g = randn(rng)
continue
v.g = randn(rng, *v.shape)
# Verify grad
vinputs = create_variables(inputs, backward)
rinputs = copy.deepcopy(inputs)
rinputs = [rinput if test else None for rinput,
test in zip(rinputs, backward)]
vgrads = [randn(rng, *o_.shape) for o_ in o]
def reset_ograds():
'''
Reset output grads everytime we call backward.
This is required because the output grad might
be inplaced and modified during backward operation.
'''
for ovar, g in zip(o, vgrads):
ovar.g = g
agrads, ngrads = compute_analytical_and_numerical_grad(
o[0].parent, vinputs, o, rinputs, vgrads, epsilon=dstep,
rng=rng, ref_grad=ref_grad)
if ref_grad is not None:
rinputs = copy.deepcopy(inputs)
doutputs = copy.deepcopy(vgrads)
ngrads = ref_grad(*(rinputs + doutputs + func_args),
**func_kwargs, need_grad_flags=backward)
assert_allclose(ngrads, agrads, atol=atol_b,
err_msg="{} backward w/o accumulation test fails".format(func_name))
# Check if need_grad works
for v, b in zip(vinputs, backward):
if not b or v is None:
continue
v.grad.zero()
v.need_grad = False
reset_ograds()
try:
o[0].parent.forward(
list(filter(lambda x: x is not None, vinputs)), o)
o[0].parent.backward(
list(filter(lambda x: x is not None, vinputs)), o)
except RuntimeError as e:
continue # TODO
assert np.all(v.g == 0)
# test accum=False
for i in range(len(vinputs)):
if vinputs[i] is None:
continue
v = vinputs[i]
v.need_grad = backward[i]
for i in range(len(vinputs)):
if vinputs[i] is None:
continue
v = vinputs[i]
if not backward[i]:
continue
f = o[0].parent
# Prepare function inputs
finputs = list(filter(lambda x: x is not None, vinputs))
# Save accum gradient result
g = randn(rng, *v.shape)
v.g = g
reset_ograds()
f.forward(finputs, o)
f.backward(finputs, o)
true_g = v.g - g
# Check accum=False
accum = [j != i for j, vv in enumerate(vinputs) if vv is not None]
v.g = randn(rng, *v.shape)
reset_ograds()
f.forward(finputs, o)
f.backward(finputs, o, accum)
assert_allclose(
v.g, true_g, atol=atol_accum,
err_msg="{} backward w/ accumulation test fails.".format(func_name))
# Check accum=False with NaN gradient
v.g = np.float32('nan')
reset_ograds()
f.forward(finputs, o)
f.backward(finputs, o, accum)
assert not np.any(np.isnan(v.g))
def clear_no_need_grad_tester(rng, func, inputs, func_args=[], func_kwargs={}, backward=None, atol_f=1e-6, ctx=None,
func_name=None, insert_identity=[], auto_forward=False):
if ctx is None:
ctx = nn.Context()
if backward is None:
backward = [True for _ in inputs]
if not True in backward:
return
state_rng = None
if rng is not None:
state_rng = rng.get_state()
else:
rng = rng = np.random.RandomState(313)
def create_variables(inputs, backward):
vinputs = []
for i, b in zip(inputs, backward):
if i is None:
vinputs += [None]
continue
vinputs += [nn.Variable(i.shape, need_grad=b)]
vinputs[-1].data.cast(i.dtype)[...] = i
return vinputs
vinputs = create_variables(inputs, backward)
vinputs_clear_buffer = create_variables(inputs, backward)
vinputs_identity_clear_buffer = []
if not insert_identity:
insert_identity = [True] * len(vinputs)
with nn.context_scope(ctx), nn.auto_forward(auto_forward):
for idx, i in enumerate(vinputs_clear_buffer):
if i is None:
vinputs_identity_clear_buffer += [None]
elif insert_identity[idx]:
vinputs_identity_clear_buffer += [F.identity(i)]
else:
vinputs_identity_clear_buffer += [i]
# Checking forward(clear_no_need_grad=True)
with nn.context_scope(ctx), nn.auto_forward(auto_forward):
o = func(*(vinputs + func_args), **func_kwargs)
o = force_tuple(o)
F.sink(*o).forward(clear_no_need_grad=False)
o_clear_buffer = func(
*(vinputs_identity_clear_buffer + func_args), **func_kwargs)
o_clear_buffer = force_tuple(o_clear_buffer)
o_identity_clear_buffer = list(map(lambda x: F.identity(
x) if x is not None else None, o_clear_buffer))
o_identity_clear_buffer = list(
filter(lambda x: x is not None, o_identity_clear_buffer))
F.sink(*o_identity_clear_buffer).forward(clear_no_need_grad=True)
for i in range(len(o)):
if o[i] is None:
continue
ref = o[i].d
res = o_identity_clear_buffer[i].d
assert_allclose(ref, res, atol=atol_f,
err_msg="{} forward(clear_no_need_grad=True) test fails".format(func_name))
vinputs = list(filter(lambda x: x is not None, vinputs))
vinputs_clear_buffer = list(
filter(lambda x: x is not None, vinputs_clear_buffer))
for i in range(len(vinputs)):
vinputs[i].grad.zero()
vinputs_clear_buffer[i].grad.zero()
for i in range(len(o)):
if o[i] is None:
continue
o[i].g = randn(rng, *o[i].shape)
o_identity_clear_buffer[i].g = o[i].g
F.sink(*o).backward()
F.sink(*o_identity_clear_buffer).backward(clear_buffer=True)
for i in range(len(vinputs)):
ref = vinputs[i].g
res = vinputs_clear_buffer[i].g
assert_allclose(ref, res, atol=atol_f,
err_msg="{} forward(clear_no_need_grad=True) and backward test fails".format(func_name))
if state_rng:
rng.set_state(state_rng)
def _nn(self, d, n=1):
"""
Internal method to be implemented by sub-classes to return the nearest
`N` neighbors to the given descriptor element.
When this internal method is called, we have already checked that there
is a vector in ``d`` and our index is not empty.
:param d: Descriptor element to compute the neighbors of.
:type d: smqtk.representation.DescriptorElement
:param n: Number of nearest neighbors to find.
:type n: int
:return: Tuple of nearest N DescriptorElement instances, and a tuple of
the distance values to those neighbors.
:rtype: (tuple[smqtk.representation.DescriptorElement], tuple[float])
"""
log = self._log
q = d.vector()[np.newaxis, :].astype(np.float32)
log.debug("Received query for %d nearest neighbors", n)
with self._model_lock:
# Attempt to set n-probe of an IVF index
self._set_index_nprobe()
# noinspection PyArgumentList
s_dists, s_ids = self._faiss_index.search(
q, k=min(n, self._faiss_index.ntotal))
s_dists, s_ids = np.sqrt(s_dists[0, :]), s_ids[0, :]
s_ids = s_ids.astype(object)
# s_id (the FAISS index indices) can equal -1 if fewer than the
# requested number of nearest neighbors is returned. In this case,
# eliminate the -1 entries
self._log.debug("Getting descriptor UIDs from idx2uid mapping.")
uuids = list(
self._idx2uid_kvs.get_many(
filter(lambda s_id_: s_id_ >= 0, s_ids)))
if len(uuids) < n:
warnings.warn(
"Less than n={} neighbors were retrieved from "
"the FAISS index instance. Maybe increase "
"nprobe if this is an IVF index?".format(n),
RuntimeWarning)
descriptors = tuple(
self._descriptor_set.get_many_descriptors(uuids))
log.debug("Min and max FAISS distances: %g, %g", min(s_dists),
max(s_dists))
d_vectors = np.vstack(DescriptorElement.get_many_vectors(descriptors))
d_dists = metrics.euclidean_distance(d_vectors, q)
log.debug("Min and max descriptor distances: %g, %g", min(d_dists),
max(d_dists))
order = d_dists.argsort()
uuids, d_dists = zip(*((uuids[oidx], d_dists[oidx]) for oidx in order))
log.debug("Returning query result of size %g", len(uuids))
return descriptors, tuple(d_dists)
def get_images_info(self):
command = 'docker stack services --format {{.Image}} %s' % self.name
images = filter(None, fabricio.run(command).splitlines())
bucket = json.dumps(self._get_digests(images), sort_keys=True)
return b64encode(bucket.encode()).decode()
def __init__(
self,
worker_id,
conf,
namespaces=None,
pollster_list=None,
):
namespaces = namespaces or ['compute', 'central']
pollster_list = pollster_list or []
group_prefix = conf.polling.partitioning_group_prefix
# features of using coordination and pollster-list are exclusive, and
# cannot be used at one moment to avoid both samples duplication and
# samples being lost
if pollster_list and conf.coordination.backend_url:
raise PollsterListForbidden()
super(AgentManager, self).__init__(worker_id, conf)
def _match(pollster):
"""Find out if pollster name matches to one of the list."""
return any(
fnmatch.fnmatch(pollster.name, pattern)
for pattern in pollster_list)
if type(namespaces) is not list:
namespaces = [namespaces]
# we'll have default ['compute', 'central'] here if no namespaces will
# be passed
extensions = (self._extensions('poll', namespace, self.conf).extensions
for namespace in namespaces)
# get the extensions from pollster builder
extensions_fb = (self._extensions_from_builder('poll', namespace)
for namespace in namespaces)
if pollster_list:
extensions = (moves.filter(_match, exts) for exts in extensions)
extensions_fb = (moves.filter(_match, exts)
for exts in extensions_fb)
self.extensions = list(itertools.chain(*list(extensions))) + list(
itertools.chain(*list(extensions_fb)))
if self.extensions == []:
raise EmptyPollstersList()
discoveries = (self._extensions('discover', namespace,
self.conf).extensions
for namespace in namespaces)
self.discoveries = list(itertools.chain(*list(discoveries)))
self.polling_periodics = None
self.partition_coordinator = coordination.PartitionCoordinator(
self.conf)
self.heartbeat_timer = utils.create_periodic(
target=self.partition_coordinator.heartbeat,
spacing=self.conf.coordination.heartbeat,
run_immediately=True)
# Compose coordination group prefix.
# We'll use namespaces as the basement for this partitioning.
namespace_prefix = '-'.join(sorted(namespaces))
self.group_prefix = ('%s-%s' % (namespace_prefix, group_prefix)
if group_prefix else namespace_prefix)
self.notifier = oslo_messaging.Notifier(
messaging.get_transport(self.conf),
driver=self.conf.publisher_notifier.telemetry_driver,
publisher_id="ceilometer.polling")
self._keystone = None
self._keystone_last_exception = None
def __str__(self):
return '/'.join(filter(bool, [self.service, self.version, self.id]))
def gettile(self,
base_url=None,
layer=None,
style=None,
format=None,
tilematrixset=None,
tilematrix=None,
row=None,
column=None,
**kwargs):
"""Return a tile from the WMTS.
Returns the tile image as a file-like object.
Parameters
----------
base_url : string
Optional URL for request submission. Defaults to the URL of
the GetTile operation as declared in the GetCapabilities
response.
layer : string
Content layer name.
style : string
Optional style name. Defaults to the first style defined for
the relevant layer in the GetCapabilities response.
format : string
Optional output image format, such as 'image/jpeg'.
Defaults to the first format defined for the relevant layer
in the GetCapabilities response.
tilematrixset : string
Optional name of tile matrix set to use.
Defaults to the first tile matrix set defined for the
relevant layer in the GetCapabilities response.
tilematrix : string
Name of the tile matrix to use.
row : integer
Row index of tile to request.
column : integer
Column index of tile to request.
**kwargs : extra arguments
anything else e.g. vendor specific parameters
Example
-------
>>> url = 'http://map1c.vis.earthdata.nasa.gov/wmts-geo/wmts.cgi'
>>> wmts = WebMapTileService(url)
>>> img = wmts.gettile(layer='VIIRS_CityLights_2012',\
tilematrixset='EPSG4326_500m',\
tilematrix='6',\
row=4, column=4)
>>> out = open('tile.jpg', 'wb')
>>> bytes_written = out.write(img.read())
>>> out.close()
"""
vendor_kwargs = self.vendor_kwargs or {}
vendor_kwargs.update(kwargs)
# REST only WMTS
if self.restonly:
resurl = self.buildTileResource(layer, style, format,
tilematrixset, tilematrix, row,
column, **vendor_kwargs)
u = openURL(resurl, username=self.username, password=self.password)
return u
# KVP implemetation
data = self.buildTileRequest(layer, style, format, tilematrixset,
tilematrix, row, column, **vendor_kwargs)
if base_url is None:
base_url = self.url
try:
methods = self.getOperationByName('GetTile').methods
get_verbs = [
x for x in methods if x.get('type').lower() == 'get'
]
if len(get_verbs) > 1:
# Filter by constraints
base_url = next(x for x in filter(list, ([
pv.get('url') for const in pv.get('constraints')
if 'kvp' in [x.lower() for x in const.values]
] for pv in get_verbs if pv.get('constraints'))))[0]
elif len(get_verbs) == 1:
base_url = get_verbs[0].get('url')
except StopIteration:
pass
u = openURL(base_url,
data,
username=self.username,
password=self.password)
# check for service exceptions, and return
if u.info()['Content-Type'] == 'application/vnd.ogc.se_xml':
se_xml = u.read()
se_tree = etree.fromstring(se_xml)
err_message = six.text_type(se_tree.find('ServiceException').text)
raise ServiceException(err_message.strip(), se_xml)
return u
def test_filter():
from six.moves import filter
f = filter(lambda x: x % 2, range(10))
assert six.advance_iterator(f) == 1
def splits(text):
return list(filter(lambda s: len(s) != 0, re.split('\s+', text)))
def test_complex_structure(self):
# in slightly more compact format:
# ((foo=bar) or (foo1=bar1 and foo2=bar2 and (foo3=bar3 or foo4=bar4)))
filter = FilterFactory.from_spec({
"type":
"or",
"filters": [
{
"type": "property_match",
"property_name": "foo",
"property_value": "bar"
},
{
"type":
"and",
"filters": [
{
"type": "property_match",
"property_name": "foo1",
"property_value": "bar1"
},
{
"type": "property_match",
"property_name": "foo2",
"property_value": "bar2"
},
{
"type":
"or",
"filters": [{
"type": "property_match",
"property_name": "foo3",
"property_value": "bar3"
}, {
"type": "property_match",
"property_name": "foo4",
"property_value": "bar4"
}]
},
]
},
]
})
# first level or
self.assertTrue(filter(dict(foo='bar')))
# first level and with both or's
self.assertTrue(filter(dict(foo1='bar1', foo2='bar2', foo3='bar3')))
self.assertTrue(filter(dict(foo1='bar1', foo2='bar2', foo4='bar4')))
# first and not right
self.assertFalse(
filter(dict(foo1='not bar1', foo2='bar2', foo3='bar3')))
# second and not right
self.assertFalse(
filter(dict(foo1='bar1', foo2='not bar2', foo3='bar3')))
# last and not right
self.assertFalse(
filter(
dict(foo1='bar1',
foo2='bar2',
foo3='not bar3',
foo4='not bar4')))
def type_journeys(resp, req):
"""
Set the type of the journeys
"""
best_crit = arrival_crit if req["clockwise"] else departure_crit
# first, we want a type for every journey. Just pick "rapid" by default.
for j in resp.journeys:
j.type = "rapid"
# Then, we want something like the old types
trip_caracs = [
# comfort tends to limit the number of transfers and fallback
("comfort",
trip_carac([has_no_car],
[transfers_crit, nonTC_crit, best_crit, duration_crit])),
# for car we want at most one journey, the earliest one
(
"car",
trip_carac(
[has_car, has_pt
], # We don't want car only solution, we MUST have PT
[best_crit, transfers_crit, nonTC_crit, duration_crit],
),
),
# less_fallback tends to limit the fallback while walking
(
"less_fallback_walk",
trip_carac([has_no_car, has_no_bike],
[nonTC_crit, transfers_crit, duration_crit, best_crit]),
),
# less_fallback tends to limit the fallback for biking and bss
(
"less_fallback_bike",
trip_carac([has_no_car, has_bike, has_no_bss],
[nonTC_crit, transfers_crit, duration_crit, best_crit]),
),
# less_fallback tends to limit the fallback for biking and bss
(
"less_fallback_bss",
trip_carac([has_no_car, has_bss],
[nonTC_crit, transfers_crit, duration_crit, best_crit]),
),
# the fastest is quite explicit
("fastest",
trip_carac([has_no_car],
[duration_crit, transfers_crit, nonTC_crit, best_crit])),
# the non_pt journeys is the earliest journey without any public transport
("non_pt_walk",
trip_carac([non_pt_journey, has_no_car, has_walk], [best_crit])),
# the non_pt journey is the earliest journey without any public transport
# only walking, biking or driving
("non_pt_bike",
trip_carac([non_pt_journey, has_no_car, has_bike], [best_crit])),
("non_pt_bss",
trip_carac([non_pt_journey, has_no_car, has_bss], [best_crit])),
("non_pt_car", trip_carac([non_pt_journey, has_car], [best_crit])),
]
for name, carac in trip_caracs:
sublist = list(filter(and_filters(carac.constraints), resp.journeys))
best = min_from_criteria(sublist, carac.criteria)
if best is not None:
best.type = name
# Finally, we want exactly one best, the ASAP one
best = min_from_criteria(
resp.journeys, [best_crit, duration_crit, transfers_crit, nonTC_crit])
if best is not None:
best.type = "best"
def walk_tree_depth_first(tree, show_levels=False,
preprocess=None, filter=None, start=None):
"""
A non-recursive generator function that walks through a tree (and all
children) yielding results.
The tree should be of the form:
[(NodeOne, None),
(NodeTwo,
[(Node2A, None),
(Node2B, None),
(Node2C,
[(Node2C1, None), etc]
]
(NodeThree, None),
etc...]
If show_levels is True, the values returned are (level, value) where level
is zero for the top level items in the tree. Otherwise, just "value" is
returned.
If a callable "preprocess" is supplied, it is applied BEFORE the filter,
as each element is encountered.
If a callable "filter" is supplied, it is applied to whatever "preprocess"
returned, and if it returns False for an item, the item and its children
will be skipped.
If "start" is given, the walk will be applied only to that node and its
children. No preprocessing or filtering will be applied to other elements.
"""
annotated_tree = [(0, element) for element in tree]
to_process = deque(annotated_tree)
# If we're not given a starting point, the top is the start
found_starting_point = not start
while to_process:
(level, (node, children)) = to_process.popleft()
if not found_starting_point and (node == start):
# If we're given a starting point and we've found it, clear the list
# and start from here
found_starting_point = True
level = 0
to_process.clear()
# This should NOT be an else case, since we may have just set this flag
# to "True" above.
if found_starting_point:
if preprocess:
value = preprocess(node)
else:
value = node
if filter and not filter(value):
continue
if show_levels:
yield (level, value)
else:
yield value
if children:
annotated_children = [(level+1, child) for child in children]
annotated_children.reverse()
to_process.extendleft(annotated_children)
def test_less_than_equal(self):
filter = self.get_filter('lte', 3)
for match in (-10, 0, 2, 3):
self.assertTrue(filter({'foo': match}))
for non_match in (4, 11, '2'):
self.assertFalse(filter({'foo': non_match}))
def _filter_similarity(mols, distance, generator, query_fps, cutoff):
"""Filter molecules by a certain distance to the reference fingerprints.
User must supply distance funtion, FP generator, query FPs and cutoff."""
return list(filter(
lambda q: any(distance(generator(q), q_fp) >= float(cutoff)
for q_fp in query_fps), mols))
def ParseResults(ycsb_result_string, data_type='histogram'):
"""Parse YCSB results.
Example input for histogram datatype:
YCSB Client 0.1
Command line: -db com.yahoo.ycsb.db.HBaseClient -P /tmp/pkb/workloada
[OVERALL], RunTime(ms), 1800413.0
[OVERALL], Throughput(ops/sec), 2740.503428935472
[UPDATE], Operations, 2468054
[UPDATE], AverageLatency(us), 2218.8513395574005
[UPDATE], MinLatency(us), 554
[UPDATE], MaxLatency(us), 352634
[UPDATE], 95thPercentileLatency(ms), 4
[UPDATE], 99thPercentileLatency(ms), 7
[UPDATE], Return=0, 2468054
[UPDATE], 0, 398998
[UPDATE], 1, 1015682
[UPDATE], 2, 532078
...
Example input for hdrhistogram datatype:
YCSB Client 0.12.0
Command line: -db com.yahoo.ycsb.db.RedisClient -P /opt/pkb/workloadb
[OVERALL], RunTime(ms), 29770.0
[OVERALL], Throughput(ops/sec), 33590.86328518643
[UPDATE], Operations, 49856.0
[UPDATE], AverageLatency(us), 1478.0115532734276
[UPDATE], MinLatency(us), 312.0
[UPDATE], MaxLatency(us), 24623.0
[UPDATE], 95thPercentileLatency(us), 3501.0
[UPDATE], 99thPercentileLatency(us), 6747.0
[UPDATE], Return=OK, 49856
...
Example input for ycsb version after 0.13.0:
...
Command line: -db com.yahoo.ycsb.db.HBaseClient10 ... -load
YCSB Client 0.14.0
Loading workload...
Starting test.
...
[OVERALL], RunTime(ms), 11411
[OVERALL], Throughput(ops/sec), 8763.473841030585
[INSERT], Operations, 100000
[INSERT], AverageLatency(us), 74.92
[INSERT], MinLatency(us), 5
[INSERT], MaxLatency(us), 98495
[INSERT], 95thPercentileLatency(us), 42
[INSERT], 99thPercentileLatency(us), 1411
[INSERT], Return=OK, 100000
...
Example input for timeseries datatype:
...
[OVERALL], RunTime(ms), 240007.0
[OVERALL], Throughput(ops/sec), 10664.605615669543
...
[READ], Operations, 1279253
[READ], AverageLatency(us), 3002.7057071587874
[READ], MinLatency(us), 63
[READ], MaxLatency(us), 93584
[READ], Return=OK, 1279281
[READ], 0, 528.6142757498257
[READ], 500, 360.95347448674966
[READ], 1000, 667.7379547689283
[READ], 1500, 731.5389357265888
[READ], 2000, 778.7992281717318
...
Args:
ycsb_result_string: str. Text output from YCSB.
data_type: Either 'histogram' or 'timeseries' or 'hdrhistogram'.
'histogram' and 'hdrhistogram' datasets are in the same format, with the
difference being lacking the (millisec, count) histogram component. Hence
are parsed similarly.
Returns:
A dictionary with keys:
client: containing YCSB version information.
command_line: Command line executed.
groups: list of operation group descriptions, each with schema:
group: group name (e.g., update, insert, overall)
statistics: dict mapping from statistic name to value
histogram: list of (ms_lower_bound, count) tuples, e.g.:
[(0, 530), (19, 1)]
indicates that 530 ops took between 0ms and 1ms, and 1 took between
19ms and 20ms. Empty bins are not reported.
Raises:
IOError: If the results contained unexpected lines.
"""
# TODO: YCSB 0.9.0 output client and command line string to stderr, so
# we need to support it in the future.
lines = []
client_string = 'YCSB'
command_line = 'unknown'
fp = six.StringIO(ycsb_result_string)
result_string = next(fp).strip()
def IsHeadOfResults(line):
return line.startswith('[OVERALL]')
while not IsHeadOfResults(result_string):
if result_string.startswith('YCSB Client 0.'):
client_string = result_string
if result_string.startswith('Command line:'):
command_line = result_string
try:
result_string = next(fp).strip()
except StopIteration:
raise IOError(
'Could not parse YCSB output: {}'.format(ycsb_result_string))
if result_string.startswith('[OVERALL]'): # YCSB > 0.7.0.
lines.append(result_string)
else:
# Received unexpected header
raise IOError('Unexpected header: {0}'.format(client_string))
# Some databases print additional output to stdout.
# YCSB results start with [<OPERATION_NAME>];
# filter to just those lines.
def LineFilter(line):
return re.search(r'^\[[A-Z]+\]', line) is not None
lines = itertools.chain(lines, filter(LineFilter, fp))
r = csv.reader(lines)
by_operation = itertools.groupby(r, operator.itemgetter(0))
result = collections.OrderedDict([('client', client_string),
('command_line', command_line),
('groups', collections.OrderedDict())])
for operation, lines in by_operation:
operation = operation[1:-1].lower()
if operation == 'cleanup':
continue
op_result = {'group': operation, data_type: [], 'statistics': {}}
latency_unit = 'ms'
for _, name, val in lines:
name = name.strip()
val = val.strip()
# Drop ">" from ">1000"
if name.startswith('>'):
name = name[1:]
val = float(val) if '.' in val or 'nan' in val.lower() else int(
val)
if name.isdigit():
if val:
if data_type == TIMESERIES and latency_unit == 'us':
val /= 1000.0
op_result[data_type].append((int(name), val))
else:
if '(us)' in name:
name = name.replace('(us)', '(ms)')
val /= 1000.0
latency_unit = 'us'
op_result['statistics'][name] = val
result['groups'][operation] = op_result
return result
def process_tokens(self, tokens):
"""process tokens and search for :
_ non strict indentation (i.e. not always using the <indent> parameter as
indent unit)
_ too long lines (i.e. longer than <max_chars>)
_ optionally bad construct (if given, bad_construct must be a compiled
regular expression).
"""
self._bracket_stack = [None]
indents = [0]
check_equal = False
line_num = 0
self._lines = {}
self._visited_lines = {}
token_handlers = self._prepare_token_dispatcher()
self._last_line_ending = None
last_blank_line_num = 0
self._current_line = ContinuedLineState(tokens, self.config)
for idx, (tok_type, token, start, _, line) in enumerate(tokens):
if start[0] != line_num:
line_num = start[0]
# A tokenizer oddity: if an indented line contains a multi-line
# docstring, the line member of the INDENT token does not contain
# the full line; therefore we check the next token on the line.
if tok_type == tokenize.INDENT:
self.new_line(TokenWrapper(tokens), idx - 1, idx + 1)
else:
self.new_line(TokenWrapper(tokens), idx - 1, idx)
if tok_type == tokenize.NEWLINE:
# a program statement, or ENDMARKER, will eventually follow,
# after some (possibly empty) run of tokens of the form
# (NL | COMMENT)* (INDENT | DEDENT+)?
# If an INDENT appears, setting check_equal is wrong, and will
# be undone when we see the INDENT.
check_equal = True
self._process_retained_warnings(TokenWrapper(tokens), idx)
self._current_line.next_logical_line()
self._check_line_ending(token, line_num)
elif tok_type == tokenize.INDENT:
check_equal = False
self.check_indent_level(token, indents[-1] + 1, line_num)
indents.append(indents[-1] + 1)
elif tok_type == tokenize.DEDENT:
# there's nothing we need to check here! what's important is
# that when the run of DEDENTs ends, the indentation of the
# program statement (or ENDMARKER) that triggered the run is
# equal to what's left at the top of the indents stack
check_equal = True
if len(indents) > 1:
del indents[-1]
elif tok_type == tokenize.NL:
if not line.strip('\r\n'):
last_blank_line_num = line_num
self._check_continued_indentation(TokenWrapper(tokens),
idx + 1)
self._current_line.next_physical_line()
elif tok_type != tokenize.COMMENT:
self._current_line.handle_line_start(idx)
# This is the first concrete token following a NEWLINE, so it
# must be the first token of the next program statement, or an
# ENDMARKER; the "line" argument exposes the leading whitespace
# for this statement; in the case of ENDMARKER, line is an empty
# string, so will properly match the empty string with which the
# "indents" stack was seeded
if check_equal:
check_equal = False
self.check_indent_level(line, indents[-1], line_num)
if tok_type == tokenize.NUMBER and token.endswith('l'):
self.add_message('lowercase-l-suffix', line=line_num)
try:
handler = token_handlers[token]
except KeyError:
pass
else:
handler(tokens, idx)
line_num -= 1 # to be ok with "wc -l"
if line_num > self.config.max_module_lines:
# Get the line where the too-many-lines (or its message id)
# was disabled or default to 1.
symbol = self.linter.msgs_store.check_message_id('too-many-lines')
names = (symbol.msgid, 'too-many-lines')
line = next(
filter(None, map(self.linter._pragma_lineno.get, names)), 1)
self.add_message('too-many-lines',
args=(line_num, self.config.max_module_lines),
line=line)
# See if there are any trailing lines. Do not complain about empty
# files like __init__.py markers.
if line_num == last_blank_line_num and line_num > 0:
self.add_message('trailing-newlines', line=line_num)
def enhance_input_model(input_model):
"""
Uses the input model schema to enhance the input model data structure,
in order to make it easier to navigate:
- replaces sub-element lists with key-indexed sub-element maps (to
simplify lookup operations)
- replaces attributes representing foreign keys with direct references to
foreign elements (to simplify lookup and dereference operations)
- deletes unused elements from the input models (e.g. server roles that
aren't referenced by any control plane)
Processes neutron configuration data, network group tags and routes and
extends the input model data structure with information pertaining to
identified neutron networks.
:param input_model: original input model, as loaded from disk
:return: enhanced input model data structure
"""
input_model = deepcopy(input_model)
map_list_attrs(input_model, input_model_schema)
map_foreign_keys(input_model, 'input-model', input_model,
input_model_schema)
prune_input_model(input_model, input_model_schema)
# Assume there is at most one neutron configuration data
neutron_config_data = list(
filter(lambda config_data: 'neutron' in config_data['services'],
input_model['configuration-data'].values()))
neutron_config_data = \
input_model['neutron-config-data'] = \
neutron_config_data[0] if neutron_config_data else None
# Collect all network group tags in a single map,
# indexed by neutron group name
neutron_network_tags = dict()
# Collect all neutron provider/external networks in a single map,
# indexed by network name
neutron_networks = input_model['neutron-networks'] = dict()
def add_neutron_network_tags(network_group_name, tags):
tag = neutron_network_tags.setdefault(
network_group_name, {'network-group': network_group_name})
link_elements_by_foreign_key(
tag,
'network-group',
input_model['network-groups'],
ref_list_attr='neutron-tags',
# Use a null key_attr value
# to create a list of references
element_key=None)
tag.setdefault('tags', []).extend(tags)
if neutron_config_data:
# Starting in SUSE OpenStack Cloud 8, network tags may be defined
# as part of a Neutron configuration-data object rather than as part
# of a network-group object.
for network_tag in neutron_config_data.get('network-tags', []):
add_neutron_network_tags(network_tag['network-group'],
network_tag['tags'])
external_networks = convert_element_list_to_map(
neutron_config_data['data'], 'neutron_external_networks')
provider_networks = convert_element_list_to_map(
neutron_config_data['data'], 'neutron_provider_networks')
neutron_networks.update(external_networks)
neutron_networks.update(provider_networks)
for network in external_networks.itervalues():
network['external'] = True
for network in provider_networks.itervalues():
network['external'] = False
for network_group in input_model['network-groups'].itervalues():
if neutron_config_data and 'tags' in network_group:
add_neutron_network_tags(network_group['name'],
network_group['tags'])
link_elements_by_foreign_key_list(network_group,
'routes',
input_model['network-groups'],
ref_list_attr='network-group-routes')
link_elements_by_foreign_key_list(network_group,
'routes',
neutron_networks,
ref_list_attr='network-group-routes')
# Network groups may contain references to control plane load
# balancers, which we have to transform into object references
# explicitly here
for cp in input_model['control-planes'].values():
link_elements_by_foreign_key_list(network_group,
'load-balancers',
cp['load-balancers'],
ref_list_attr=None)
# Based on the collected neutron networks and network tags, identify
# which network group is linked to which neutron network, by looking
# at the provider physical network settings
neutron_physnets = dict()
for neutron_network in neutron_networks.itervalues():
# The only neutron network without a provider is the external
# "bridge" network.
# Assume a default 'external' physnet value for this network.
if 'provider' not in neutron_network:
if neutron_network['external']:
physnet = 'external'
else:
continue
else:
physnet = neutron_network['provider'][0]['physical_network']
neutron_physnets[physnet] = neutron_network
for network_tag in neutron_network_tags.itervalues():
for tag in network_tag['tags']:
if isinstance(tag, dict):
tag = tag.values()[0]
# The only relevant tag without a provider is the external
# "bridge" network.
# Assume a default 'external' physnet value for this network.
if 'provider-physical-network' not in tag:
if tag == 'neutron.l3_agent.external_network_bridge':
physnet = 'external'
else:
continue
else:
physnet = tag['provider-physical-network']
if physnet not in neutron_physnets:
continue
# Create a 'neutron-networks' attribute in the network group
# element as a map of neutron networks indexed by physical
# network name
network_tag['network-group'].setdefault(
'neutron-networks',
dict())[physnet] = neutron_physnets[physnet]
# Create a 'network-groups' attribute in the neutron network
# element as a map of neutron groups indexed by network group name
neutron_physnets[physnet].setdefault(
'network-groups',
dict())[network_tag['network-group']['name']] = \
network_tag['network-group']
return input_model
def handle_compressed_file(
filename,
datatypes_registry,
ext='auto',
tmp_prefix='sniff_uncompress_',
tmp_dir=None,
in_place=False,
check_content=True,
auto_decompress=True,
):
"""
Check uploaded files for compression, check compressed file contents, and uncompress if necessary.
Supports GZip, BZip2, and the first file in a Zip file.
For performance reasons, the temporary file used for uncompression is located in the same directory as the
input/output file. This behavior can be changed with the `tmp_dir` param.
``ext`` as returned will only be changed from the ``ext`` input param if the param was an autodetect type (``auto``)
and the file was sniffed as a keep-compressed datatype.
``is_valid`` as returned will only be set if the file is compressed and contains invalid contents (or the first file
in the case of a zip file), this is so lengthy decompression can be bypassed if there is invalid content in the
first 32KB. Otherwise the caller should be checking content.
"""
CHUNK_SIZE = 2**20 # 1Mb
is_compressed = False
compressed_type = None
keep_compressed = False
is_valid = False
uncompressed = filename
tmp_dir = tmp_dir or os.path.dirname(filename)
for compressed_type, check_compressed_function in COMPRESSION_CHECK_FUNCTIONS:
is_compressed, is_valid = check_compressed_function(
filename, check_content=check_content)
if is_compressed:
break # found compression type
if is_compressed and is_valid:
if ext in AUTO_DETECT_EXTENSIONS:
# attempt to sniff for a keep-compressed datatype (observing the sniff order)
sniff_datatypes = filter(lambda d: getattr(d, 'compressed', False),
datatypes_registry.sniff_order)
sniffed_ext = run_sniffers_raw(filename, sniff_datatypes)
if sniffed_ext:
ext = sniffed_ext
keep_compressed = True
else:
datatype = datatypes_registry.get_datatype_by_extension(ext)
keep_compressed = getattr(datatype, 'compressed', False)
# don't waste time decompressing if we sniff invalid contents
if is_compressed and is_valid and auto_decompress and not keep_compressed:
fd, uncompressed = tempfile.mkstemp(prefix=tmp_prefix, dir=tmp_dir)
compressed_file = DECOMPRESSION_FUNCTIONS[compressed_type](filename)
# TODO: it'd be ideal to convert to posix newlines and space-to-tab here as well
while True:
try:
chunk = compressed_file.read(CHUNK_SIZE)
except IOError as e:
os.close(fd)
os.remove(uncompressed)
compressed_file.close()
raise IOError(
'Problem uncompressing %s data, please try retrieving the data uncompressed: %s'
% (compressed_type, util.unicodify(e)))
if not chunk:
break
os.write(fd, chunk)
os.close(fd)
compressed_file.close()
if in_place:
# Replace the compressed file with the uncompressed file
shutil.move(uncompressed, filename)
uncompressed = filename
elif not is_compressed or not check_content:
is_valid = True
return is_valid, ext, uncompressed, compressed_type
def generate_heat_model(input_model, virt_config):
"""
Create a data structure that more or less describes the heat resources
required to deploy the input model. The data structure can then later
be used to generate a heat orchestration template.
:param input_model: enhanced input model data structure
:param virt_config: additional information regarding the virtual setup
(images, flavors, disk sizes)
:return: dictionary describing heat resources
"""
heat_template = dict(description='Template for deploying Ardana {}'.format(
input_model['cloud']['name']))
clm_cidr = IPNetwork(input_model['baremetal']['subnet'],
input_model['baremetal']['netmask'])
clm_network = None
heat_networks = heat_template['networks'] = dict()
# First, add L2 neutron provider networks defined in the input
# model's neutron configuration
for neutron_network in input_model['neutron-networks'].itervalues():
heat_network = dict(name=neutron_network['name'],
is_conf=False,
is_mgmt=False,
external=neutron_network['external'])
if neutron_network.get('cidr'):
heat_network['cidr'] = neutron_network['cidr']
if neutron_network.get('gateway'):
heat_network['gateway'] = neutron_network['gateway']
if neutron_network.get('provider'):
provider = neutron_network['provider'][0]
if provider['network_type'] == 'vlan':
if not provider.get('segmentation_id'):
# Neutron network is incompletely defined (VLAN tag is
# dynamically allocated), so it cannot be defined as an
# individual heat network
continue
heat_network['vlan'] = provider['segmentation_id']
elif provider['network_type'] not in ['flat', 'vlan']:
# Only layer 2 neutron provider networks are considered
continue
heat_networks[heat_network['name']] = heat_network
# Collect all the routers required by routes configured in the input model,
# as pairs of networks
routers = set()
# Next, add global networks
for network in input_model['networks'].itervalues():
cidr = None
vlan = network['vlanid'] if network.get('tagged-vlan', True) else None
gateway = IPAddress(
network['gateway-ip']) if network.get('gateway-ip') else None
if network.get('cidr'):
cidr = IPNetwork(network['cidr'])
heat_network = dict(name=network['name'],
is_conf=False,
is_mgmt=False,
external=False)
if cidr:
heat_network['cidr'] = str(cidr)
if gateway:
heat_network['gateway'] = str(gateway)
# There is the special case of global networks being used to implement
# flat neutron provider networks. For these networks, we need to
# create a heat network based on the global network parameters
# (i.e. VLAN) and a heat subnet based on the neutron network
# parameters
for neutron_network in network['network-group'].get(
'neutron-networks', {}).itervalues():
heat_neutron_network = heat_networks.get(neutron_network['name'])
if not heat_neutron_network or heat_neutron_network.get('vlan'):
# Ignore neutron networks that:
# - were not already considered at the previous step (i.e.
# are not fully defined or are not layer 2 based)
# - have a vlan (i.e. are not flat)
continue
# Replace the heat neutron network with this global network
# This is the same as updating the heat global network with subnet
# attributes taken from the neutron network
del heat_networks[neutron_network['name']]
heat_network = heat_neutron_network
heat_network['name'] = network['name']
# Only one flat neutron provider network can be associated with a
# global network
break
if vlan:
heat_network['vlan'] = vlan
# For each route, track down the target network
for route in network['network-group']['routes']:
if route == 'default':
# The default route is satisfied by adding the network to the
# external router
heat_network['external'] = True
elif 'networks' in route:
# If the route points to a network group, track down the
# networks associated with it
for network_name in route['networks'].keys():
routers.add((
heat_network['name'],
network_name,
))
else:
routers.add((
heat_network['name'],
route['name'],
))
if cidr and cidr in clm_cidr:
clm_network = heat_network
heat_network['external'] = heat_network['is_conf'] = True
# Create an address pool range that excludes the list of server
# static IP addresses
fixed_ip_addr_list = \
[IPAddress(server['ip-addr'])
for server in input_model['servers'].itervalues()]
if gateway:
fixed_ip_addr_list.append(gateway)
start_addr = cidr[1]
end_addr = cidr[-2]
for fixed_ip_addr in sorted(list(set(fixed_ip_addr_list))):
if start_addr <= fixed_ip_addr <= end_addr:
if fixed_ip_addr - start_addr < end_addr - fixed_ip_addr:
start_addr = fixed_ip_addr + 1
else:
end_addr = fixed_ip_addr - 1
heat_network['allocation_pools'] = \
[[str(start_addr), str(end_addr)]]
elif ('component-endpoints' in network['network-group'] and 'default'
in network['network-group']['component-endpoints']):
heat_network['external'] = heat_network['is_mgmt'] = True
# Create an address pool range that is outside of the range
# of IP addresses allocated by Ardana
mgmt_net_last = IPAddress(IPNetwork(network['cidr']).last)
heat_network['allocation_pools'] = \
[[str(mgmt_net_last - EXTERNAL_MGMT_ADDR_RANGE),
str(mgmt_net_last - 1)]]
elif True in [
('public' in lb['roles'])
for lb in network['network-group'].get('load-balancers', [])
]:
heat_network['external'] = True
heat_networks[network['name']] = heat_network
heat_template['routers'] = []
for network1, network2 in routers:
if network1 not in heat_template['networks'] or \
network2 not in heat_template['networks']:
continue
network1 = heat_template['networks'][network1]
network2 = heat_template['networks'][network2]
# Re-use the external router, if at least one of the networks is
# already attached to it
if network1['external'] or network2['external']:
network1['external'] = network2['external'] = True
else:
heat_template['routers'].append(
[network1['name'], network2['name']])
heat_interface_models = heat_template['interface_models'] = dict()
for interface_model in input_model['interface-models'].itervalues():
heat_interface_model = \
heat_interface_models[interface_model['name']] = \
dict(
name=interface_model['name'],
ports=[]
)
ports = dict()
clm_ports = dict()
for interface in interface_model['network-interfaces'].itervalues():
devices = interface['bond-data']['devices'] \
if 'bond-data' in interface \
else [interface['device']]
for device in devices:
port_list = ports
port = dict(name=device['name'], networks=[])
if 'bond-data' in interface:
port['bond'] = interface['device']['name']
port['primary'] = \
(device['name'] ==
interface['bond-data']['options'].get('primary',
device['name']))
for network_group in \
interface.get('network-groups', []) + \
interface.get('forced-network-groups', []):
port['networks'].extend([
network['name']
for network in network_group['networks'].itervalues()
])
# Attach the port only to those neutron networks that have
# been validated during the previous steps
port['networks'].extend([
network['name'] for network in network_group.get(
'neutron-networks', dict()).itervalues()
if network['name'] in heat_networks
])
if clm_network['name'] in network_group['networks']:
# if the CLM port is a bond port, then only the
# primary is considered if configured
if not clm_ports and port.get('primary', True):
# Collect the CLM port separately, to put it at
# the top of the list and to mark it as the
# "management" port - the port to which the
# server's management IP address is assigned
port_list = clm_ports
port_list[device['name']] = port
# Add a port for each device, starting with those ports attached to
# the CLM network while at the same time preserving the order of the
# original ports. Ultimately, the port names will be re-aligned to
# those in the input model by an updated NIC mappings input model
# configuration
heat_interface_model['ports'] = [
p[1] for _, p in enumerate(
sorted(clm_ports.items()) + sorted(ports.items()))
]
# Generate storage setup (volumes)
#
# General strategy:
# - one volume for each physical volume specified in the disk model
# - the size of each volume cannot be determined from the input model,
# so this information needs to be supplied separately (TBD)
heat_disk_models = heat_template['disk_models'] = dict()
disks = virt_config['disks']
for disk_model in input_model['disk-models'].itervalues():
heat_disk_model = heat_disk_models[disk_model['name']] = dict(
name=disk_model['name'], volumes=[])
devices = []
for volume_group in disk_model.get('volume-groups', []):
devices += volume_group['physical-volumes']
for device_group in disk_model.get('device-groups', []):
devices += [device['name'] for device in device_group['devices']]
for device in sorted(list(set(devices))):
if device.endswith('da_root'):
continue
device = device.replace('/dev/sd', '/dev/vd')
volume_name = device.replace('/dev/', '')
size = virt_config['disk_size']
# Check if disk size is configured explicitly for the disk model
if disk_model['name'] in disks:
size = disks[disk_model['name']]
if isinstance(size, dict):
# Use the disk size specified for the volume name, or
# the disk model default, or the global default
size = size.get(volume_name) or \
size.get('default') or \
virt_config['disk_size']
heat_disk_model['volumes'].append(
dict(name=volume_name, mountpoint=device, size=size))
# Generate VM setup (servers)
#
# General strategy:
# - one server for each server specified in the disk model
# - the CLM server is special:
# - identification: server hosting the lifecycle-manager
# service component
# - the floating IP is associated with the "CLM" port attached to it
# - the image and flavor used for the server cannot be determined from
# the input model so this information needs to be supplied separately
heat_servers = heat_template['servers'] = []
images = virt_config['images']
flavors = virt_config['flavors']
clm_server = None
for server in input_model['servers'].itervalues():
distro_id = server.get('distro-id', virt_config['sles_distro_id'])
image = None
# Check if image is configured explicitly
# for the server or for the role
if server['id'] in images:
image = images[server['id']]
elif server['role']['name'] in images:
image = images[server['role']['name']]
if isinstance(image, dict):
# Use the image specified for the distribution, or
# the global default
image = image.get(distro_id)
if not image:
image = virt_config['sles_image']
if distro_id == virt_config['rhel_distro_id']:
image = virt_config['rhel_image']
flavor = None
# Check if image is configured explicitly
# for the server or for the role
if server['id'] in flavors:
flavor = flavors[server['id']]
elif server['role']['name'] in flavors:
flavor = flavors[server['role']['name']]
heat_server = dict(
name=server['id'],
ip_addr=server['ip-addr'],
role=server['role']['name'],
interface_model=server['role']['interface-model']['name'],
disk_model=server['role']['disk-model']['name'],
image=image,
flavor=flavor,
is_admin=False,
is_controller=False,
is_compute=False)
# Figure out which server is the CLM host, which are controllers
# and which are computes. This information is used e.g. to determine
# the reboot order during the MU workflow and to identify flavors
# unless explicitly specified for each server or server role
service_groups = list(server['role'].get('clusters', {}).values())
service_groups += list(server['role'].get('resources', {}).values())
for service_group in service_groups:
# The CLM server is the first server hosting the lifecycle-manager
# service component.
# Compute nodes host the nova-compute service component
if 'nova-compute' in service_group['service-components']:
heat_server['is_compute'] = True
if not heat_server['flavor']:
heat_server['flavor'] = virt_config['compute_flavor']
# Every server that is not a compute node and hosts service
# components other than those required by the CLM is considered
# a controller node
else:
ctrl_service_components = filter(
lambda sc: sc not in virt_config['clm_service_components'],
service_group['service-components'])
if ctrl_service_components:
heat_server['is_controller'] = True
if not heat_server['flavor']:
heat_server['flavor'] = \
virt_config['controller_flavor']
if not clm_server and \
'lifecycle-manager' in service_group['service-components']:
clm_server = heat_server
heat_server['is_admin'] = True
if not heat_server['flavor']:
heat_server['flavor'] = virt_config['clm_flavor']
heat_servers.append(heat_server)
return heat_template
def GetAppHostname(app=None, app_id=None, service=None, version=None,
use_ssl=appinfo.SECURE_HTTP, deploy=True):
"""Returns the hostname of the given version of the deployed app.
Args:
app: Application resource. One of {app, app_id} must be given.
app_id: str, project ID. One of {app, app_id} must be given. If both are
provided, the hostname from app is preferred.
service: str, the (optional) service being deployed
version: str, the deployed version ID (omit to get the default version URL).
use_ssl: bool, whether to construct an HTTPS URL.
deploy: bool, if this is called during a deployment.
Returns:
str. Constructed URL.
Raises:
TypeError: if neither an app nor an app_id is provided
"""
if not app and not app_id:
raise TypeError('Must provide an application resource or application ID.')
version = version or ''
service_name = service or ''
if service == DEFAULT_SERVICE:
service_name = ''
domain = DEFAULT_DOMAIN
if not app and ':' in app_id:
api_client = appengine_api_client.AppengineApiClient.GetApiClient()
app = api_client.GetApplication()
if app:
app_id, domain = app.defaultHostname.split('.', 1)
# Normally, AppEngine URLs are of the form
# 'http[s]://version.service.app.appspot.com'. However, the SSL certificate
# for appspot.com is not valid for subdomains of subdomains of appspot.com
# (e.g. 'https://app.appspot.com/' is okay; 'https://service.app.appspot.com/'
# is not). To deal with this, AppEngine recognizes URLs like
# 'http[s]://version-dot-service-dot-app.appspot.com/'.
#
# This works well as long as the domain name part constructed in this fashion
# is less than 63 characters long, as per the DNS spec. If the domain name
# part is longer than that, we are forced to use the URL with an invalid
# certificate.
#
# We've tried to do the best possible thing in every case here.
subdomain_parts = list(filter(bool, [version, service_name, app_id]))
scheme = 'http'
if use_ssl == appinfo.SECURE_HTTP:
subdomain = '.'.join(subdomain_parts)
scheme = 'http'
else:
subdomain = ALT_SEPARATOR.join(subdomain_parts)
if len(subdomain) <= MAX_DNS_LABEL_LENGTH:
scheme = 'https'
else:
if deploy:
format_parts = ['$VERSION_ID', '$SERVICE_ID', '$APP_ID']
subdomain_format = ALT_SEPARATOR.join(
[j for (i, j) in zip([version, service_name, app_id], format_parts)
if i])
msg = ('This deployment will result in an invalid SSL certificate for '
'service [{0}]. The total length of your subdomain in the '
'format {1} should not exceed {2} characters. Please verify '
'that the certificate corresponds to the parent domain of your '
'application when you connect.').format(service,
subdomain_format,
MAX_DNS_LABEL_LENGTH)
log.warning(msg)
subdomain = '.'.join(subdomain_parts)
if use_ssl == appinfo.SECURE_HTTP_OR_HTTPS:
scheme = 'http'
elif use_ssl == appinfo.SECURE_HTTPS:
if not deploy:
msg = ('Most browsers will reject the SSL certificate for '
'service [{0}].').format(service)
log.warning(msg)
scheme = 'https'
return '{0}://{1}.{2}'.format(scheme, subdomain, domain)
def apply_event(state, event, user_profile, include_subscribers):
# type: (Dict[str, Any], Dict[str, Any], UserProfile, bool) -> None
if event['type'] == "message":
state['max_message_id'] = max(state['max_message_id'], event['message']['id'])
if 'raw_unread_msgs' in state:
apply_unread_message_event(
user_profile,
state['raw_unread_msgs'],
event['message'],
event['flags'],
)
elif event['type'] == "hotspots":
state['hotspots'] = event['hotspots']
elif event['type'] == "custom_profile_fields":
state['custom_profile_fields'] = event['fields']
elif event['type'] == "pointer":
state['pointer'] = max(state['pointer'], event['pointer'])
elif event['type'] == "realm_user":
person = event['person']
person_user_id = person['user_id']
if event['op'] == "add":
person = copy.deepcopy(person)
person['is_active'] = True
state['raw_users'][person_user_id] = person
elif event['op'] == "remove":
state['raw_users'][person_user_id]['is_active'] = False
elif event['op'] == 'update':
is_me = (person_user_id == user_profile.id)
if is_me:
if ('avatar_url' in person and 'avatar_url' in state):
state['avatar_source'] = person['avatar_source']
state['avatar_url'] = person['avatar_url']
state['avatar_url_medium'] = person['avatar_url_medium']
for field in ['is_admin', 'email', 'full_name']:
if field in person and field in state:
state[field] = person[field]
# In the unlikely event that the current user
# just changed to/from being an admin, we need
# to add/remove the data on all bots in the
# realm. This is ugly and probably better
# solved by removing the all-realm-bots data
# given to admin users from this flow.
if ('is_admin' in person and 'realm_bots' in state):
prev_state = state['raw_users'][user_profile.id]
was_admin = prev_state['is_admin']
now_admin = person['is_admin']
if was_admin and not now_admin:
state['realm_bots'] = []
if not was_admin and now_admin:
state['realm_bots'] = get_owned_bot_dicts(user_profile)
if person_user_id in state['raw_users']:
p = state['raw_users'][person_user_id]
for field in p:
if field in person:
p[field] = person[field]
elif event['type'] == 'realm_bot':
if event['op'] == 'add':
state['realm_bots'].append(event['bot'])
if event['op'] == 'remove':
email = event['bot']['email']
for bot in state['realm_bots']:
if bot['email'] == email:
bot['is_active'] = False
if event['op'] == 'update':
for bot in state['realm_bots']:
if bot['email'] == event['bot']['email']:
if 'owner_id' in event['bot']:
bot['owner'] = get_user_profile_by_id(event['bot']['owner_id']).email
else:
bot.update(event['bot'])
elif event['type'] == 'stream':
if event['op'] == 'create':
for stream in event['streams']:
if not stream['invite_only']:
stream_data = copy.deepcopy(stream)
if include_subscribers:
stream_data['subscribers'] = []
# Add stream to never_subscribed (if not invite_only)
state['never_subscribed'].append(stream_data)
state['streams'].append(stream)
state['streams'].sort(key=lambda elt: elt["name"])
if event['op'] == 'delete':
deleted_stream_ids = {stream['stream_id'] for stream in event['streams']}
state['streams'] = [s for s in state['streams'] if s['stream_id'] not in deleted_stream_ids]
state['never_subscribed'] = [stream for stream in state['never_subscribed'] if
stream['stream_id'] not in deleted_stream_ids]
if event['op'] == 'update':
# For legacy reasons, we call stream data 'subscriptions' in
# the state var here, for the benefit of the JS code.
for obj in state['subscriptions']:
if obj['name'].lower() == event['name'].lower():
obj[event['property']] = event['value']
# Also update the pure streams data
for stream in state['streams']:
if stream['name'].lower() == event['name'].lower():
prop = event['property']
if prop in stream:
stream[prop] = event['value']
elif event['op'] == "occupy":
state['streams'] += event['streams']
elif event['op'] == "vacate":
stream_ids = [s["stream_id"] for s in event['streams']]
state['streams'] = [s for s in state['streams'] if s["stream_id"] not in stream_ids]
elif event['type'] == 'default_streams':
state['realm_default_streams'] = event['default_streams']
elif event['type'] == 'realm':
if event['op'] == "update":
field = 'realm_' + event['property']
state[field] = event['value']
# Tricky interaction: Whether we can create streams can get changed here.
if (field in ['realm_create_stream_by_admins_only',
'realm_waiting_period_threshold']) and 'can_create_streams' in state:
state['can_create_streams'] = user_profile.can_create_streams()
elif event['op'] == "update_dict":
for key, value in event['data'].items():
state['realm_' + key] = value
# It's a bit messy, but this is where we need to
# update the state for whether password authentication
# is enabled on this server.
if key == 'authentication_methods':
state['realm_password_auth_enabled'] = (value['Email'] or value['LDAP'])
state['realm_email_auth_enabled'] = value['Email']
elif event['type'] == "subscription":
if not include_subscribers and event['op'] in ['peer_add', 'peer_remove']:
return
if event['op'] in ["add"]:
if not include_subscribers:
# Avoid letting 'subscribers' entries end up in the list
for i, sub in enumerate(event['subscriptions']):
event['subscriptions'][i] = copy.deepcopy(event['subscriptions'][i])
del event['subscriptions'][i]['subscribers']
def name(sub):
# type: (Dict[str, Any]) -> Text
return sub['name'].lower()
if event['op'] == "add":
added_names = set(map(name, event["subscriptions"]))
was_added = lambda s: name(s) in added_names
# add the new subscriptions
state['subscriptions'] += event['subscriptions']
# remove them from unsubscribed if they had been there
state['unsubscribed'] = [s for s in state['unsubscribed'] if not was_added(s)]
# remove them from never_subscribed if they had been there
state['never_subscribed'] = [s for s in state['never_subscribed'] if not was_added(s)]
elif event['op'] == "remove":
removed_names = set(map(name, event["subscriptions"]))
was_removed = lambda s: name(s) in removed_names
# Find the subs we are affecting.
removed_subs = list(filter(was_removed, state['subscriptions']))
# Remove our user from the subscribers of the removed subscriptions.
if include_subscribers:
for sub in removed_subs:
sub['subscribers'] = [id for id in sub['subscribers'] if id != user_profile.id]
# We must effectively copy the removed subscriptions from subscriptions to
# unsubscribe, since we only have the name in our data structure.
state['unsubscribed'] += removed_subs
# Now filter out the removed subscriptions from subscriptions.
state['subscriptions'] = [s for s in state['subscriptions'] if not was_removed(s)]
elif event['op'] == 'update':
for sub in state['subscriptions']:
if sub['name'].lower() == event['name'].lower():
sub[event['property']] = event['value']
elif event['op'] == 'peer_add':
user_id = event['user_id']
for sub in state['subscriptions']:
if (sub['name'] in event['subscriptions'] and
user_id not in sub['subscribers']):
sub['subscribers'].append(user_id)
for sub in state['never_subscribed']:
if (sub['name'] in event['subscriptions'] and
user_id not in sub['subscribers']):
sub['subscribers'].append(user_id)
elif event['op'] == 'peer_remove':
user_id = event['user_id']
for sub in state['subscriptions']:
if (sub['name'] in event['subscriptions'] and
user_id in sub['subscribers']):
sub['subscribers'].remove(user_id)
elif event['type'] == "presence":
# TODO: Add user_id to presence update events / state format!
presence_user_profile = get_user(event['email'], user_profile.realm)
state['presences'][event['email']] = UserPresence.get_status_dict_by_user(
presence_user_profile)[event['email']]
elif event['type'] == "update_message":
# We don't return messages in /register, so we don't need to
# do anything for content updates, but we may need to update
# the unread_msgs data if the topic of an unread message changed.
if 'subject' in event:
stream_dict = state['raw_unread_msgs']['stream_dict']
topic = event['subject']
for message_id in event['message_ids']:
if message_id in stream_dict:
stream_dict[message_id]['topic'] = topic
elif event['type'] == "delete_message":
max_message = Message.objects.filter(
usermessage__user_profile=user_profile).order_by('-id').first()
if max_message:
state['max_message_id'] = max_message.id
else:
state['max_message_id'] = -1
remove_id = event['message_id']
remove_message_id_from_unread_mgs(state, remove_id)
elif event['type'] == "reaction":
# The client will get the message with the reactions directly
pass
elif event['type'] == 'typing':
# Typing notification events are transient and thus ignored
pass
elif event['type'] == "update_message_flags":
# We don't return messages in `/register`, so most flags we
# can ignore, but we do need to update the unread_msgs data if
# unread state is changed.
if event['flag'] == 'read' and event['operation'] == 'add':
for remove_id in event['messages']:
remove_message_id_from_unread_mgs(state, remove_id)
elif event['type'] == "realm_domains":
if event['op'] == 'add':
state['realm_domains'].append(event['realm_domain'])
elif event['op'] == 'change':
for realm_domain in state['realm_domains']:
if realm_domain['domain'] == event['realm_domain']['domain']:
realm_domain['allow_subdomains'] = event['realm_domain']['allow_subdomains']
elif event['op'] == 'remove':
state['realm_domains'] = [realm_domain for realm_domain in state['realm_domains']
if realm_domain['domain'] != event['domain']]
elif event['type'] == "realm_emoji":
state['realm_emoji'] = event['realm_emoji']
elif event['type'] == "alert_words":
state['alert_words'] = event['alert_words']
elif event['type'] == "muted_topics":
state['muted_topics'] = event["muted_topics"]
elif event['type'] == "realm_filters":
state['realm_filters'] = event["realm_filters"]
elif event['type'] == "update_display_settings":
assert event['setting_name'] in UserProfile.property_types
state[event['setting_name']] = event['setting']
elif event['type'] == "update_global_notifications":
assert event['notification_name'] in UserProfile.notification_setting_types
state[event['notification_name']] = event['setting']
else:
raise AssertionError("Unexpected event type %s" % (event['type'],))
def split_by_synapse_domain(bandwidth, locations, arbors, treenode_connector,
minis):
""" locations: dictionary of treenode ID vs tuple with x,y,z
arbors: dictionary of skeleton ID vs list of DiGraph (that were, or not, split by confidence)
treenode_connectors: dictionary of treenode ID vs list of tuples of connector_id, string of 'presynaptic_to' or 'postsynaptic_to'
"""
arbors2 = {} # Some arbors will be split further
for skeleton_id, graphs in six.iteritems(arbors):
subdomains = []
arbors2[skeleton_id] = subdomains
for graph in graphs:
treenode_ids = []
connector_ids = []
relation_ids = []
for treenode_id in filter(treenode_connector.has_key,
graph.nodes_iter()):
for c in treenode_connector.get(treenode_id):
connector_id, relation = c
treenode_ids.append(treenode_id)
connector_ids.append(connector_id)
relation_ids.append(relation)
if not connector_ids:
subdomains.append(graph)
continue
for parent_id, treenode_id in graph.edges_iter():
loc0 = locations[treenode_id]
loc1 = locations[parent_id]
graph[parent_id][treenode_id]['weight'] = norm(
subtract(loc0, loc1))
# Invoke Casey's magic
max_density = tree_max_density(graph.to_undirected(), treenode_ids,
connector_ids, relation_ids,
[bandwidth])
synapse_group = next(six.itervalues(max_density))
# The list of nodes of each synapse_group contains only nodes that have connectors
# A local_max is the skeleton node most central to a synapse_group
anchors = {}
for domain in six.itervalues(synapse_group):
g = nx.DiGraph()
g.add_nodes_from(
domain.node_ids
) # bogus graph, containing treenodes that point to connectors
subdomains.append(g)
anchors[domain.local_max] = g
# Define edges between domains: create a simplified graph
mini = simplify(graph, anchors.keys())
# Replace each node by the corresponding graph, or a graph of a single node
for node in mini.nodes_iter():
g = anchors.get(node)
if not g:
# A branch node that was not an anchor, i.e. did not represent a synapse group
g = nx.Graph()
g.add_node(node, {'branch': True})
subdomains.append(g)
# Associate the Graph with treenodes that have connectors
# with the node in the minified tree
mini.node[node]['g'] = g
# Put the mini into a map of skeleton_id and list of minis,
# to be used later for defining intra-neuron edges in the circuit graph
minis[skeleton_id].append(mini)
return arbors2, minis