def get_data(self, request, **kwargs):
"""
Returns the data for the page requested with the specified
parameters applied
filters: filter and action name, e.g. "outcome:build_succeeded"
filter_value: value to pass to the named filter+action, e.g. "on"
(for a toggle filter) or "2015-12-11,2015-12-12" (for a date range filter)
"""
page_num = request.GET.get("page", 1)
limit = request.GET.get("limit", 10)
search = request.GET.get("search", None)
filters = request.GET.get("filter", None)
filter_value = request.GET.get("filter_value", "on")
orderby = request.GET.get("orderby", None)
nocache = request.GET.get("nocache", None)
# Make a unique cache name
cache_name = self.__class__.__name__
for key, val in request.GET.iteritems():
if key == 'nocache':
continue
cache_name = cache_name + str(key) + str(val)
for key, val in kwargs.iteritems():
cache_name = cache_name + str(key) + str(val)
# No special chars allowed in the cache name apart from dash
cache_name = re.sub(r'[^A-Za-z0-9-]', "", cache_name)
if nocache:
cache.delete(cache_name)
data = cache.get(cache_name)
if data:
logger.debug("Got cache data for table '%s'" % self.title)
return data
self.setup_columns(**kwargs)
if search:
self.apply_search(search)
if filters:
self.apply_filter(filters, filter_value, **kwargs)
if orderby:
self.apply_orderby(orderby)
paginator = Paginator(self.queryset, limit)
try:
page = paginator.page(page_num)
except EmptyPage:
page = paginator.page(1)
data = {
'total': self.queryset.count(),
'default_orderby': self.default_orderby,
'columns': self.columns,
'rows': [],
'error': "ok",
}
try:
for row in page.object_list:
#Use collection to maintain the order
required_data = collections.OrderedDict()
for col in self.columns:
field = col['field_name']
if not field:
field = col['static_data_name']
if not field:
raise Exception(
"Must supply a field_name or static_data_name for column %s.%s"
% (self.__class__.__name__, col))
# Check if we need to process some static data
if "static_data_name" in col and col['static_data_name']:
required_data[
"static:%s" %
col['static_data_name']] = self.render_static_data(
col['static_data_template'], row)
# Overwrite the field_name with static_data_name
# so that this can be used as the html class name
col['field_name'] = col['static_data_name']
# compute the computation on the raw data if needed
model_data = row
if col['computation']:
model_data = col['computation'](row)
else:
# Traverse to any foriegn key in the object hierachy
for subfield in field.split("__"):
if hasattr(model_data, subfield):
model_data = getattr(model_data, subfield)
# The field could be a function on the model so check
# If it is then call it
if isinstance(model_data, types.MethodType):
model_data = model_data()
required_data[col['field_name']] = model_data
data['rows'].append(required_data)
except FieldError:
# pass it to the user - programming-error here
raise
data = json.dumps(data, indent=2, default=objtojson)
cache.set(cache_name, data, 60 * 30)
return data
def contact_summary(request):
"""
Method returning the template for the Your login and contact details: summary page (for a given application)
displaying entered data for this task and navigating to the Type of childcare page when successfully completed
:param request: a request object used to generate the HttpResponse
:return: an HttpResponse object with the rendered Your login and contact details: summary template
"""
if request.method == 'GET':
app_id = request.GET["id"]
application = Application.objects.get(pk=app_id)
user_details = UserDetails.objects.get(application_id=app_id)
email = user_details.email
mobile_number = user_details.mobile_number
add_phone_number = user_details.add_phone_number
contact_info_fields = collections.OrderedDict([
('email_address', email), ('mobile_number', mobile_number),
('add_phone_number', add_phone_number)
])
contact_info_table = collections.OrderedDict({
'table_object':
Table([user_details.pk]),
'fields':
contact_info_fields,
'title':
'',
'error_summary_title':
'There was a problem'
})
table_list = create_tables([contact_info_table],
contact_info_name_dict,
contact_info_link_dict)
form = ContactSummaryForm()
variables = {
'form': form,
'application_id': app_id,
'table_list': table_list,
'page_title': 'Check your answers: your sign in details',
'login_details_status': application.login_details_status,
'childcare_type_status': application.childcare_type_status
}
variables = submit_link_setter(variables, table_list, 'login_details',
app_id)
if application.childcare_type_status == 'COMPLETED' or application.childcare_type_status == 'FLAGGED':
variables['submit_link'] = reverse('Task-List-View')
elif application.childcare_type_status != 'COMPLETED':
variables['submit_link'] = reverse(
'Type-Of-Childcare-Guidance-View')
if 'f' in request.GET:
return render(request, 'no-link-summary-template.html', variables)
else:
return render(request, 'generic-summary-template.html', variables)
if request.method == 'POST':
app_id = request.POST["id"]
application = Application.objects.get(pk=app_id)
status.update(app_id, 'login_details_status', 'COMPLETED')
if application.childcare_type_status == 'COMPLETED' or application.childcare_type_status == 'FLAGGED':
return HttpResponseRedirect(
reverse('Task-List-View') + '?id=' + app_id)
elif application.childcare_type_status != 'COMPLETED':
return HttpResponseRedirect(
reverse('Type-Of-Childcare-Guidance-View') + '?id=' + app_id)
variables = {
'application_id': app_id,
'login_details_status': application.login_details_status,
'childcare_type_status': application.childcare_type_status
}
return render(request, 'contact-summary.html', variables)
count_doc[word] = count_doc[word] + 1
for key, value in countall.items():
for k, v in value.items():
tf[k] = v/len(value)
all_word_one[key] = tf
tf = dict()
for key, value in count_doc.items():
idf[key] = math.log(len(count_doc)/value)
for key, value in all_word_one.items():
for k, v in value.items():
if k in idf:
tfidf[k] = v * idf[k]
tfidf = collections.OrderedDict(tfidf)
tfidf_final[key] = tfidf
tfidf = dict()
for key, value in tfidf_final.items():
j = 0
for k, v in value.items():
if j < 3:
fi.append(k)
final[key] = fi
j = j + 1
else:
fi = []
j = j + 1
if (ent['type'] == 'TASK' and ent['confidence'] > 0.75):
tasks.append(ent['text'].lower())
total_tasks.append("_".join(ent['text'].split()))
if (ent['type'] == 'DATASET' and ent['confidence'] > 0.75):
datasets.append(ent['text'].lower())
total_datasets.append("_".join(ent['text'].split()))
if (ent['type'] == 'METRIC' and ent['confidence'] > 0.75):
metrics.append(ent['text'].lower())
total_metrics.append("_".join(ent['text'].split()))
except:
pass
dictionary_yearwise[key].append(sentences.split())
import collections
od = collections.OrderedDict(sorted(dictionary_yearwise.items()))
sentences = {}
i = 1
for k, v in sorted(dictionary_yearwise.items()):
sentences[str(k)] = []
i = 1979
for k, v in sorted(dictionary_yearwise.items()):
for elem in v:
sentences[k].append(elem)
t1_sents = {}
t2_sents = {}
t3_sents = {}
t4_sents = {}
def empty(self): return collections.OrderedDict()
weights = 'vgg16_model_custom.h5'
import multiprocessing as mp
num_processes = 6
pool = mp.Pool(processes=num_processes)
# create datasets
class_to_ix = {}
ix_to_class = {}
with open('classes.txt', 'r') as txt:
classes = [l.strip() for l in txt.readlines()]
class_to_ix = dict(zip(classes, range(len(classes))))
ix_to_class = dict(zip(range(len(classes)), classes))
class_to_ix = {v: k for k, v in ix_to_class.items()}
sorted_class_to_ix = collections.OrderedDict(sorted(class_to_ix.items()))
# Load dataset images and resize to meet minimum width and height pixel size
def load_images(root, min_side=550):
all_imgs = []
all_classes = []
resize_count = 0
invalid_count = 0
for i, subdir in enumerate(listdir(root)):
imgs = listdir(join(root, subdir))
class_ix = class_to_ix[subdir]
print(i, class_ix, subdir)
for img_name in imgs:
img_arr = img.imread(join(root, subdir, img_name))
img_arr_rs = img_arr
def parse_delimited_filenames(files, format_str, delimiter):
"""Parses list of files, extracting data identified by format_str
Parameters
----------
files : list
List of files
format_str : string with python format codes
Provides the naming pattern of the instrument files and the
locations of date information so an ordered list may be produced.
Supports 'year', 'month', 'day', 'hour', 'minute', 'second', 'version',
and 'revision'
Ex: 'cnofs_cindi_ivm_500ms_{year:4d}{month:02d}{day:02d}_v01.cdf'
Returns
-------
OrderedDict
Information parsed from filenames
'year', 'month', 'day', 'hour', 'minute', 'second', 'version',
'revision'
'files' - input list of files
'format_str' - formatted string from input
"""
import collections
# create storage for data to be parsed from filenames
ordered_keys = ['year', 'month', 'day', 'hour', 'minute', 'second',
'version', 'revision']
stored = collections.OrderedDict({kk: list() for kk in ordered_keys})
# exit early if there are no files
if len(files) == 0:
stored['files'] = []
# include format string as convenience for later functions
stored['format_str'] = format_str
return stored
# parse format string to get information needed to parse filenames
search_dict = construct_searchstring_from_format(format_str, wildcard=True)
snips = search_dict['string_blocks']
keys = search_dict['keys']
# going to parse string on delimiter
# it is possible that other regions have the delimiter but aren't
# going to be parsed out
# so apply delimiter breakdown to the string blocks as a guide
pblock = []
parsed_block = [snip.split(delimiter) for snip in snips]
for _ in parsed_block:
if _ != ['', '']:
if _[0] == '':
_ = _[1:]
if _[-1] == '':
_ = _[:-1]
pblock.extend(_)
pblock.append('')
parsed_block = pblock[:-1]
# need to parse out dates for datetime index
for temp in files:
split_name = temp.split(delimiter)
idx = 0
for sname, bname in zip(split_name, parsed_block):
if bname == '':
# areas with data to be parsed are indicated with a
# '' in parsed_block
stored[keys[idx]].append(sname)
idx += 1
# convert to numpy arrays
for key in stored.keys():
stored[key] = np.array(stored[key]).astype(int)
if len(stored[key]) == 0:
stored[key] = None
# include files in output
stored['files'] = files
# include format string as convenience for later functions
stored['format_str'] = format_str
return stored
def __backgroundUpdatePostCall(self, backgroundResult):
for (k, widget) in self.__dataWidgets.items():
widget.setEnabled(True)
self.__busyWidget.setBusy(False)
if self.__scenePlug:
targetPath = GafferSceneUI.ContextAlgo.getLastSelectedPath(
self.getContext())
if targetPath:
if backgroundResult:
self.__locationLabel.setText(targetPath)
else:
self.__locationFrame._qtWidget().setProperty(
"gafferDiff", "Other")
self.__locationFrame._repolish()
self.__locationLabel.setText("Location %s does not exist" %
targetPath)
if isinstance(backgroundResult, IECoreScene.Primitive):
headers = collections.OrderedDict()
primVars = collections.OrderedDict()
toolTips = collections.OrderedDict()
for primvarName in _orderPrimitiveVariables(
backgroundResult.keys()):
primvar = backgroundResult[primvarName]
if not primvar.interpolation in primVars:
headers[primvar.interpolation] = []
primVars[primvar.interpolation] = []
toolTips[primvar.interpolation] = []
headers[primvar.interpolation].append(primvarName)
primVars[primvar.interpolation].append(
conditionPrimvar(primvar))
toolTips[primvar.interpolation].append(
_getPrimvarToolTip(primvarName, primvar))
for interpolation in self.__dataWidgets.keys():
pv = primVars.get(interpolation, None)
h = headers.get(interpolation, None)
t = toolTips.get(interpolation, [])
self.__tabbedContainer.setLabel(
self.__tabbedChildWidgets[interpolation],
str(interpolation) +
(" ({0})".format(len(pv)) if pv else ""))
self.__dataWidgets[interpolation].setToolTips(t)
self.__dataWidgets[interpolation].setHeader(h)
self.__dataWidgets[interpolation].setData(pv)
else:
for interpolation in self.__dataWidgets.keys():
self.__dataWidgets[interpolation].setData(None)
self.__dataWidgets[interpolation].setToolTips([])
self.__tabbedContainer.setLabel(
self.__tabbedChildWidgets[interpolation],
str(interpolation))
class Duration(RDFInteger):
"""Duration value stored in seconds internally."""
data_store_type = "unsigned_integer"
DIVIDERS = collections.OrderedDict(
(("w", 60 * 60 * 24 * 7), ("d", 60 * 60 * 24), ("h", 60 * 60),
("m", 60), ("s", 1)))
def __init__(self, initializer=None, age=None):
super(Duration, self).__init__(None, age)
if isinstance(initializer, Duration):
self._value = initializer._value # pylint: disable=protected-access
elif isinstance(initializer, basestring):
self.ParseFromHumanReadable(initializer)
elif isinstance(initializer, (int, long, float)):
self._value = initializer
elif isinstance(initializer, RDFInteger):
self._value = int(initializer)
elif initializer is None:
self._value = 0
else:
raise InitializeError("Unknown initializer for Duration: %s." %
type(initializer))
@classmethod
def FromSeconds(cls, seconds):
return cls(seconds)
def Validate(self, value, **_):
self.ParseFromString(value)
def ParseFromString(self, string):
self.ParseFromHumanReadable(string)
def SerializeToString(self):
return str(self)
@property
def seconds(self):
return self._value
@property
def microseconds(self):
return self._value * 1000000
def __str__(self):
time_secs = self._value
for label, divider in self.DIVIDERS.items():
if time_secs % divider == 0:
return "%d%s" % (time_secs / divider, label)
def __unicode__(self):
return utils.SmartUnicode(str(self))
def __add__(self, other):
if isinstance(other, (int, long, float, Duration)):
# Assume other is in seconds
return self.__class__(self._value + other)
return NotImplemented
def __iadd__(self, other):
if isinstance(other, (int, long, float, Duration)):
# Assume other is in seconds
self._value += other
return self
return NotImplemented
def __mul__(self, other):
if isinstance(other, (int, long, float, Duration)):
return self.__class__(self._value * other)
return NotImplemented
def __rmul__(self, other):
return self.__mul__(other)
def __sub__(self, other):
if isinstance(other, (int, long, float, Duration)):
# Assume other is in seconds
return self.__class__(self._value - other)
return NotImplemented
def __isub__(self, other):
if isinstance(other, (int, long, float, Duration)):
# Assume other is in seconds
self._value -= other
return self
return NotImplemented
def __abs__(self):
return Duration(abs(self._value))
def Expiry(self, base_time=None):
if base_time is None:
base_time = RDFDatetime.Now()
else:
base_time = base_time.Copy()
base_time_sec = base_time.AsSecondsFromEpoch()
return base_time.FromSecondsFromEpoch(base_time_sec + self._value)
def ParseFromHumanReadable(self, timestring):
"""Parse a human readable string of a duration.
Args:
timestring: The string to parse.
"""
if not timestring:
return
orig_string = timestring
multiplicator = 1
if timestring[-1].isdigit():
pass
else:
try:
multiplicator = self.DIVIDERS[timestring[-1]]
except KeyError:
raise RuntimeError(
"Invalid duration multiplicator: '%s' ('%s')." %
(timestring[-1], orig_string))
timestring = timestring[:-1]
try:
self._value = int(timestring) * multiplicator
except ValueError:
raise InitializeError("Could not parse expiration time '%s'." %
orig_string)
def build_mlp(
in_size: int,
hid_sizes: Iterable[int],
out_size: int = 1,
name: Optional[str] = None,
activation: Type[nn.Module] = nn.ReLU,
squeeze_output=False,
flatten_input=False,
) -> nn.Module:
"""Constructs a Torch MLP.
Args:
in_size: size of individual input vectors; input to the MLP will be of
shape (batch_size, in_size).
hid_sizes: sizes of hidden layers.
out_size: required size of output vector.
activation: activation to apply after hidden layers.
squeeze_output: if out_size=1, then squeeze_input=True ensures that MLP
output is of size (B,) instead of (B,1).
flatten_input: should input be flattened along axes 1, 2, 3, …? Useful
if you want to, e.g., process small images inputs with an MLP.
Returns:
nn.Module: an MLP mapping from inputs of size (batch_size, in_size) to
(batch_size, out_size), unless out_size=1 and squeeze_output=True,
in which case the output is of size (batch_size, ).
Raises:
ValueError: if squeeze_output was supplied with out_size!=1."""
layers = collections.OrderedDict()
if name is None:
prefix = ""
else:
prefix = f"{name}_"
if flatten_input:
layers[f"{prefix}flatten"] = nn.Flatten()
# Hidden layers
prev_size = in_size
for i, size in enumerate(hid_sizes):
layers[f"{prefix}dense{i}"] = nn.Linear(prev_size, size)
prev_size = size
if activation:
layers[f"{prefix}act{i}"] = activation()
# Final layer
layers[f"{prefix}dense_final"] = nn.Linear(prev_size, out_size)
# sigmoid; hehua 20210719 20:59;
layers[f"{prefix}act_final"] = nn.Sigmoid()
if squeeze_output:
if out_size != 1:
raise ValueError("squeeze_output is only applicable when out_size=1")
layers[f"{prefix}squeeze"] = SqueezeLayer()
model = nn.Sequential(layers)
return model
def _cast_spell():
# Complicated because spell mastery
#tuple - name, level, charges
m_spells = [(s[0],s[1],instance.memory.spell_charges[s[0]]) for s in instance.memory.spell_list.items()]
left_hand = main.get_equipped_in_slot(instance.fighter.inventory, 'left hand')
if hasattr(left_hand, 'spell_list') and len(left_hand.spell_list) > 0:
for s in left_hand.flat_spell_list:
if left_hand.spell_list[s] > 0:
m_spells.append((s, left_hand.spell_list[s], left_hand.spell_charges[s]))
#m_spells += [(s[0],s[1],left_hand.spell_charges[s[0]]) for s in left_hand.spell_list.items() if s[1] > 0]
if len(m_spells) < 1:
ui.message("You have no spells available", libtcod.light_blue)
return 'didnt-take-turn'
else:
letter_index = ord('a')
ops = collections.OrderedDict()
sp = {}
for spell,level,charges in m_spells:
spell_data = spells.library[spell]
name_color = libtcod.white
stamina_cost = spell_data.levels[level-1]['stamina_cost']
if stamina_cost > instance.fighter.stamina:
stamina_color = libtcod.dark_red
name_color = libtcod.dark_gray
else:
stamina_color = libtcod.dark_green
miscast = get_miscast_chance(spells.library[spell])
miscast_color = libtcod.color_lerp(libtcod.blue, libtcod.red, float(miscast) / 100.0)
if miscast >= 100:
name_color = libtcod.dark_gray
if charges <= 0:
charges_color = libtcod.dark_red
name_color = libtcod.dark_gray
else:
charges_color = libtcod.yellow
d = collections.OrderedDict()
d['spell'] = {
'text' : spell_data.name.title(),
'color' : name_color
}
d['stamina'] = {
'text': '[%d]' % stamina_cost,
'color': stamina_color
}
d['charges'] = {
'text': '%d/%d' % (charges, spell_data.max_spell_charges(level)),
'color': charges_color
}
d['miscast'] = {
'text': str(miscast) + '%%',
'color': miscast_color
}
d['description'] = string.capwords(spell_data.name) + '\n\n' + spell_data.description
ops[chr(letter_index)] = d
sp[chr(letter_index)] = spell
letter_index += 1
selection = ui.menu_ex('Cast which spell?', ops, 50, return_as_char=True)
if selection is not None:
s = sp[selection]
if left_hand is not None:
level = left_hand.spell_list[s]
cost = spells.library[s].levels[level-1]['stamina_cost']
cast_check = left_hand.can_cast(s,instance)
if cast_check == 'miscast':
instance.fighter.adjust_stamina(-2 * cost)
return 'miscast'
elif cast_check:
result = actions.invoke_ability(spells.library[s].ability_name,instance,
spell_context=spells.library[s].levels[level-1])
if result == 'success':
left_hand.spell_charges[s] -= 1
if main.has_skill('blood_magic') and instance.fighter.stamina < cost:
instance.fighter.hp -= cost - instance.fighter.stamina
instance.fighter.stamina = 0
else:
instance.fighter.adjust_stamina(-cost)
instance.fighter.apply_status_effect(effects.meditate())
return 'cast-spell'
else:
return 'didnt-take-turn'
else:
return 'didnt-take-turn'
if s in instance.memory.spell_list and instance.memory.can_cast(s, instance):
cast_check = instance.memory.can_cast(s, instance)
level = instance.memory.spell_list[s]
cost = spells.library[s].levels[level - 1]['stamina_cost']
if cast_check == 'miscast':
instance.fighter.adjust_stamina(-2 * cost)
return 'miscast'
elif cast_check:
result = actions.invoke_ability(spells.library[s].ability_name, instance,
spell_context=spells.library[s].levels[level - 1])
if result == 'success':
instance.memory.spell_charges[s] -= 1
if main.has_skill('blood_magic') and instance.fighter.stamina < cost:
instance.fighter.hp -= cost - instance.fighter.stamina
instance.fighter.stamina = 0
else:
instance.fighter.adjust_stamina(-cost)
instance.fighter.apply_status_effect(effects.StatusEffect('meditate', time_limit=None,
color=libtcod.yellow, description='Meditating will renew your missing spells.'))
return 'cast-spell'
else:
return 'didnt-take-turn'
else:
return 'didnt-take-turn'
else:
return 'didnt-take-turn'
return 'didnt-take-turn'
def dict_constructor(loader, node):
return collections.OrderedDict(loader.construct_pairs(node))
def write_instance_to_example_files(instances, tokenizer, max_seq_length,
max_predictions_per_seq, output_files):
"""Create TF example files from `TrainingInstance`s."""
writers = []
for output_file in output_files:
writers.append(tf.python_io.TFRecordWriter(output_file))
writer_index = 0
total_written = 0
for (inst_index, instance) in enumerate(instances):
input_ids = tokenizer.convert_tokens_to_ids(instance.tokens)
input_mask = [1] * len(input_ids)
segment_ids = list(instance.segment_ids)
assert len(input_ids) <= max_seq_length
while len(input_ids) < max_seq_length:
input_ids.append(0)
input_mask.append(0)
segment_ids.append(0)
assert len(input_ids) == max_seq_length
assert len(input_mask) == max_seq_length
assert len(segment_ids) == max_seq_length
masked_lm_positions = list(instance.masked_lm_positions)
masked_lm_ids = tokenizer.convert_tokens_to_ids(
instance.masked_lm_labels)
masked_lm_weights = [1.0] * len(masked_lm_ids)
while len(masked_lm_positions) < max_predictions_per_seq:
masked_lm_positions.append(0)
masked_lm_ids.append(0)
masked_lm_weights.append(0.0)
next_sentence_label = 1 if instance.is_random_next else 0
features = collections.OrderedDict()
features["input_ids"] = create_int_feature(input_ids)
features["input_mask"] = create_int_feature(input_mask)
features["segment_ids"] = create_int_feature(segment_ids)
features["masked_lm_positions"] = create_int_feature(
masked_lm_positions)
features["masked_lm_ids"] = create_int_feature(masked_lm_ids)
features["masked_lm_weights"] = create_float_feature(masked_lm_weights)
features["next_sentence_labels"] = create_int_feature(
[next_sentence_label])
tf_example = tf.train.Example(features=tf.train.Features(
feature=features))
writers[writer_index].write(tf_example.SerializeToString())
writer_index = (writer_index + 1) % len(writers)
total_written += 1
if inst_index < 20:
tf.logging.info("*** Example ***")
tf.logging.info("tokens: %s" % " ".join(
[tokenization.printable_text(x) for x in instance.tokens]))
for feature_name in features.keys():
feature = features[feature_name]
values = []
if feature.int64_list.value:
values = feature.int64_list.value
elif feature.float_list.value:
values = feature.float_list.value
tf.logging.info(
"%s: %s" %
(feature_name, " ".join([str(x) for x in values])))
for writer in writers:
writer.close()
tf.logging.info("Wrote %d total instances", total_written)
def _skip(self, data):
return collections.OrderedDict({k:v for k, v in data.items() if k not in self._skip_keys})
def test_notes_str(self):
self.trade.notes = collections.OrderedDict({"1": 1, 2: "2", 3: 3, 4: "four"})
self.assertEqual(self.trade.notes_str, "1,2,3,four")
self.trade.notes = collections.OrderedDict()
self.assertEqual(self.trade.notes_str, "")
print "================== Firewall Rules ========================"
print "=========================================================="
print ""
print "Source Zone,Dest Zone,Source Net,Dest Net, Dest Service, Action, Comment"
for x in rules:
print '%s,%s,%s,%s,%s,%s,%s' % (
x["ruleSrcZone"], x["ruleDestZone"], x["ruleSrcNet"], x["ruleDestNet"],
x["ruleDestService"], x["ruleAction"], x["ruleComment"])
print ""
print "=========================================================="
print "================== Address Objects ======================="
print "=========================================================="
print ""
print "Address Name,Zone,IP,Subnet"
oAddrObjects = collections.OrderedDict(sorted(addrObjects.items()))
for addr, addrFields in oAddrObjects.iteritems():
print '%s,%s,%s,%s' % (addr, addrFields["addrZone"], addrFields["addrIP"],
addrFields["addrSubnet"])
print ""
print "=========================================================="
print "================== Address Groups ========================"
print "=========================================================="
print ""
for group, groupObjects in addrGroups.iteritems():
print group
for groupObj in groupObjects:
print "\t%s" % groupObj
print ""
print ""
import os import time import collections import numpy as np import tidy_headers # --- define ------------------------------------------------------------------------------------- here = os.path.abspath(os.path.dirname(__file__)) # --- workspace ---------------------------------------------------------------------------------- filepath = os.path.join(here, 'minimal.txt') # create metadata meta = collections.OrderedDict() meta['time'] = time.time() meta['name'] = 'Blaise Thompson' meta['colors'] = ['red', 'blue', 'green', 'white'] meta['number'] = 42 meta['array'] = np.random.random((5, 3)) # write metadata tidy_headers.write(filepath, meta) # read metadata print(tidy_headers.read(filepath))
from geometry_msgs.msg import Twist from std_msgs.msg import ColorRGBA, Float32, Bool import numpy as np import collections import time import copy import random as rand import random as rand import numpy as np import pickle import time import glob from GUI import experiment as getFeedback from DemoNotification import show_experiment_notification PRIMITIVES_CONFIGS = collections.OrderedDict( ) #Ordered dict with values as [amplitudes,frequency] #no movement PRIMITIVES_CONFIGS['no_action'] = [0, 0, 0] #oscillatory movement PRIMITIVES_CONFIGS['oscillate_smallAmp_slow'] = [50, 0, .5] PRIMITIVES_CONFIGS['oscillate_smallAmp_medium'] = [100, 0, 1] PRIMITIVES_CONFIGS['oscillate_smallAmp_fast'] = [300, 0, 3] PRIMITIVES_CONFIGS['oscillate_largeAmp_slow'] = [50, 0, .25] PRIMITIVES_CONFIGS['oscillate_largeAmp_medium'] = [100, 0, .5] #spinning movement PRIMITIVES_CONFIGS['spin_forward_slow'] = [25, 25, .25] PRIMITIVES_CONFIGS['spin_forward_medium'] = [50, 50, .5]
def parse_fixed_width_filenames(files, format_str):
"""Parses list of files, extracting data identified by format_str
Parameters
----------
files : list
List of files
format_str : string with python format codes
Provides the naming pattern of the instrument files and the
locations of date information so an ordered list may be produced.
Supports 'year', 'month', 'day', 'hour', 'minute', 'second', 'version',
and 'revision'
Ex: 'cnofs_cindi_ivm_500ms_{year:4d}{month:02d}{day:02d}_v01.cdf'
Returns
-------
OrderedDict
Information parsed from filenames
'year', 'month', 'day', 'hour', 'minute', 'second', 'version',
'revision'
'files' - input list of files
"""
import collections
# create storage for data to be parsed from filenames
stored = collections.OrderedDict()
stored['year'] = []
stored['month'] = []
stored['day'] = []
stored['hour'] = []
stored['minute'] = []
stored['second'] = []
stored['version'] = []
stored['revision'] = []
if len(files) == 0:
stored['files'] = []
# include format string as convenience for later functions
stored['format_str'] = format_str
return stored
# parse format string to get information needed to parse filenames
search_dict = construct_searchstring_from_format(format_str)
snips = search_dict['string_blocks']
lengths = search_dict['lengths']
keys = search_dict['keys']
# determine the locations the date/version information in a filename is
# stored use these indices to slice out date from filenames
idx = 0
begin_key = []
end_key = []
for i, snip in enumerate(snips):
idx += len(snip)
if i < (len(lengths)):
begin_key.append(idx)
idx += lengths[i]
end_key.append(idx)
max_len = idx
# setting up negative indexing to pick out filenames
key_str_idx = [np.array(begin_key, dtype=int) - max_len,
np.array(end_key, dtype=int) - max_len]
# need to parse out dates for datetime index
for i, temp in enumerate(files):
for j, key in enumerate(keys):
val = temp[key_str_idx[0][j]:key_str_idx[1][j]]
stored[key].append(val)
# convert to numpy arrays
for key in stored.keys():
stored[key] = np.array(stored[key]).astype(int)
if len(stored[key]) == 0:
stored[key] = None
# include files in output
stored['files'] = files
# include format string as convenience for later functions
stored['format_str'] = format_str
return stored
def extend_results(cls, results, serializable=False):
"""Modify and extend results with aggregated data.
This is a workaround method that tries to adapt task results
to schema of planned DB refactoring, so this method is expected
to be simplified after DB refactoring since all the data should
be taken as-is directly from the database.
Each scenario results have extra `info' with aggregated data,
and iterations data is represented by iterator - this simplifies
its future implementation as generator and gives ability to process
arbitrary number of iterations with low memory usage.
:param results: list of db.sqlalchemy.models.TaskResult
:param serializable: bool, whether to convert json non-serializable
types (like datetime) to serializable ones
:returns: list of dicts, each dict represents scenario results:
key - dict, scenario input data
sla - list, SLA results
iterations - if serializable, then iterator with
iterations data, otherwise a list
created_at - if serializable, then str datetime,
otherwise absent
updated_at - if serializable, then str datetime,
otherwise absent
info:
atomic - dict where key is one of atomic action names
and value is dict {min_duration: number,
max_duration: number}
iterations_count - int number of iterations
iterations_failed - int number of iterations with errors
min_duration - float minimum iteration duration
max_duration - float maximum iteration duration
tstamp_start - float timestamp of the first iteration
full_duration - float full scenario duration
load_duration - float load scenario duration
"""
extended = []
for scenario_result in results:
scenario = dict(scenario_result)
tstamp_start = 0
min_duration = 0
max_duration = 0
iterations_failed = 0
atomic = collections.OrderedDict()
for itr in scenario["data"]["raw"]:
for atomic_name, duration in itr["atomic_actions"].items():
duration = duration or 0
if atomic_name not in atomic:
atomic[atomic_name] = {"min_duration": duration,
"max_duration": duration}
elif duration < atomic[atomic_name]["min_duration"]:
atomic[atomic_name]["min_duration"] = duration
elif duration > atomic[atomic_name]["max_duration"]:
atomic[atomic_name]["max_duration"] = duration
if not tstamp_start or itr["timestamp"] < tstamp_start:
tstamp_start = itr["timestamp"]
if "output" not in itr:
itr["output"] = {"additive": [], "complete": []}
# NOTE(amaretskiy): Deprecated "scenario_output"
# is supported for backward compatibility
if ("scenario_output" in itr
and itr["scenario_output"]["data"]):
itr["output"]["additive"].append(
{"items": itr["scenario_output"]["data"].items(),
"title": "Scenario output",
"description": "",
"chart": "OutputStackedAreaChart"})
del itr["scenario_output"]
if itr["error"]:
iterations_failed += 1
else:
duration = itr["duration"] or 0
if not min_duration or duration < min_duration:
min_duration = duration
if not max_duration or duration > max_duration:
max_duration = duration
for k in "created_at", "updated_at":
if serializable:
# NOTE(amaretskiy): convert datetime to str,
# because json.dumps() does not like datetime
if scenario[k]:
scenario[k] = scenario[k].strftime("%Y-%d-%mT%H:%M:%S")
else:
del scenario[k]
durations_stat = charts.MainStatsTable(
{"iterations_count": len(scenario["data"]["raw"]),
"atomic": atomic})
for itr in scenario["data"]["raw"]:
durations_stat.add_iteration(itr)
scenario["info"] = {
"stat": durations_stat.render(),
"atomic": atomic,
"iterations_count": len(scenario["data"]["raw"]),
"iterations_failed": iterations_failed,
"min_duration": min_duration,
"max_duration": max_duration,
"tstamp_start": tstamp_start,
"full_duration": scenario["data"]["full_duration"],
"load_duration": scenario["data"]["load_duration"]}
iterations = sorted(scenario["data"]["raw"],
key=lambda itr: itr["timestamp"])
if serializable:
scenario["iterations"] = list(iterations)
else:
scenario["iterations"] = iter(iterations)
scenario["sla"] = scenario["data"]["sla"]
scenario["hooks"] = scenario["data"].get("hooks", [])
del scenario["data"]
del scenario["task_uuid"]
del scenario["id"]
extended.append(scenario)
return extended
def __init__(self, capacity):
self.capacity = capacity
self.cache = collections.OrderedDict()
def statistics(
initimage, imagewin=None,
vistable=None, amptable=None, bstable=None, catable=None,
lambl1=1., lambtv=-1, lambtsv=1, logreg=False, normlambda=True,
totalflux=None, fluxconst=False,
istokes=0, ifreq=0, fulloutput=True, **args):
'''
'''
# Check Arguments
if ((vistable is None) and (amptable is None) and
(bstable is None) and (catable is None)):
print("Error: No data are input.")
return -1
# Total Flux constraint: Sanity Check
dofluxconst = False
if ((vistable is None) and (amptable is None) and (totalflux is None)):
print("Error: No absolute amplitude information in the input data.")
print(" You need to set the total flux constraint by totalflux.")
return -1
elif ((totalflux is None) and (fluxconst is True)):
print("Error: No total flux is specified, although you set fluxconst=True.")
print(" You need to set the total flux constraint by totalflux.")
return -1
elif ((vistable is None) and (amptable is None) and
(totalflux is not None) and (fluxconst is False)):
print("Warning: No absolute amplitude information in the input data.")
print(" The total flux will be constrained, although you do not set fluxconst=True.")
dofluxconst = True
elif fluxconst is True:
dofluxconst = True
# get initial images
Iin = np.float64(initimage.data[istokes, ifreq])
# size of images
Nx = np.int32(initimage.header["nx"])
Ny = np.int32(initimage.header["ny"])
Nyx = Nx * Ny
# pixel coordinates
x, y = initimage.get_xygrid(twodim=True, angunit="rad")
x = np.float64(x)
y = np.float64(y)
xidx = np.int32(np.arange(Nx) + 1)
yidx = np.int32(np.arange(Ny) + 1)
xidx, yidx = np.meshgrid(xidx, yidx)
# apply the imaging area
if imagewin is None:
print("Imaging Window: Not Specified. We solve the image on all the pixels.")
Iin = Iin.reshape(Nyx)
x = x.reshape(Nyx)
y = y.reshape(Nyx)
xidx = xidx.reshape(Nyx)
yidx = yidx.reshape(Nyx)
else:
print("Imaging Window: Specified. Images will be solved on specified pixels.")
idx = np.where(imagewin)
Iin = Iin[idx]
x = x[idx]
y = y[idx]
xidx = xidx[idx]
yidx = yidx[idx]
# dammy array
dammyreal = np.zeros(1, dtype=np.float64)
# Full Complex Visibility
Ndata = 0
if vistable is None:
fcvtable = None
else:
fcvtable = vistable.copy()
if fcvtable is None:
isfcv = False
vfcvr = dammyreal
vfcvi = dammyreal
varfcv = dammyreal
else:
isfcv = True
phase = np.deg2rad(np.array(fcvtable["phase"], dtype=np.float64))
amp = np.array(fcvtable["amp"], dtype=np.float64)
vfcvr = amp * np.cos(phase)
vfcvi = amp * np.sin(phase)
varfcv = np.square(np.array(fcvtable["sigma"], dtype=np.float64))
Ndata += len(vfcvr)
del phase, amp
# Visibility Amplitude
if amptable is None:
isamp = False
vamp = dammyreal
varamp = dammyreal
else:
isamp = True
vamp = np.array(amptable["amp"], dtype=np.float64)
varamp = np.square(np.array(amptable["sigma"], dtype=np.float64))
Ndata += len(vamp)
# Closure Phase
if bstable is None:
iscp = False
cp = dammyreal
varcp = dammyreal
else:
iscp = True
cp = np.deg2rad(np.array(bstable["phase"], dtype=np.float64))
varcp = np.square(
np.array(bstable["sigma"] / bstable["amp"], dtype=np.float64))
Ndata += len(cp)
# Closure Amplitude
if catable is None:
isca = False
ca = dammyreal
varca = dammyreal
else:
isca = True
ca = np.array(catable["logamp"], dtype=np.float64)
varca = np.square(np.array(catable["logsigma"], dtype=np.float64))
Ndata += len(ca)
# Normalize Lambda
if normlambda:
# Guess Total Flux
if totalflux is None:
fluxscale = []
if vistable is not None:
fluxscale.append(vistable["amp"].max())
if amptable is not None:
fluxscale.append(amptable["amp"].max())
fluxscale = np.max(fluxscale)
print("Flux Scaling Factor for lambda: The expected total flux is not given.")
print("The scaling factor will be %g" % (fluxscale))
else:
fluxscale = np.float64(totalflux)
print("Flux Scaling Factor for lambda: The scaling factor will be %g" % (fluxscale))
if logreg:
lambl1_sim = lambl1 / (len(xidx)*np.log(1+fluxscale/len(xidx)))
lambtv_sim = lambtv / (len(xidx)*np.log(1+fluxscale/len(xidx)))
lambtsv_sim = lambtsv / (len(xidx)*np.log(1+fluxscale/len(xidx)))**2
else:
lambl1_sim = lambl1 / fluxscale
lambtv_sim = lambtv / fluxscale
lambtsv_sim = lambtsv / fluxscale**2
else:
lambl1_sim = lambl1
lambtv_sim = lambtv
lambtsv_sim = lambtsv
# get uv coordinates and uv indice
u, v, uvidxfcv, uvidxamp, uvidxcp, uvidxca = _get_uvlist(
fcvtable=fcvtable, amptable=amptable, bstable=bstable, catable=catable
)
# calculate all
out = fortlib.dftim2d.statistics(
# Images
iin=Iin, x=x, y=y, xidx=xidx, yidx=yidx, nx=Nx, ny=Ny,
# UV coordinates,
u=u, v=v,
# Imaging Parameters
lambl1=lambl1_sim, lambtv=lambtv_sim, lambtsv=lambtsv_sim,
logreg=logreg,
# Full Complex Visibilities
isfcv=isfcv, uvidxfcv=uvidxfcv, vfcvr=vfcvr, vfcvi=vfcvi, varfcv=varfcv,
# Visibility Ampltiudes
isamp=isamp, uvidxamp=uvidxamp, vamp=vamp, varamp=varamp,
# Closure Phase
iscp=iscp, uvidxcp=uvidxcp, cp=cp, varcp=varcp,
# Closure Amplituds
isca=isca, uvidxca=uvidxca, ca=ca, varca=varca
)
stats = collections.OrderedDict()
# Cost and Chisquares
stats["cost"] = out[0]
stats["chisq"] = out[3] + out[4] + out[5] + out[6]
stats["rchisq"] = stats["chisq"] / Ndata
stats["isfcv"] = isfcv
stats["isamp"] = isamp
stats["iscp"] = iscp
stats["isca"] = isca
stats["chisqfcv"] = out[3]
stats["chisqamp"] = out[4]
stats["chisqcp"] = out[5]
stats["chisqca"] = out[6]
stats["rchisqfcv"] = out[3] / len(vfcvr)
stats["rchisqamp"] = out[4] / len(vamp)
stats["rchisqcp"] = out[5] / len(cp)
stats["rchisqca"] = out[6] / len(ca)
# Regularization functions
if lambl1 > 0:
stats["lambl1"] = lambl1
stats["lambl1_sim"] = lambl1_sim
stats["l1"] = out[7]
stats["l1cost"] = out[7] * lambl1_sim
else:
stats["lambl1"] = 0.
stats["lambl1_sim"] = 0.
stats["l1"] = out[7]
stats["l1cost"] = 0.
if lambtv > 0:
stats["lambtv"] = lambtv
stats["lambtv_sim"] = lambtv_sim
stats["tv"] = out[8]
stats["tvcost"] = out[8] * lambtv_sim
else:
stats["lambtv"] = 0.
stats["lambtv_sim"] = 0.
stats["tv"] = out[8]
stats["tvcost"] = 0.
if lambtsv > 0:
stats["lambtsv"] = lambtsv
stats["lambtsv_sim"] = lambtsv_sim
stats["tsv"] = out[9]
stats["tsvcost"] = out[9] * lambtsv_sim
else:
stats["lambtsv"] = 0.
stats["lambtsv_sim"] = 0.
stats["tsv"] = out[9]
stats["tsvcost"] = 0.
if fulloutput:
# gradcost
gradcostimage = initimage.data[istokes, ifreq, :, :].copy()
gradcostimage[:, :] = 0.
for i in np.arange(len(xidx)):
gradcostimage[yidx[i] - 1, xidx[i] - 1] = out[1][i]
stats["gradcost"] = gradcostimage
del gradcostimage
if isfcv:
stats["fcvampmod"] = np.sqrt(out[10] * out[10] + out[11] * out[11])
stats["fcvphamod"] = np.angle(out[10] + 1j * out[11], deg=True)
stats["fcvrmod"] = out[10]
stats["fcvimod"] = out[11]
stats["fcvres"] = out[12]
else:
stats["fcvampmod"] = None
stats["fcvphamod"] = None
stats["fcvres"] = None
if isamp:
stats["ampmod"] = out[13]
stats["ampres"] = out[14]
else:
stats["ampmod"] = None
stats["ampres"] = None
if iscp:
stats["cpmod"] = np.rad2deg(out[15])
stats["cpres"] = np.rad2deg(out[16])
else:
stats["cpmod"] = None
stats["cpres"] = None
if isca:
stats["camod"] = out[17]
stats["cares"] = out[18]
else:
stats["camod"] = None
stats["cares"] = None
return stats
def write_predictions(logger,
all_examples,
all_features,
all_results,
n_best_size,
max_answer_length,
do_lower_case,
output_prediction_file,
output_nbest_file,
verbose_logging,
write_prediction=True,
with_key=False,
is_bridge=True):
"""Write final predictions to the json file."""
example_index_to_features = collections.defaultdict(list)
for feature in all_features:
example_index_to_features[feature.example_index].append(feature)
unique_id_to_result = {}
for result in all_results:
unique_id_to_result[result.unique_id] = result
_PrelimPrediction = collections.namedtuple( # pylint: disable=invalid-name
"PrelimPrediction",
["start_index", "end_index", "keyword_index", "logit"])
all_predictions = collections.OrderedDict()
all_nbest_json = collections.OrderedDict()
for (example_index, example) in enumerate(all_examples):
features = example_index_to_features[example_index]
prelim_predictions = []
yn_predictions = []
feature = sorted(features, key=lambda f: f.unique_id)[0]
gold_start_positions = feature.start_position
gold_end_positions = feature.end_position
result = unique_id_to_result[feature.unique_id]
switch = np.argmax(result.switch)
if switch == 1:
prelim_predictions.append(
_PrelimPrediction(start_index=-1,
end_index=-1,
keyword_index=-1,
logit=result.switch[1]))
elif switch == 0:
scores = []
start_logits = result.start_logits[:len(feature.tokens)]
end_logits = result.end_logits[:len(feature.tokens)]
if with_key:
keyword_logits = result.keyword_logits[:len(feature.tokens)]
for (i, s) in enumerate(start_logits):
for (j, e) in enumerate(end_logits[i:]):
for (k, key) in enumerate(keyword_logits[i:i + j + 1]):
if not (i == 0 and j == len(feature.tokens) - 1):
scores.append(((i, i + j, i + k), s + e + key))
else:
for (i, s) in enumerate(start_logits):
for (j, e) in enumerate(end_logits[i:]):
scores.append(((i, i + j, i), s + e))
scores = sorted(scores, key=lambda x: x[1], reverse=True)
for (start_index, end_index, keyword_index), score in scores:
if start_index >= len(feature.tokens):
continue
if end_index >= len(feature.tokens):
continue
if not (start_index <= keyword_index <= end_index):
continue
if start_index not in feature.token_to_orig_map or end_index not in feature.token_to_orig_map:
continue
if start_index - 1 in feature.token_to_orig_map and feature.token_to_orig_map[
start_index -
1] == feature.token_to_orig_map[start_index]:
continue
if end_index + 1 in feature.token_to_orig_map and feature.token_to_orig_map[
end_index + 1] == feature.token_to_orig_map[end_index]:
continue
if end_index < start_index:
continue
length = end_index - start_index + 1
prelim_predictions.append(
_PrelimPrediction(start_index=start_index,
end_index=end_index,
keyword_index=keyword_index,
logit=score))
else:
raise NotImplementedError()
prelim_predictions = sorted(prelim_predictions,
key=lambda x: x.logit,
reverse=True)
if len(prelim_predictions) == 0:
embed()
assert False
_NbestPrediction = collections.namedtuple( # pylint: disable=invalid-name
"NbestPrediction", ["text", "text2", "logit"])
seen_predictions = {}
nbest = []
def get_text(start_index, end_index, keyword_index):
if start_index == end_index == -1:
final_text = example.all_answers[-1]
else:
feature = features[0]
tok_tokens = feature.tokens[start_index:(end_index + 1)]
orig_doc_start = feature.token_to_orig_map[start_index]
orig_doc_end = feature.token_to_orig_map[end_index]
orig_doc_keyword = feature.token_to_orig_map[keyword_index]
orig_tokens = feature.doc_tokens[orig_doc_start:(orig_doc_end +
1)]
orig_tokens2 = orig_tokens.copy()
for i in range(orig_doc_keyword, orig_doc_keyword - 5, -1):
if i - orig_doc_start < 0: break
if orig_tokens[i - orig_doc_start] in ['the', 'a', 'an']:
orig_tokens2[i - orig_doc_start] = 'which'
assert orig_tokens[i - orig_doc_start] != 'which'
break
tok_text = " ".join(tok_tokens)
# De-tokenize WordPieces that have been split off.
tok_text = tok_text.replace(" ##", "")
tok_text = tok_text.replace("##", "")
# Clean whitespace
tok_text = tok_text.strip()
tok_text = " ".join(tok_text.split())
final_text = get_final_text(tok_text, " ".join(orig_tokens), do_lower_case, \
logger, verbose_logging)
final_text2 = get_final_text(tok_text, " ".join(orig_tokens2), do_lower_case, \
logger, verbose_logging)
if '##' in final_text:
print(tok_text)
print(' '.join(orig_tokens))
print(final_text)
embed()
assert False
return final_text, final_text2
for pred in prelim_predictions:
if len(nbest) >= n_best_size:
break
final_text, final_text2 = get_text(pred.start_index,
pred.end_index,
pred.keyword_index)
if final_text in seen_predictions:
continue
nbest.append(
_NbestPrediction(text=final_text,
text2=final_text2,
logit=pred.logit))
# In very rare edge cases we could have no valid predictions. So we
# just create a nonce prediction in this case to avoid failure.
if not nbest:
nbest.append(
_NbestPrediction(text="empty", text2="empty", logit=0.0))
assert len(nbest) >= 1
total_scores = []
for entry in nbest:
total_scores.append(entry.logit)
probs = _compute_softmax(total_scores)
nbest_json = []
for (i, entry) in enumerate(nbest):
output = collections.OrderedDict()
output['text'] = entry.text
output['text2'] = entry.text2
output['probability'] = probs[i]
output['logit'] = entry.logit
nbest_json.append(output)
assert len(nbest_json) >= 1
all_predictions[example.qas_id] = (
nbest_json[0]["text"],
nbest_json[0]["text2"],
example.all_answers[:-1], # groundtruth
example.all_answers[-1]) # orig_question
all_nbest_json[example.qas_id] = nbest_json
f1_scores = []
for (prediction, _, groundtruth,
orig_question) in all_predictions.values():
f1_scores.append(
max([f1_score(prediction, gt)[0] for gt in groundtruth]))
if write_prediction:
logger.info("Writing predictions to: %s" % (output_prediction_file))
logger.info("Writing nbest to: %s" % (output_nbest_file))
final_predictions = {}
final_nbest_predictions = defaultdict(list)
for key in all_predictions:
orig_question = all_predictions[key][-1]
for d in all_nbest_json[key]:
orig_question, question1, question2 = \
get_decomposed(orig_question, d['text'], d['text2'], is_bridge, with_key)
final_nbest_predictions[key].append(
(question1, question2, orig_question, orig_question))
final_predictions[key] = final_nbest_predictions[key][0]
l = [
v for k, v in sorted(final_predictions.items(), key=lambda x: x[0])
]
print(l[0])
with open(output_prediction_file, "w") as writer:
writer.write(json.dumps(final_predictions, indent=4) + "\n")
with open(output_nbest_file, "w") as writer:
writer.write(json.dumps(final_nbest_predictions, indent=4) + "\n")
return np.mean(f1_scores)
def pipeline(
initimage,
imagefunc=iterative_imaging,
imageprm={},
imagefargs={},
lambl1s=[-1.],
lambtvs=[-1.],
lambtsvs=[-1.],
workdir="./",
skip=False,
sumtablefile="summary.csv",
docv=False,
seed=1,
nfold=10,
cvsumtablefile="summary.cv.csv",
angunit="uas",
uvunit="gl"):
'''
A pipeline imaging function using imaging and related fucntions.
Args:
initimage (imdata.IMFITS object):
initial image
imagefunc (function; default=uvdata.iterative_imaging):
Function of imageing. It should be defined as
def imagefunc(initimage, imageprm, **imagefargs)
imageprm (dict-like; default={}):
parameter sets for each imaging
imagefargs (dict-like; default={}):
parameter sets for imagefunc
workdir (string; default = "./"):
The directory where images and summary files will be output.
sumtablefile (string; default = "summary.csv"):
The name of the output csv file that summerizes results.
docv (boolean; default = False):
Do cross validation
seed (integer; default = 1):
Random seed to make CV data sets.
nfold (integer; default = 10):
Number of folds in CV.
cvsumtablefile (string; default = "cvsummary.csv"):
The name of the output csv file that summerizes results of CV.
angunit (string; default = None):
Angular units for plotting results.
uvunit (string; default = None):
Units of baseline lengths for plotting results.
Returns:
sumtable:
pd.DataFrame table summerising statistical quantities of each
parameter set.
cvsumtable (if docv=True):
pd.DataFrame table summerising results of cross validation.
'''
if not os.path.isdir(workdir):
os.makedirs(workdir)
cvworkdir = os.path.join(workdir,"cv")
if docv:
if not os.path.isdir(cvworkdir):
os.makedirs(cvworkdir)
# Lambda Parameters
lambl1s = -np.sort(-np.asarray(lambl1s))
lambtvs = -np.sort(-np.asarray(lambtvs))
lambtsvs = -np.sort(-np.asarray(lambtsvs))
nl1 = len(lambl1s)
ntv = len(lambtvs)
ntsv = len(lambtsvs)
# Summary Data
sumtable = pd.DataFrame()
if docv:
cvsumtable = pd.DataFrame()
isvistable = False
isamptable = False
isbstable = False
iscatable = False
if "vistable" in imageprm.keys():
if imageprm["vistable"] is not None:
isvistable = True
vistables = imageprm["vistable"].gencvtables(nfold=nfold, seed=seed)
if "amptable" in imageprm.keys():
if imageprm["amptable"] is not None:
isamptable = True
amptables = imageprm["amptable"].gencvtables(nfold=nfold, seed=seed)
if "bstable" in imageprm.keys():
if imageprm["bstable"] is not None:
isbstable = True
bstables = imageprm["bstable"].gencvtables(nfold=nfold, seed=seed)
if "catable" in imageprm.keys():
if imageprm["catable"] is not None:
iscatable = True
catables = imageprm["catable"].gencvtables(nfold=nfold, seed=seed)
# Start Imaging
for itsv, itv, il1 in itertools.product(np.arange(ntsv),
np.arange(ntv),
np.arange(nl1)):
header = "tsv%02d.tv%02d.l1%02d" % (itsv, itv, il1)
# output
imageprm["lambl1"] = lambl1s[il1]
imageprm["lambtv"] = lambtvs[itv]
imageprm["lambtsv"] = lambtsvs[itsv]
# Imaging and Plotting Results
filename = header + ".fits"
filename = os.path.join(workdir, filename)
if (skip is False) or (os.path.isfile(filename) is False):
newimage = imagefunc(initimage, imageprm=imageprm, **imagefargs)
newimage.save_fits(filename)
else:
newimage = imdata.IMFITS(filename)
filename = header + ".summary.pdf"
filename = os.path.join(workdir, filename)
plots(newimage, imageprm, filename=filename,
angunit=angunit, uvunit=uvunit)
newstats = statistics(newimage, fulloutput=False, **imageprm)
# Make Summary
tmpsum = collections.OrderedDict()
tmpsum["itsv"] = itsv
tmpsum["itv"] = itv
tmpsum["il1"] = il1
for key in newstats.keys():
tmpsum[key] = newstats[key]
# Cross Validation
if docv:
# Initialize Summary Table
# add keys
tmpcvsum = pd.DataFrame()
tmpcvsum["icv"] = np.arange(nfold)
tmpcvsum["itsv"] = np.zeros(nfold, dtype=np.int32)
tmpcvsum["itv"] = np.zeros(nfold, dtype=np.int32)
tmpcvsum["il1"] = np.zeros(nfold, dtype=np.int32)
tmpcvsum["lambtsv"] = np.zeros(nfold, dtype=np.float64)
tmpcvsum["lambtv"] = np.zeros(nfold, dtype=np.float64)
tmpcvsum["lambl1"] = np.zeros(nfold, dtype=np.float64)
tmpcvsum["tchisq"] = np.zeros(nfold, dtype=np.float64)
tmpcvsum["trchisq"] = np.zeros(nfold, dtype=np.float64)
tmpcvsum["tchisqfcv"] = np.zeros(nfold, dtype=np.float64)
tmpcvsum["tchisqamp"] = np.zeros(nfold, dtype=np.float64)
tmpcvsum["tchisqcp"] = np.zeros(nfold, dtype=np.float64)
tmpcvsum["tchisqca"] = np.zeros(nfold, dtype=np.float64)
tmpcvsum["trchisqfcv"] = np.zeros(nfold, dtype=np.float64)
tmpcvsum["trchisqamp"] = np.zeros(nfold, dtype=np.float64)
tmpcvsum["trchisqcp"] = np.zeros(nfold, dtype=np.float64)
tmpcvsum["trchisqca"] = np.zeros(nfold, dtype=np.float64)
tmpcvsum["vchisq"] = np.zeros(nfold, dtype=np.float64)
tmpcvsum["vrchisq"] = np.zeros(nfold, dtype=np.float64)
tmpcvsum["vchisqfcv"] = np.zeros(nfold, dtype=np.float64)
tmpcvsum["vchisqamp"] = np.zeros(nfold, dtype=np.float64)
tmpcvsum["vchisqcp"] = np.zeros(nfold, dtype=np.float64)
tmpcvsum["vchisqca"] = np.zeros(nfold, dtype=np.float64)
tmpcvsum["vrchisqfcv"] = np.zeros(nfold, dtype=np.float64)
tmpcvsum["vrchisqamp"] = np.zeros(nfold, dtype=np.float64)
tmpcvsum["vrchisqcp"] = np.zeros(nfold, dtype=np.float64)
tmpcvsum["vrchisqca"] = np.zeros(nfold, dtype=np.float64)
# initialize some columns
tmpcvsum.loc[:, "itsv"] = itsv
tmpcvsum.loc[:, "itv"] = itv
tmpcvsum.loc[:, "il1"] = il1
tmpcvsum.loc[:, "lambtsv"] = lambtsvs[itsv]
tmpcvsum.loc[:, "lambtv"] = lambtvs[itv]
tmpcvsum.loc[:, "lambl1"] = lambl1s[il1]
# Imaging parameters
cvimageprm = copy.deepcopy(imageprm)
# N-fold CV
for icv in np.arange(nfold):
# Header of output files
cvheader = header+".cv%02d" % (icv)
# Generate Data sets for imaging
if isvistable:
cvimageprm["vistable"] = vistables["t%d" % (icv)]
if isamptable:
cvimageprm["amptable"] = amptables["t%d" % (icv)]
if isbstable:
cvimageprm["bstable"] = bstables["t%d" % (icv)]
if iscatable:
cvimageprm["catable"] = catables["t%d" % (icv)]
# Image Training Data
filename = cvheader + ".t.fits"
filename = os.path.join(cvworkdir, filename)
if (skip is False) or (os.path.isfile(filename) is False):
cvnewimage = imagefunc(newimage, imageprm=cvimageprm,
**imagefargs)
cvnewimage.save_fits(filename)
else:
cvnewimage = imdata.IMFITS(filename)
# Make Plots
filename = cvheader + ".t.summary.pdf"
filename = os.path.join(cvworkdir, filename)
plots(cvnewimage, cvimageprm, filename=filename,
angunit=angunit, uvunit=uvunit)
# Check Training data
trainstats = statistics(cvnewimage, fulloutput=False,
**cvimageprm)
# Check validating data
# Switch to Validating data
if isvistable:
cvimageprm["vistable"] = vistables["v%d" % (icv)]
if isamptable:
cvimageprm["amptable"] = amptables["v%d" % (icv)]
if isbstable:
cvimageprm["bstable"] = bstables["v%d" % (icv)]
if iscatable:
cvimageprm["catable"] = catables["v%d" % (icv)]
# Make Plots
filename = cvheader + ".v.summary.pdf"
filename = os.path.join(cvworkdir, filename)
plots(cvnewimage, cvimageprm, filename=filename,
angunit=angunit, uvunit=uvunit)
# Check Statistics
validstats = statistics(cvnewimage, fulloutput=False,
**cvimageprm)
# Save Results
tmpcvsum.loc[icv, "tchisq"] = trainstats["chisq"]
tmpcvsum.loc[icv, "trchisq"] = trainstats["rchisq"]
tmpcvsum.loc[icv, "tchisqfcv"] = trainstats["chisqfcv"]
tmpcvsum.loc[icv, "tchisqamp"] = trainstats["chisqamp"]
tmpcvsum.loc[icv, "tchisqcp"] = trainstats["chisqcp"]
tmpcvsum.loc[icv, "tchisqca"] = trainstats["chisqca"]
tmpcvsum.loc[icv, "trchisqfcv"] = trainstats["rchisqfcv"]
tmpcvsum.loc[icv, "trchisqamp"] = trainstats["rchisqamp"]
tmpcvsum.loc[icv, "trchisqcp"] = trainstats["rchisqcp"]
tmpcvsum.loc[icv, "trchisqca"] = trainstats["rchisqca"]
tmpcvsum.loc[icv, "vchisq"] = validstats["chisq"]
tmpcvsum.loc[icv, "vrchisq"] = validstats["rchisq"]
tmpcvsum.loc[icv, "vchisqfcv"] = validstats["chisqfcv"]
tmpcvsum.loc[icv, "vchisqamp"] = validstats["chisqamp"]
tmpcvsum.loc[icv, "vchisqcp"] = validstats["chisqcp"]
tmpcvsum.loc[icv, "vchisqca"] = validstats["chisqca"]
tmpcvsum.loc[icv, "vrchisqfcv"] = validstats["rchisqfcv"]
tmpcvsum.loc[icv, "vrchisqamp"] = validstats["rchisqamp"]
tmpcvsum.loc[icv, "vrchisqcp"] = validstats["rchisqcp"]
tmpcvsum.loc[icv, "vrchisqca"] = validstats["rchisqca"]
# add current cv summary to the log file.
cvsumtable = pd.concat([cvsumtable,tmpcvsum], ignore_index=True)
cvsumtable.to_csv(os.path.join(workdir, cvsumtablefile))
# Average Varidation Errors and memorized them
tmpsum["tchisq"] = np.mean(tmpcvsum["tchisq"])
tmpsum["trchisq"] = np.mean(tmpcvsum["trchisq"])
tmpsum["tchisqfcv"] = np.mean(tmpcvsum["tchisqfcv"])
tmpsum["tchisqamp"] = np.mean(tmpcvsum["tchisqamp"])
tmpsum["tchisqcp"] = np.mean(tmpcvsum["tchisqcp"])
tmpsum["tchisqca"] = np.mean(tmpcvsum["tchisqca"])
tmpsum["trchisqfcv"] = np.mean(tmpcvsum["trchisqfcv"])
tmpsum["trchisqamp"] = np.mean(tmpcvsum["trchisqamp"])
tmpsum["trchisqcp"] = np.mean(tmpcvsum["trchisqcp"])
tmpsum["trchisqca"] = np.mean(tmpcvsum["trchisqca"])
tmpsum["vchisq"] = np.mean(tmpcvsum["vchisq"])
tmpsum["vrchisq"] = np.mean(tmpcvsum["vrchisq"])
tmpsum["vchisqfcv"] = np.mean(tmpcvsum["vchisqfcv"])
tmpsum["vchisqamp"] = np.mean(tmpcvsum["vchisqamp"])
tmpsum["vchisqcp"] = np.mean(tmpcvsum["vchisqcp"])
tmpsum["vchisqca"] = np.mean(tmpcvsum["vchisqca"])
tmpsum["vrchisqfcv"] = np.mean(tmpcvsum["vrchisqfcv"])
tmpsum["vrchisqamp"] = np.mean(tmpcvsum["vrchisqamp"])
tmpsum["vrchisqcp"] = np.mean(tmpcvsum["vrchisqcp"])
tmpsum["vrchisqca"] = np.mean(tmpcvsum["vrchisqca"])
# Output Summary Table
tmptable = pd.DataFrame([tmpsum.values()], columns=tmpsum.keys())
sumtable = pd.concat([sumtable, tmptable], ignore_index=True)
sumtable.to_csv(os.path.join(workdir, sumtablefile))
if docv:
return sumtable, cvsumtable
else:
return sumtable
def non_empty(self): ordered_dict = collections.OrderedDict() ordered_dict['A'] = 'A' ordered_dict[2] = 2 return ordered_dict
`ymNNNNN+ .mNNNNNh:
.mNNN+ .mNm` oNN+ .mNNN+
.mNNN+ `+o+ :oo- .mNNN+
.mNNNo```` .::::::: ````-mNNN+
oNNNNNNm`oh++++om`oNNNNNNm`
oNNNNNo. oNNNNNNm``:NNNNNm`
./NNm:. oNNNNNNm` `:yNNs-
`+hhhhhmNNNNNNNhhhhhy.`
:oooooo+
"""
# These have to go first, or they will get overridden by later keys.
REMYSPEAK = collections.OrderedDict({
"(ye )?new(e)?( )?biz": "new orders of business",
"(ye )?old(e)?( )?biz": "previously discussed business",
"cycle on": "spend time on",
"open loop": "unfinished task",
})
# Order-insensitive keys
REMYSPEAK.update({
"what's good": "how are you",
"kettle of fish": "matter",
"cycle": "period of time",
"cycles": "time available to spend on work",
"loop": "task",
"loops": "current tasks",
"m**********r": "fellow",
"biz": "five",
"hunny": "a hundred",
"hundo": "a hundred",
"step out": "smoke",
def ordered_alerts(self):
return collections.OrderedDict(sorted(self.alerts.items()))
# Bonus: the amount of non-unique words in the paragraph.
# Exercise 4
# Instructions
# Write a program that prints the frequency of the words from the input.
import collections
# Suppose the following input is supplied to the program:
text = "New to Python or choosing between Python 2 and Python 3? Read Python 2 or Python 3."
textlist = text.split()
print(textlist)
wordfreq = {}
for word in textlist:
wordfreq[word] = textlist.count(word)
# sorts the keys in order
ordered = collections.OrderedDict(sorted(wordfreq.items()))
for k, v in ordered.items():
print(f"{k}:{v}")
# Then, the output should be:
# 2:2
# 3.:1
# 3?:1
# New:1
# Python:5
# Read:1
# and:1
# between:1
# choosing:1
# or:2
def tree_reduce(function: Callable[[T, Any], T],
tree: Any,
initializer: Any = no_initializer) -> T:
if initializer is no_initializer:
return functools.reduce(function, tree_leaves(tree))
else:
return functools.reduce(function, tree_leaves(tree), initializer)
def tree_all(tree):
return all(tree_leaves(tree))
register_pytree_node(
collections.OrderedDict,
lambda x: (tuple(x.values()), tuple(x.keys())),
lambda keys, values: collections.OrderedDict(safe_zip(keys, values)))
register_pytree_node(
collections.defaultdict,
lambda x: (tuple(x.values()), (x.default_factory, tuple(x.keys()))),
lambda s, values: collections.defaultdict(s[0], safe_zip(s[1], values))) # type: ignore[index]
class Partial(functools.partial):
"""A version of functools.partial that works in pytrees.
Use it for partial function evaluation in a way that is compatible with JAX's
transformations, e.g., ``Partial(func, *args, **kwargs)``.
(You need to explicitly opt-in to this behavior because we didn't want to give
functools.partial different semantics than normal function closures.)
#!/usr/bin/python3
import sys
import collections
wordDict = {}
for line in sys.stdin:
word = line.split("\t")[0]
if word in wordDict:
wordDict[word] += 1
else:
wordDict[word] = 1
sortedDict = collections.OrderedDict(sorted(wordDict.items()))
for key in sortedDict:
print (key, sortedDict[key])
