def expand_uids_to_download(self, crispin_client, uids, metadata):
# During Gmail initial sync, we expand threads: given a UID to
# download, we want to also download other UIDs on the same thread, so
# that you don't see incomplete thread views for the duration of the
# sync. Given a 'seed set' of UIDs, this function returns a generator
# which yields the 'expanded' set of UIDs to download.
thrids = OrderedDict()
for uid in sorted(uids, reverse=True):
g_thrid = metadata[uid].g_thrid
if g_thrid in thrids:
thrids[g_thrid].append(uid)
else:
thrids[g_thrid] = [uid]
for g_thrid, uids in thrids.items():
g_msgid = metadata[uids[0]].g_msgid
# Because `uids` is ordered newest-to-oldest here, uids[0] is the
# last UID on the thread. If g_thrid is equal to its g_msgid, that
# means it's also the first UID on the thread. In that case, we can
# skip thread expansion for greater sync throughput.
if g_thrid != g_msgid:
uids = set(uids).union(crispin_client.expand_thread(g_thrid))
metadata.update(crispin_client.g_metadata(uids))
for uid in sorted(uids, reverse=True):
yield uid
def build_new_row(row, val_idx, h_friends, headers, val_h):
new_row = OrderedDict()
for h_idx, h in enumerate(headers):
h_hash = h['hash']
h_name = h['name']
if h_hash not in [f['hash'] for f in h_friends]:
if len(row[h_idx]) == 1:
# if this column is not correlated to the column we
# are working on and it's a single value, set
# it to the same value in the new row
new_row[h_name] = row[h_idx]
else:
# if this column is not correlated to the column we are
# working on and it is a multi value, set it
# to "UNRELATED"
new_row[h_name] = [
"UNRELATED TO {}".format(val_h['mod_name'])
]
else:
# if this column is correlated to the column we are
# working on, set the value to the indexed value of this
# value
new_row[h_name] = [row[h_idx][val_idx]]
new_row = new_row.values()
return new_row
def uniquify_points_and_return_input_index_to_unique_index_map( pts, threshold = 0 ): ''' Given a sequence of N points 'pts', and an optional 'threshold' indicating how many decimal places of accuracy (default: 0) returns two items: a sequence of all the unique elements in 'pts' and a list of length N where the i-th item in the list tells you where pts[i] can be found in the unique elements. ''' from collections import OrderedDict unique_pts = OrderedDict() pts_map = [] ## Add rounded points to a dictionary and set the key to ## ( the index into the ordered dictionary, the non-rounded point ) for i, ( pt, rounded_pt ) in enumerate( zip( pts, map( tuple, asarray( pts ).round( threshold ) ) ) ): index = unique_pts.setdefault( rounded_pt, ( len( unique_pts ), pt ) )[0] ## For fancier schemes: # index = unique_pts.setdefault( rounded_pt, ( len( unique_pts ), [] ) )[0] # unique_pts[ rounded_pt ][1].append( pt ) pts_map.append( index ) ## Return the original resolution points. ## The average of all points that round: # return [ tuple( average( pt, axis = 0 ) ) for i, pt in unique_pts.itervalues() ], pts_map ## The closest point to the rounded point: # return [ tuple( pt[ abs( asarray( pt ).round( threshold ) - asarray( pt ) ).sum(axis=1).argmin() ] ) for i, pt in unique_pts.itervalues() ], pts_map ## Simplest, the first rounded point: return [ tuple( pt ) for i, pt in unique_pts.itervalues() ], pts_map
def post(self, request, pk):
""" Clean the data and save opening hours in the database.
Old opening hours are purged before new ones are saved.
"""
location = self.get_object()
# open days, disabled widget data won't make it into request.POST
present_prefixes = [x.split('-')[0] for x in request.POST.keys()]
day_forms = OrderedDict()
for day_no, day_name in WEEKDAYS:
for slot_no in (1, 2):
prefix = self.form_prefix(day_no, slot_no)
# skip closed day as it would be invalid form due to no data
if prefix not in present_prefixes:
continue
day_forms[prefix] = (day_no, Slot(request.POST, prefix=prefix))
if all([day_form[1].is_valid() for pre, day_form in day_forms.items()]):
OpeningHours.objects.filter(company=location).delete()
for prefix, day_form in day_forms.items():
day, form = day_form
opens, shuts = [str_to_time(form.cleaned_data[x])
for x in ('opens', 'shuts')]
if opens != shuts:
OpeningHours(from_hour=opens, to_hour=shuts,
company=location, weekday=day).save()
return redirect(request.path_info)
def from_files(filenames):
"""Return an iterator that provides a sequence of Histograms for
the histograms defined in filenames.
"""
all_histograms = OrderedDict()
for filename in filenames:
parser = FILENAME_PARSERS[os.path.basename(filename)]
histograms = parser(filename)
# OrderedDicts are important, because then the iteration order over
# the parsed histograms is stable, which makes the insertion into
# all_histograms stable, which makes ordering in generated files
# stable, which makes builds more deterministic.
if not isinstance(histograms, OrderedDict):
raise BaseException, "histogram parser didn't provide an OrderedDict"
for (name, definition) in histograms.iteritems():
if all_histograms.has_key(name):
raise DefinitionException, "duplicate histogram name %s" % name
all_histograms[name] = definition
# We require that all USE_COUNTER2_* histograms be defined in a contiguous
# block.
use_counter_indices = filter(lambda x: x[1].startswith("USE_COUNTER2_"),
enumerate(all_histograms.iterkeys()));
if use_counter_indices:
lower_bound = use_counter_indices[0][0]
upper_bound = use_counter_indices[-1][0]
n_counters = upper_bound - lower_bound + 1
if n_counters != len(use_counter_indices):
raise DefinitionException, "use counter histograms must be defined in a contiguous block"
for (name, definition) in all_histograms.iteritems():
yield Histogram(name, definition)
def __init__(self, logger=None):
self.log = logger or dulog
self.section = None
self.filename = None
self.compiler_info = OrderedDict((
('mpicc' , None),
('mpicxx' , None),
('mpifort', None),
('mpif77' , None),
('mpif90' , None),
('mpif08' , None),
('mpild' , None),
))
self.library_info = OrderedDict((
('define_macros' , []),
('undef_macros' , []),
('include_dirs' , []),
('libraries' , []),
('library_dirs' , []),
('runtime_library_dirs' , []),
('extra_compile_args' , []),
('extra_link_args' , []),
('extra_objects' , []),
))
def test_rename(self):
data = create_test_data()
newnames = {'var1': 'renamed_var1', 'dim2': 'renamed_dim2'}
renamed = data.rename(newnames)
variables = OrderedDict(data.variables)
for k, v in newnames.iteritems():
variables[v] = variables.pop(k)
for k, v in variables.iteritems():
dims = list(v.dimensions)
for name, newname in newnames.iteritems():
if name in dims:
dims[dims.index(name)] = newname
self.assertVariableEqual(Variable(dims, v.values, v.attrs),
renamed.variables[k])
self.assertEqual(v.encoding, renamed.variables[k].encoding)
self.assertEqual(type(v), type(renamed.variables[k]))
self.assertTrue('var1' not in renamed.variables)
self.assertTrue('dim2' not in renamed.variables)
with self.assertRaisesRegexp(ValueError, "cannot rename 'not_a_var'"):
data.rename({'not_a_var': 'nada'})
# verify that we can rename a variable without accessing the data
var1 = data['var1']
data['var1'] = (var1.dimensions, InaccessibleArray(var1.values))
renamed = data.rename(newnames)
with self.assertRaises(UnexpectedDataAccess):
renamed['renamed_var1'].values
def force_trace_widths(board):
microstrip_layers = frozenset(('1_top', '6_bot'))
stripline_layers = frozenset(('3_inner', '4_inner'))
se_50_microstrip_width = '0.1778'
se_50_stripline_width = '0.1651'
diff_90_microstrip_width = '0.127'
diff_90_stripline_width = '0.127'
for element in board:
if element[0] == 'segment':
segment = OrderedDict([(v[0], v[1:]) for v in element[1:]])
assert len(segment['net']) == 1
net_name = net_by_number[int(segment['net'][0])]
assert len(segment['layer']) == 1
layer = segment['layer'][0]
new_width = None
if net_name in nets_by_net_class['50_se']:
if layer in microstrip_layers:
new_width = [se_50_microstrip_width]
if layer in stripline_layers:
new_width = [se_50_stripline_width]
elif net_name in nets_by_net_class['90_diff']:
if layer in microstrip_layers:
new_width = [diff_90_microstrip_width]
if layer in stripline_layers:
new_width = [diff_90_stripline_width]
if new_width:
segment['width'] = new_width
new_elements = [[a] + b for a, b in segment.items()]
element[1:] = new_elements
def getDependencies(self,
available_components = None,
search_dirs = None,
target = None,
available_only = False,
test = False,
warnings = True
):
''' Returns {component_name:component}
'''
if search_dirs is None:
search_dirs = [self.modulesPath()]
available_components = self.ensureOrderedDict(available_components)
components, errors = self.__getDependenciesWithProvider(
available_components = available_components,
search_dirs = search_dirs,
target = target,
update_installed = False,
provider = self.provideInstalled,
test = test
)
if warnings:
for error in errors:
logger.warning(error)
if available_only:
components = OrderedDict((k, v) for k, v in components.items() if v)
return components
def __init__(self, raw):
self._raw = raw
self.file_version = 0
self.elements_count = 0
self._elements_map = OrderedDict()
self._elements_index = OrderedDict()
self._jpg_map = OrderedDict()
def assert_no_duplicate_frameworks(state):
"""A function which asserts that there are no duplicate frameworks running, where
frameworks are identified by their name.
Note the extra spaces in the output strings: this is to account for the extra indentation
we add, so we can have:
frameworks:
framework: marathon count: 1
:param state: the state info from the Mesos master
:returns: a tuple containing (output, ok): output is a log of the state of frameworks, ok a boolean
indicating if there are any duplicate frameworks.
"""
frameworks = state['frameworks']
framework_counts = OrderedDict(sorted(Counter([fw['name'] for fw in frameworks]).items()))
output = ["frameworks:"]
ok = True
for framework, count in framework_counts.iteritems():
if count > 1:
ok = False
output.append(PaastaColors.red(
" CRITICAL: Framework %s has %d instances running--expected no more than 1."
% (framework, count)))
else:
output.append(" framework: %s count: %d" % (framework, count))
return (("\n").join(output), ok)
def getDbSeqRecord(db_record, id_field, seq_field, meta_fields=None,
delimiter=default_delimiter):
"""
Parses a database record into a SeqRecord
Arguments:
db_record = a dictionary containing a database record
id_field = the field containing identifiers
seq_field = the field containing sequences
meta_fields = a list of fields to add to sequence annotations
delimiter = a tuple of delimiters for (fields, values, value lists)
Returns:
a SeqRecord
"""
# Return None if ID or sequence fields are empty
if not db_record[id_field] or not db_record[seq_field]:
return None
# Create description string
desc_dict = OrderedDict([('ID', db_record[id_field])])
if meta_fields is not None:
desc_dict.update([(f, db_record[f]) for f in meta_fields if f in db_record])
desc_str = flattenAnnotation(desc_dict, delimiter=delimiter)
# Create SeqRecord
seq_record = SeqRecord(Seq(db_record[seq_field], IUPAC.ambiguous_dna),
id=desc_str, name=desc_str, description='')
return seq_record
def find_unique_entries(node):
"""
Find all uniquely named entries, without recursing through inner namespaces.
Args:
node: a Section or InnerNamespace instance
Yields:
A sequence of MergedEntry nodes representing an entry
Remarks:
This collapses multiple entries with the same fully qualified name into
one entry (e.g. if there are multiple entries in different kinds).
"""
if not isinstance(node, metadata_model.Section) and \
not isinstance(node, metadata_model.InnerNamespace):
raise TypeError("expected node to be a Section or InnerNamespace")
d = OrderedDict()
# remove the 'kinds' from the path between sec and the closest entries
# then search the immediate children of the search path
search_path = isinstance(node, metadata_model.Section) and node.kinds \
or [node]
for i in search_path:
for entry in i.entries:
d[entry.name] = entry
for k,v in d.iteritems():
yield v.merge()
def test():
from collections import OrderedDict as StdlibOrderedDict
ordered_dict = OrderedDict(((1, 'a'), (2, 'b'), (3, 'c')))
stdlib_ordered_dict = StdlibOrderedDict(((1, 'a'), (2, 'b'), (3, 'c')))
assert ordered_dict == stdlib_ordered_dict
assert stdlib_ordered_dict == ordered_dict
assert ordered_dict.items() == stdlib_ordered_dict.items()
assert ordered_dict.keys() == stdlib_ordered_dict.keys()
assert ordered_dict.values() == stdlib_ordered_dict.values()
ordered_dict.move_to_end(1)
assert ordered_dict != stdlib_ordered_dict
#assert stdlib_ordered_dict != ordered_dict
assert ordered_dict.items() != stdlib_ordered_dict.items()
assert ordered_dict.keys() != stdlib_ordered_dict.keys()
assert ordered_dict.values() != stdlib_ordered_dict.values()
del stdlib_ordered_dict[1]
stdlib_ordered_dict[1] = 'a'
assert ordered_dict == stdlib_ordered_dict
assert stdlib_ordered_dict == ordered_dict
assert ordered_dict.items() == stdlib_ordered_dict.items()
assert ordered_dict.keys() == stdlib_ordered_dict.keys()
assert ordered_dict.values() == stdlib_ordered_dict.values()
assert ordered_dict == OrderedDict(stdlib_ordered_dict) == \
stdlib_ordered_dict
assert ordered_dict == StdlibOrderedDict(ordered_dict) == \
stdlib_ordered_dict
class akmers :
def __init__(self) :
self.mers = OrderedDict()
self.smers_set = set()
#*******************************************
def get_mers(self) :
return set(self.mers.keys())
#*******************************************
def update_smer_set(self) :
self.smers_set = set(self.mers.keys())
#*******************************************
def add_mer(self, mer, count) :
self.mers[mer] = count
#*******************************************
def remove_mer(self, mer) :
del self.mers[mer]
#*******************************************
def has_mers(self) :
if len(self.mers) > 0 and max(self.mers.values()) > 1 :
return True
else :
return False
#*******************************************
def get_count(self, mer) :
return self.mers[mer]
def getDependenciesRecursive(self,
available_components = None,
processed = None,
search_dirs = None,
target = None,
available_only = False,
test = False
):
''' Get available and already installed components, don't check for
remotely available components. See also
satisfyDependenciesRecursive()
Returns {component_name:component}
'''
components, errors = self.__getDependenciesRecursiveWithProvider(
available_components = available_components,
search_dirs = search_dirs,
target = target,
traverse_links = True,
update_installed = False,
provider = self.provideInstalled,
test = test
)
for error in errors:
logger.error(error)
if available_only:
components = OrderedDict((k, v) for k, v in components.items() if v)
return components
def setUp(self):
couch_uri = 'http://ovh23.depp.it:5984'
self.couch_server = couchdb.Server(couch_uri)
self.norm_couch = self.couch_server['bilanci_voci']
self.simple_couch = self.couch_server['bilanci_simple']
self.entrate_sections = OrderedDict([
('Accertamenti', 0),
('Riscossioni in conto competenza', 1),
('Riscossioni in conto residui', 2),
])
self.spese_sections = OrderedDict([
('Impegni', 0),
('Pagamenti in conto competenza', 1),
('Pagamenti in conto residui', 2),
])
(self.year, self.city) = self.code.split("_")
self.normalized_doc_id = "{}_{}".format(self.year, self.city)
self.simplified_doc_id = self.city
self.norm_doc = self.norm_couch[self.normalized_doc_id]
self.simp_doc = self.simple_couch[self.simplified_doc_id]
self.logger = logging.getLogger('test')
self.logger.setLevel(logging.INFO)
def list_releases(moniker=None, limit=0):
"""
List currently available releases.
Arguments:
moniker -- kernel release moniker (e.g., mainline, stable, longterm, etc.).
Defaults to everything
limit -- maximum number of releases to list per moniker. Defaults to 0,
which is no limit
"""
releases = get_releases()
if moniker is None:
releases_by_moniker = OrderedDict()
for release in releases['releases']:
releases_by_moniker.setdefault(release['moniker'], []).append(release)
first = True
for moniker, r in releases_by_moniker.items():
if not first:
print()
first = False
print('%s:' % moniker)
for release in r[:limit] if limit > 0 else r:
print(release['version'])
else:
r = get_releases_by_moniker(releases, moniker)
for release in r[:limit] if limit > 0 else r:
print(release['version'])
def get_language_config(content_language=None):
language = get_language()[:2]
if content_language:
content_language = content_language[:2]
else:
content_language = language
config = {}
config['language'] = language
lang_names = OrderedDict()
for lang, name in settings.LANGUAGES:
if lang[:2] not in lang_names: lang_names[lang[:2]] = []
lang_names[lang[:2]].append(_(name))
sp_langs = []
for lang, names in lang_names.items():
if lang == content_language:
default = '+'
else:
default = ''
sp_langs.append(u'%s%s=%s' % (default, ' / '.join(names), lang))
config['spellchecker_languages'] = ','.join(sp_langs)
if content_language in settings.LANGUAGES_BIDI:
config['directionality'] = 'rtl'
else:
config['directionality'] = 'ltr'
if tinymce.settings.USE_SPELLCHECKER:
config['spellchecker_rpc_url'] = reverse('tinymce.views.spell_check')
return config
class _SlideCache(object):
def __init__(self, cache_size, dz_opts):
self.cache_size = cache_size
self.dz_opts = dz_opts
self._lock = Lock()
self._cache = OrderedDict()
def get(self, path):
with self._lock:
if path in self._cache:
# Move to end of LRU
slide = self._cache.pop(path)
self._cache[path] = slide
return slide
osr = OpenSlide(path)
slide = DeepZoomGenerator(osr, **self.dz_opts)
try:
mpp_x = osr.properties[openslide.PROPERTY_NAME_MPP_X]
mpp_y = osr.properties[openslide.PROPERTY_NAME_MPP_Y]
slide.mpp = (float(mpp_x) + float(mpp_y)) / 2
except (KeyError, ValueError):
slide.mpp = 0
with self._lock:
if path not in self._cache:
if len(self._cache) == self.cache_size:
self._cache.popitem(last=False)
self._cache[path] = slide
return slide
class LRUCache(object):
def __init__(self, capacity):
"""
:type capacity: int
"""
self.dictionary = OrderedDict()
self.capacity = capacity
def get(self, key):
"""
:rtype: int
"""
if key in self.dictionary:
temp = self.dictionary.pop(key)
self.dictionary[key] = temp
return temp
else: return -1
def set(self, key, value):
"""
:type key: int
:type value: int
:rtype: nothing
"""
if key in self.dictionary:
self.dictionary.pop(key)()
self.dictionary[key] = value
else:
if len(self.dictionary) >= self.capacity:
self.dictionary.popitem(last=False)
self.dictionary[key] = value
else:
self.dictionary[key] = value
def get_fielddata(model_object, search_tables, field_information_filter=None, extra_properties=[]):
"""
returns an ordered dict of field_name->{value:value,fieldinformation:fi}
to be used to display the item in the UI Detail views
extra_properties are non-standard getters that wouldn't normally be returned (restricted fields)
"""
# dump(self.dataset)
# data=model_to_dict(self.dataset)
property_dict = get_properties(model_object)
if len(extra_properties) > 0:
for prop in extra_properties:
property_dict[prop] = getattr(model_object, prop)
logger.info(str(("got extra prop", prop, getattr(model_object, prop))))
logger.debug(str(("property_dict", property_dict)))
ui_dict = {}
for field, value in property_dict.iteritems():
logger.debug(str(("get_field_info", field)))
details = {}
try:
fi = FieldInformation.manager.get_column_fieldinformation_by_priority(field, search_tables)
if fi and (field_information_filter and field_information_filter(fi) or field_information_filter == None):
details["fieldinformation"] = fi
details["value"] = value
ui_dict[field] = details
# ui_dict[fi.get_verbose_name()] = value
else:
logger.debug(str(("field not shown in this view: ", field, value)))
except (ObjectDoesNotExist, MultipleObjectsReturned, Exception) as e:
logger.debug(str(("no field information defined for: ", field, value)))
ui_dict = OrderedDict(sorted(ui_dict.items(), key=lambda x: x[1]["fieldinformation"].order))
if logger.isEnabledFor(logging.DEBUG):
logger.debug(str(("ui_dict", ui_dict)))
return ui_dict
def git_list(args, script_path):
ariane_git_repos = OrderedDict(sorted(json.load(
open(script_path+"/resources/sources/ariane.community.git.repos-SNAPSHOT.json")).items(),
key=lambda t: t[0]))
if args.addon:
print("\nExisting Ariane addon git repositories :\n")
print('{:40} {:110}'.format("Ariane git repository name", "Ariane git repository URL"))
print('{:40} {:110}'.format("--------------------------", "-------------------------"))
for key in ariane_git_repos.keys():
git_repo = ariane_git_repos[key]
if git_repo['type'] == "addon":
print('{:40} {:110}'.format(key, git_repo['url']))
elif args.core:
print("\nExisting Ariane core git repositories :\n")
print('{:40} {:110}'.format("Ariane git repository name", "Ariane git repository URL"))
print('{:40} {:110}'.format("--------------------------", "-------------------------"))
for key in ariane_git_repos.keys():
git_repo = ariane_git_repos[key]
if git_repo['type'] == "core":
print('{:40} {:110}'.format(key, git_repo['url']))
else:
print("\nExisting Ariane git repositories :\n")
print('{:40} {:110} {:25}'.format("Ariane git repository name", "Ariane git repository URL",
"Ariane git repository type"))
print('{:40} {:110} {:25}'.format("--------------------------", "-------------------------",
"--------------------------"))
for key in ariane_git_repos.keys():
git_repo = ariane_git_repos[key]
print('{:40} {:110} {:25}'.format(key, git_repo['url'], git_repo['type']))
def test_mqtt_subscribes_topics_on_connect(self):
"""Test subscription to topic on connect."""
from collections import OrderedDict
prev_topics = OrderedDict()
prev_topics['topic/test'] = 1,
prev_topics['home/sensor'] = 2,
prev_topics['still/pending'] = None
mqtt.MQTT_CLIENT.topics = prev_topics
mqtt.MQTT_CLIENT.progress = {1: 'still/pending'}
# Return values for subscribe calls (rc, mid)
mqtt.MQTT_CLIENT._mqttc.subscribe.side_effect = ((0, 2), (0, 3))
mqtt.MQTT_CLIENT._mqtt_on_connect(None, None, 0, 0)
self.assertFalse(mqtt.MQTT_CLIENT._mqttc.disconnect.called)
expected = [(topic, qos) for topic, qos in prev_topics.items()
if qos is not None]
self.assertEqual(
expected,
[call[1] for call in mqtt.MQTT_CLIENT._mqttc.subscribe.mock_calls])
self.assertEqual({
1: 'still/pending',
2: 'topic/test',
3: 'home/sensor',
}, mqtt.MQTT_CLIENT.progress)
def get_multipartite_form(multi_type, user):
'It returns a form for the given multipartite'
form_fields = OrderedDict()
part_defs = PARTS_TO_ASSEMBLE[multi_type]
for parts in part_defs:
features = Feature.objects.filter(type__name=parts[0],
prefix=parts[1],
suffix=parts[2])
features = filter_feature_by_user_perms(features, user)
choices = features_to_choices(features)
name = parts[0]
form_fields[name] = forms.CharField(max_length=100,
widget=Select(choices=choices))
# last we need to add the vector to the form
vector_choices = get_vector_choices(user)
form_fields[VECTOR_TYPE_NAME] = forms.CharField(max_length=100,
widget=Select(choices=vector_choices))
form = type('MultiPartiteForm', (forms.BaseForm,),
{'base_fields': form_fields})
for field_name in form_fields.keys():
setattr(form, 'clean_{0}'.format(field_name),
create_feature_validator(field_name))
return form
def insert(self,key,value,index=None,akey=None,bkey=None):
tmp1=OrderedDict()
tmp2=OrderedDict()
if ((index is not None) and (isinstance(index, int))):
if index<len(self.keys()):
for i in self.iterkeys():
if self.indexofkey(i)<index:
tmp1[i]=self[i]
elif self.indexofkey(i)>=index:
tmp2[i]=self[i]
self.clear()
for i in tmp1.items():
self[i[0]]=i[1]
self[key]=value
for i in tmp2.items():
self[i[0]]=i[1]
return self
if index==len(self.keys()):
self[key]=value
if akey is not None:
if akey in self.iterkeys():
self.insert(key,value,index=self.indexofkey(akey)+1)
else:
raise KeyError
if bkey is not None:
if bkey in self.iterkeys():
self.insert(key, value, index=self.indexofkey(bkey))
else:
raise KeyError
def get_error_messages(self, errors, question_descriptor_from="label"):
question_errors = super(QuestionSummary, self).get_error_messages(
errors,
question_descriptor_from=question_descriptor_from,
)
boolean_list_questions = self.get('boolean_list_questions')
boolean_list_values = self.get('value') or []
if self.id in question_errors and self.type == 'boolean_list' and boolean_list_questions:
# pad list of values to same length as boolean_list_questions
boolean_list_values.extend([None] * (len(boolean_list_questions) - len(boolean_list_values)))
for index, boolean_list_question in enumerate(boolean_list_questions):
if not isinstance(boolean_list_values[index], bool):
# Each non-boolean value is an error
boolean_question_id = "{}-{}".format(self.id, index)
question_errors[boolean_question_id] = {
'input_name': boolean_question_id,
'message': question_errors[self.id]['message'],
'question': boolean_list_question
}
question_errors[self.id] = True
question_errors = OrderedDict([
(k, question_errors[k]) for k in sorted(question_errors.keys())
])
return question_errors
def parse_records(self, record):
"""
Parses the record set
"""
time_series = OrderedDict([
("X", {"Original": {}, "SDOF": {}}),
("Y", {"Original": {}, "SDOF": {}}),
("V", {"Original": {}, "SDOF": {}})])
target_names = time_series.keys()
for iloc, ifile in enumerate(self.input_files):
if not os.path.exists(ifile):
if iloc < 2:
# Expected horizontal component is missing - raise error
raise ValueError("Horizontal record %s is expected but "
"not found!" % ifile)
else:
print "Vertical record file %s not found" % ifile
del time_series["V"]
continue
else:
time_series[target_names[iloc]]["Original"] = \
self._parse_time_history(ifile, record.xrecord.units)
if iloc < 2:
del time_series["V"]
return time_series
def combine(**kwargs):
"""Generate combinations based on its keyword arguments.
Two sets of returned combinations can be concatenated using +. Their product
can be computed using `times()`.
Args:
**kwargs: keyword arguments of form `option=[possibilities, ...]`
or `option=the_only_possibility`.
Returns:
a list of dictionaries for each combination. Keys in the dictionaries are
the keyword argument names. Each key has one value - one of the
corresponding keyword argument values.
"""
if not kwargs:
return [OrderedDict()]
sort_by_key = lambda k: k[0][0]
kwargs = OrderedDict(sorted(kwargs.items(), key=sort_by_key))
first = list(kwargs.items())[0]
rest = dict(list(kwargs.items())[1:])
rest_combined = combine(**rest)
key = first[0]
values = first[1]
if not isinstance(values, list):
values = [values]
return [
OrderedDict(sorted(list(combined.items()) + [(key, v)], key=sort_by_key))
for v in values
for combined in rest_combined
]
def __init__(self, data, label=None, batch_size=1, shuffle=False,
last_batch_handle='pad', data_name='data',
label_name='softmax_label'):
super(NDArrayIter, self).__init__(batch_size)
self.data = _init_data(data, allow_empty=False, default_name=data_name)
self.label = _init_data(label, allow_empty=True, default_name=label_name)
# shuffle data
if shuffle:
idx = np.arange(self.data[0][1].shape[0])
np.random.shuffle(idx)
self.data = [(k, array(v.asnumpy()[idx], v.context)) for k, v in self.data]
self.label = [(k, array(v.asnumpy()[idx], v.context)) for k, v in self.label]
# batching
if last_batch_handle == 'discard':
new_n = self.data[0][1].shape[0] - self.data[0][1].shape[0] % batch_size
data_dict = OrderedDict(self.data)
label_dict = OrderedDict(self.label)
for k, _ in self.data:
data_dict[k] = data_dict[k][:new_n]
for k, _ in self.label:
label_dict[k] = label_dict[k][:new_n]
self.data = data_dict.items()
self.label = label_dict.items()
self.data_list = [x[1] for x in self.data] + [x[1] for x in self.label]
self.num_source = len(self.data_list)
self.num_data = self.data_list[0].shape[0]
assert self.num_data >= batch_size, \
"batch_size need to be smaller than data size."
self.cursor = -batch_size
self.batch_size = batch_size
self.last_batch_handle = last_batch_handle
