def __getitem__(self, key):
if OrderedDict.__contains__(self, key):
return OrderedDict.__getitem__(self, key)
elif ((not OrderedDict.__contains__(self, key)) and
(self.parent is not None)):
dct = find_next_dict(self)
if dct is None:
raise KeyError("%r not in dictionary" % key)
else:
return dct[key]
else:
raise KeyError("%r not in dictionary" % key)
def __getitem__(self, key):
if OrderedDict.__contains__(self, key):
return OrderedDict.__getitem__(self, key)
elif ((not OrderedDict.__contains__(self, key))
and (self.parent is not None)):
dct = find_next_dict(self)
if dct is None:
raise KeyError("%r not in dictionary" % key)
else:
return dct[key]
else:
raise KeyError("%r not in dictionary" % key)
def __contains__(self, key):
if (
self._allow_contains_default
and key in getattr(self, "defaults_", {})
):
return True
return OrderedDict.__contains__(self, key)
def get_topk_query_app_lick_log(app_click_log_file, wanted_keys,
name_dict: OrderedDict, topk):
"""
get topk's doc from every query's app lick log
:param wanted_keys: the queries to get DOC
:param name_dict: the dictionary of APPID and DOC
:param topk: limit to get the size of topk's doc
:return: a ordered dictionary of ordered dictionary
"""
click_log_dict = {}
with codecs.open(app_click_log_file, 'r') as rf:
for line in rf.readlines():
items = line.split('|')
current_key = items[0]
app_id = items[1]
app_download_count = items[-1]
if click_log_dict.__contains__(current_key):
click_log_dict[current_key][app_id] = (name_dict[app_id],
app_download_count)
else:
click_log_dict[current_key] = OrderedDict()
if name_dict.__contains__(app_id):
click_log_dict[current_key][app_id] = (name_dict[app_id],
app_download_count)
for key, doc_dict in click_log_dict.items():
click_log_dict[key] = OrderedDict(
list(sorted(doc_dict.items(), key=lambda x: -x[1][1])[:topk]))
return click_log_dict
class Metadata(MutableMapping):
def __init__(self, seq=None):
self.dct = OrderedDict(seq) if seq else OrderedDict()
def __contains__(self, key):
return self.dct.__contains__(key)
def __getitem__(self, key):
return self.dct.__getitem__(key)
def __setitem__(self, key, value):
self.dct.__setitem__(key, value)
def __delitem__(self, key):
return self.dct.__delitem__(key)
def __iter__(self):
return self.dct.__iter__()
def __len__(self):
return self.dct.__len__()
def __repr__(self):
return repr(self.dct)
def __str__(self):
return str(self.dct)
def find_path(self, graph: Graph, start: Node, end: Node):
NodeRecord = namedtuple('NodeRecord', ['node', 'connection', 'cost_so_far', 'estimated_total_cost'])
current: NodeRecord
connections: List[GraphConnection]
start_record = NodeRecord()
start_record.node = start
start_record.connection = None
start_record.cost_so_far = 0
start_record.estimated_total_cost = self.estimate(start, end)
open = OrderedDict(int, NodeRecord)
open.add(start_record.estimated_total_cost, start_record)
closed = OrderedDict(int, NodeRecord)
while len(open) > 0:
current = open.pop()
if current.node == end:
break
connections = current.node.get_connections()
for connection in connections:
end_node = connection.to_node
end_node_cost = current.cost_so_far + connection.get_cost()
if closed.__contains__(end_node):
end_node_record = closed.
if end_node_record.cost_so_far <= end_node_cost
continue
class UssdAppRouter(object):
__slots__ = ('routes', 'base_code')
def __init__(self, code=None):
self.routes = OrderedDict()
self.base_code = code and UssdCode(code)
def route(self, code, handler):
code = code if isinstance(code, UssdCode) else UssdCode(code)
key_code = tuple(code)
if key_code in self.routes:
raise ValueError(
'Ussd code %s already registered in ussd router.' % (code, ))
self.routes[key_code] = (code, handler)
def resolve(self, request):
ussd_string = request.ussd_string
if ussd_string.startswith(self.base_code):
ussd_string = ussd_string[len(self.base_code):]
codes = sorted(filter(ussd_string.startswith, self.routes.keys()))
if codes:
code = codes[-1]
route_code = '%s*%s' % (self.base_code,
code) if self.base_code else code
request.route_code = route_code
return self.routes[code], route_code
return None, None
def __len__(self):
return self.routes.__len__()
def __contains__(self, code):
return self.routes.__contains__(code)
class InfoRow():
def __init__(self):
self.dict=OrderedDict()
def __getitem__(self, item):
if isinstance(item, int):
return self.dict.items()[item][1]
return self.dict.__getitem__(item.lower())
def __contains__(self, item):
return self.dict.__contains__(item.lower())
def __setitem__(self, key, value):
if isinstance(key, int):
self.dict.items()[key][1]=value
self.dict.__setitem__(key.lower(),value)
def get(self, k, d=None):
if isinstance(k, int):
try:
return self.dict.items()[k][1]
except:
return d
return self.dict.get(k.lower(), d)
def __str__(self, rowSeparator="\n", columnSeparator="\t"):
return getStr(dict2Str(self.dict))
def __unicode__(self):
return str(self)
def __repr__(self):
return str(self)
class KeyCollection:
def __init__(self):
self._dictionary = OrderedDict()
def __getitem__(self, key):
if isinstance(key, int):
return list(self._dictionary.values())[key]
return self._dictionary[key]
def _add(self, value):
self._dictionary[value.id] = value
def __contains__(self, key):
return self._dictionary.__contains__(key)
def __len__(self):
return self._dictionary.values().__len__()
def __iter__(self):
return self._dictionary.values().__iter__()
def __next__(self):
return self._dictionary.values().__next__()
def _ipython_key_completions_(self):
return list(self._dictionary.keys())
def consume_dots(config, key, create_default):
sub_keys = key.split('.', 1)
sub_key = sub_keys[0]
if sub_key in Config.__dict__:
raise KeyError('"{}" is a preserved API name for runner.Config, '
'should not be used as normal dictionary key'.format(sub_key))
if not OrderedDict.__contains__(config, sub_key) and len(sub_keys) == 2:
if create_default:
config[sub_key] = Config()
# OrderedDict.__setitem__(config, sub_key, Config())
else:
raise KeyError(key)
if len(sub_keys) == 1:
return config, sub_key
else:
sub_config = OrderedDict.__getitem__(config, sub_key)
if type(sub_config) != Config:
if create_default:
sub_config = Config()
config[sub_key] = sub_config
# OrderedDict.__setitem__(config, sub_key, sub_config)
else:
raise KeyError(key)
return consume_dots(sub_config, sub_keys[1], create_default)
class GetCategoryList(object):
def __init__(self):
self.category_file = config.category_file
self.project = config.project
self.sheet_name = config.project.lower()
self.function_list = OrderedDict()
def get_category_list(self):
workbook = xlrd.open_workbook(self.category_file)
# all_sheet_names = workbook.sheet_names()
sheet = workbook.sheet_by_name(self.sheet_name)
row_title_data = sheet.row_values(0)
for index, row_value in enumerate(row_title_data):
results = []
language, net_stauts = row_value.split('_')[0].title(), row_value.split('_')[1]
# column_index = row_title_data.index(self.column_name)
results = [value.replace(" ", "") for value in filter(None, sheet.col_values(index, 1))]
if language.lower() == "hkmc":
language = "HK_Cantonese"
if self.function_list.__contains__(language):
self.function_list.get(language).update({net_stauts: results})
else:
self.function_list.setdefault(language, {}).update({net_stauts: results})
return self.function_list
def iterkeys(self):
"""Include default keys"""
for key in OrderedDict.keys(self):
yield key
for key in getattr(self, 'defaults_', []):
if not OrderedDict.__contains__(self, key):
yield key
class Metadata(MutableMapping):
def __init__(self, seq=None):
self.dct = OrderedDict(seq) if seq else OrderedDict()
def __contains__(self, key):
return self.dct.__contains__(key)
def __getitem__(self, key):
return self.dct.__getitem__(key)
def __setitem__(self, key, value):
self.dct.__setitem__(key, value)
def __delitem__(self, key):
return self.dct.__delitem__(key)
def __iter__(self):
return self.dct.__iter__()
def __len__(self):
return self.dct.__len__()
def __repr__(self):
return repr(self.dct)
def __str__(self):
return str(self.dct)
def get_category_lists(analyze_lang):
category_sort = OrderedDict()
if analyze_lang.lower() == "all":
data_category = dbc.sql_select("select net_status, category, language \
from %s \
where scene=1 \
order by language" % BENCHMARK_NAME)
languages = []
[
languages.append(info[2]) for info in data_category
if info[2] not in languages
]
else:
data_category = dbc.sql_select(
"select net_status, category \
from %s \
where scene=1 \
and language='%s' \
order by net_status" %
(BENCHMARK_NAME, analyze_lang.capitalize()))
languages = [analyze_lang.capitalize()]
for lang in languages:
for data in data_category:
if data[2] == lang:
if category_sort.__contains__(lang):
category_sort.get(lang).get(data[0]).append(data[1]) if category_sort.get(lang).__contains__(data[0]) \
else category_sort.get(lang).setdefault(data[0], []).append(data[1])
else:
category_sort.setdefault(lang,
{}).update({data[0]: [data[1]]})
#print(category_sort)
return category_sort
class ClassRegistry(object):
def __init__(self):
self._items = []
self._dict = OrderedDict()
def register(self, cls):
if cls in self._items:
warnings.warn("Class registered multiple times: %s" %
cls.__name__,
stacklevel=2)
return cls
self._items.append(cls)
if hasattr(cls, 'identity'):
self._dict[cls.identity] = cls
return cls
def __iter__(self):
for i in self._items:
yield i
def __getitem__(self, identity):
return self._dict[identity]
def __contains__(self, identity):
return self._dict.__contains__(identity)
def get(self, identity, default=None):
return self._dict.get(identity, default)
def get_hotword_list():
hotword_sort = OrderedDict()
if config.analyze_lang.lower() == "all":
data_hotword = dbc.sql_select(
'select language, hotword '
'from %s '
'where scene="总计" '
'and sdk_version = "%s" '
'order by language' %
(HOTWORD_RESULT_NAME, config.new_sdk_version))
languages = []
[
languages.append(info[0]) for info in data_hotword
if info[0] not in languages
]
else:
data_hotword = dbc.sql_select(
"select hotword \
from %s \
where scene='总计' \
and language='%s' \
order by hotword" %
(HOTWORD_RESULT_NAME, config.analyze_lang.capitalize()))
languages = [config.analyze_lang.capitalize()]
for lang in languages:
for data in data_hotword:
if data[0] == lang:
hotword_sort.get(lang).append(data[1]) if hotword_sort.__contains__(lang) \
else hotword_sort.setdefault(lang, []).append(data[1])
return hotword_sort
def get_keyword_list():
keyword_sort = OrderedDict()
if config.analyze_lang.lower() == "all":
data_keyword = dbc.sql_select("select language, keyword \
from %s \
where scene=1 \
order by language" %
KEYWORD_BENCHMARK_NAME)
languages = []
[
languages.append(info[0]) for info in data_keyword
if info[0] not in languages
]
else:
data_keyword = dbc.sql_select(
"select keyword \
from %s \
where scene=1 \
and language='%s' \
order by keyword" %
(BENCHMARK_NAME, config.analyze_lang.capitalize()))
languages = [config.analyze_lang.capitalize()]
for lang in languages:
for data in data_keyword:
if data[0] == lang:
keyword_sort.get(lang).append(data[1]) if keyword_sort.__contains__(lang) \
else keyword_sort.setdefault(lang, []).append(data[1])
return keyword_sort
def iterkeys(self):
"""Include default keys"""
for key in OrderedDict.iterkeys(self):
yield key
for key in getattr(self, 'defaults_', []):
if not OrderedDict.__contains__(self, key):
yield key
def main():
header = ['日期', '工作内容', '备注']
the_datas = OrderedDict()
current_month = 1
current_day = date(2020, 1, 1)
while (current_day.year == 2020):
if not the_datas.__contains__(current_month):
the_datas[current_month] = []
if current_day.month == current_month:
the_datas[current_month].append(current_day.strftime('%m-%d'))
else:
current_month = current_day.month
current_day += timedelta(days=1)
workbook = xlsxwriter.Workbook('模板.xlsx')
format_h = workbook.add_format(datas.format_header)
for month, days in the_datas.items():
sheet = workbook.add_worksheet('%d月' % month)
sheet.default_row_height = 100
sheet.default_col_width = 10
sheet.set_row(0, 20)
sheet.set_column(1, 1, 120)
sheet.set_column(2, 2, 30)
sheet.write_row(0, 0, header, format_h)
sheet.write_column(1, 0, days, format_h)
workbook.close()
print(the_datas)
class OrderedAttrDict(object):
def __init__(self):
self._odict = OrderedDict()
def add(self, name, obj):
self._odict[name] = obj
self.__dict__[name] = obj
return obj
def keys(self):
return self._odict.keys()
def values(self):
return self._odict.values()
def items(self):
return self._odict.items()
def __len__(self):
return len(self._odict)
def __getitem__(self, key):
return self._odict[key]
def __contains__(self, k):
return self._odict.__contains__(k)
class OrderedSet(MutableSet):
def __init__(self, sequence=None):
super().__init__()
if sequence is None:
self._data = OrderedDict()
else:
kwargs = {v: 1 for v in sequence}
self._data = OrderedDict(**kwargs)
def __contains__(self, item):
"""Override."""
return self._data.__contains__(item)
def __iter__(self):
"""Override."""
return self._data.__iter__()
def __len__(self):
"""Override."""
return self._data.__len__()
def add(self, item):
"""Override."""
self._data.__setitem__(item, 1)
def discard(self, item):
"""Override."""
if item in self._data:
self._data.__delitem__(item)
def __repr__(self):
return f"{self.__class__.__name__}({list(self._data.keys())})"
def get_class_course_from_schedu(book):
class_courses = OrderedDict()
for c in classes.get().split(' '):
class_courses[c] = []
if True:
sh = book.sheet_by_name(sheet_name_class_all)
row_one = sh.row_values(0)
for rx in range(1, sh.nrows):
row = sh.row(rx)
for ix in range(2, sh.ncols):
if row[ix].ctype == 1:
vaules = list(row[ix].value)
if len(vaules) > 0:
for v in vaules:
k = v.replace('A', '晨曦').replace(
'B', '晨光').replace('C', '曙光').replace(
'D', '朝阳').replace('E', '旭日')
if class_courses.__contains__(k):
class_courses[k].append(row_one[ix])
elif row[ix].ctype == 2:
if row[ix].value == 5:
for c in class_courses.keys():
class_courses[c].append(row_one[ix])
return class_courses
class CompoundValue(object):
def __init__(self, *args, **kwargs):
self.__values__ = OrderedDict()
# Set default values
for container_name, container in self._xsd_type.elements_nested:
values = container.default_value
if isinstance(container, Indicator):
self.__values__.update(values)
else:
self.__values__[container_name] = values
# Set attributes
for attribute_name, attribute in self._xsd_type.attributes:
self.__values__[attribute_name] = attribute.default_value
# Set elements
items = _process_signature(self._xsd_type, args, kwargs)
for key, value in items.items():
self.__values__[key] = value
def __contains__(self, key):
return self.__values__.__contains__(key)
def __len__(self):
return self.__values__.__len__()
def __iter__(self):
return self.__values__.__iter__()
def __repr__(self):
return PrettyPrinter().pformat(self.__values__)
def __delitem__(self, key):
return self.__values__.__delitem__(key)
def __getitem__(self, key):
return self.__values__[key]
def __setitem__(self, key, value):
self.__values__[key] = value
def __setattr__(self, key, value):
if key.startswith('__') or key in ('_xsd_type', '_xsd_elm'):
return super(CompoundValue, self).__setattr__(key, value)
self.__values__[key] = value
def __getattribute__(self, key):
if key.startswith('__') or key in ('_xsd_type', '_xsd_elm'):
return super(CompoundValue, self).__getattribute__(key)
try:
return self.__values__[key]
except KeyError:
raise AttributeError(
"%s instance has no attribute '%s'" % (
self.__class__.__name__, key))
class CompoundValue(object):
def __init__(self, *args, **kwargs):
self.__values__ = OrderedDict()
# Set default values
for container_name, container in self._xsd_type.elements_nested:
values = container.default_value
if isinstance(container, Indicator):
self.__values__.update(values)
else:
self.__values__[container_name] = values
# Set attributes
for attribute_name, attribute in self._xsd_type.attributes:
self.__values__[attribute_name] = attribute.default_value
# Set elements
items = _process_signature(self._xsd_type, args, kwargs)
for key, value in items.items():
self.__values__[key] = value
def __contains__(self, key):
return self.__values__.__contains__(key)
def __len__(self):
return self.__values__.__len__()
def __iter__(self):
return self.__values__.__iter__()
def __repr__(self):
return PrettyPrinter().pformat(self.__values__)
def __delitem__(self, key):
return self.__values__.__delitem__(key)
def __getitem__(self, key):
return self.__values__[key]
def __setitem__(self, key, value):
self.__values__[key] = value
def __setattr__(self, key, value):
if key.startswith('__') or key in ('_xsd_type', '_xsd_elm'):
return super(CompoundValue, self).__setattr__(key, value)
self.__values__[key] = value
def __getattribute__(self, key):
if key.startswith('__') or key in ('_xsd_type', '_xsd_elm'):
return super(CompoundValue, self).__getattribute__(key)
try:
return self.__values__[key]
except KeyError:
raise AttributeError(
"%s instance has no attribute '%s'" % (
self.__class__.__name__, key))
def __contains__(self, key) -> bool:
""" Return True if dict has key, else False, e.g. 'key in dict' """
with self.lock:
if OrderedDict.__contains__(self, key):
# Each item is a list of [value, frame time]
item = OrderedDict.__getitem__(self, key)
if self.frame - item[1] < self.max_age:
return True
del self[key]
return False
def __contains__(self, key: Union[str, int]) -> bool: # type: ignore[override]
"""Check whether a key is in the table. String keys will be hashed.
key (str / int): The key to check.
RETURNS (bool): Whether the key is in the table.
"""
key = get_string_id(key)
# This can give a false positive, so we need to check it after
if key not in self.bloom:
return False
return OrderedDict.__contains__(self, key)
class MetafeatureFunctions(object):
def __init__(self):
self.functions = OrderedDict()
self.dependencies = OrderedDict()
self.values = OrderedDict()
def clear(self):
self.values = OrderedDict()
def __iter__(self):
return self.functions.__iter__()
def __getitem__(self, item):
return self.functions.__getitem__(item)
def __setitem__(self, key, value):
return self.functions.__setitem__(key, value)
def __delitem__(self, key):
return self.functions.__delitem__(key)
def __contains__(self, item):
return self.functions.__contains__(item)
def get_value(self, key):
return self.values[key].value
def set_value(self, key, item):
self.values[key] = item
def is_calculated(self, key):
"""Return if a helper function has already been executed.
Necessary as get_value() can return None if the helper function hasn't
been executed or if it returned None."""
return key in self.values
def get_dependency(self, name):
"""Return the dependency of metafeature "name".
"""
return self.dependencies.get(name)
def define(self, name, dependency=None):
"""Decorator for adding metafeature functions to a "dictionary" of
metafeatures. This behaves like a function decorating a function,
not a class decorating a function"""
def wrapper(metafeature_class):
instance = metafeature_class()
self.__setitem__(name, instance)
self.dependencies[name] = dependency
return instance
return wrapper
def __contains__(self, key):
"""Check whether a key is in the table. String keys will be hashed.
key (unicode / int): The key to check.
RETURNS (bool): Whether the key is in the table.
"""
key = get_string_id(key)
# This can give a false positive, so we need to check it after
if key not in self.bloom:
return False
return OrderedDict.__contains__(self, key)
def __getitem__(self, key):
if OrderedDict.__contains__(self, key):
val = OrderedDict.__getitem__(self, key)
elif '.' in key:
keys = key.split('.')
val = self[keys[0]]
for k in keys[1:]:
val = val[k]
else:
return OrderedDict.__getitem__(self, key)
return val
def create_actionmap(self, group):
actionMap = OrderedDict()
if(hasattr(group, '__iter__')):
for element in group:
for actionOfElement in element.Actions.keys():
if not actionMap.__contains__(actionOfElement):
actionMap[actionOfElement] = []
actionMap[actionOfElement].append(element)
else:
actionMap[actionOfElement].append(element)
return actionMap
else:
for actionOfElement in group.Actions.keys():
if not actionMap.__contains__(actionOfElement):
actionMap[actionOfElement] = group
else:
actionMap[actionOfElement] = group
return actionMap
class MetafeatureFunctions(object):
def __init__(self):
self.functions = OrderedDict()
self.dependencies = OrderedDict()
self.values = OrderedDict()
def clear(self):
self.values = OrderedDict()
def __iter__(self):
return self.functions.__iter__()
def __getitem__(self, item):
return self.functions.__getitem__(item)
def __setitem__(self, key, value):
return self.functions.__setitem__(key, value)
def __delitem__(self, key):
return self.functions.__delitem__(key)
def __contains__(self, item):
return self.functions.__contains__(item)
def get_value(self, key):
return self.values[key].value
def set_value(self, key, item):
self.values[key] = item
def is_calculated(self, key):
"""Return if a helper function has already been executed.
Necessary as get_value() can return None if the helper function hasn't
been executed or if it returned None."""
return key in self.values
def get_dependency(self, name):
"""Return the dependency of metafeature "name".
"""
return self.dependencies.get(name)
def define(self, name, dependency=None):
"""Decorator for adding metafeature functions to a "dictionary" of
metafeatures. This behaves like a function decorating a function,
not a class decorating a function"""
def wrapper(metafeature_class):
instance = metafeature_class()
self.__setitem__(name, instance)
self.dependencies[name] = dependency
return instance
return wrapper
def get(self, key, default=None, with_age=False):
""" Return the value for key if key is in dict, else default """
with self.lock:
if OrderedDict.__contains__(self, key):
item = OrderedDict.__getitem__(self, key)
if self.frame - item[1] < self.max_age:
if with_age:
return item[0], item[1]
return item[0]
if default is None:
raise KeyError(key)
elif with_age:
return default, self.frame
return
def getEsgfQuery(request_args):
where=os.path.dirname(__file__)
facetfile=os.path.join(where,"esgf-mars-facet-mapping")
mappingsfile=os.path.join(where,"esgf-mars-default-mapping")
attribsfile=os.path.join(where,"esgf-mars-attrib-mapping")
mappingsdict=OrderedDict()
attribsdict=OrderedDict()
facetsdict=OrderedDict()
fp=open(mappingsfile,'r')
mappingsdict=json.load(fp,object_pairs_hook=OrderedDict)
fp.close()
fp=open(attribsfile,'r')
attribsdict=json.load(fp,object_pairs_hook=OrderedDict)
fp.close()
fp=open(facetfile,'r')
facetsdict=json.load(fp,object_pairs_hook=OrderedDict)
fp.close()
variablename=''
freqval=''
try:
for facet,backend in facetsdict.iteritems():
val=request_args.get(facet)
if val!= None:
for backendopt in backend:
if attribsdict.__contains__(backendopt):
if facet =='variable':
variablename=val
if facet =='frequency':
freqval=val
#we have a match for the facet..
if attribsdict[backendopt].__contains__(val):
#we even have a substitution value for the specified facet value
#print 'backend value for user supplied facet value %s is %s'%(val,attribsdict[backendopt][val])
sub=attribsdict[backendopt][val]
#print 'before change: mappingsdict val=%s'%(mappingsdict[backendopt])
mappingsdict[backendopt]=sub
#print 'after change: mappingsdict val=%s'%(mappingsdict[backendopt])
#print "%s:%s"%(facet,val)
mappingsdict['date']=mappingsdict['datestr']
mappingsdict['date']+=str(mappingsdict['freq'])
mappingsdict.pop('datestr')
mappingsdict.pop('freq')
except:
raise
fn='%s_Eur05_SMHI-HIRLAM_RegRean_v0d0_SMHI-MESAN_v1_%s.grb'%(variablename,freqval)
return (fn,mappingsdict)
def pop(self, key, default=None, with_age=False):
""" Return the item and remove it """
with self.lock:
if OrderedDict.__contains__(self, key):
item = OrderedDict.__getitem__(self, key)
if self.frame - item[1] < self.max_age:
del self[key]
if with_age:
return item[0], item[1]
return item[0]
del self[key]
if default is None:
raise KeyError(key)
elif with_age:
return default, self.frame
return default
def get_dates(sh):
dates = OrderedDict()
the_date = 'default'
col_date = 0
col_time = 1
for rx in range(1, sh.nrows):
row = sh.row(rx)
if row[col_date].ctype == 1:
if len(row[col_date].value) > 0:
the_date = row[col_date].value
elif row[col_date].ctype == 3:
temp = xlrd.xldate.xldate_as_datetime(row[col_date].value, 0)
the_date = temp.strftime('%m-%d')
if not dates.__contains__(the_date):
dates[the_date] = []
dates[the_date].append(row[col_time].value)
return dates
class HelperFunctions:
def __init__(self):
self.functions = OrderedDict()
self.values = OrderedDict()
def clear(self):
self.values = OrderedDict()
self.computation_time = OrderedDict()
def __iter__(self):
return self.functions.__iter__()
def __getitem__(self, item):
return self.functions.__getitem__(item)
def __setitem__(self, key, value):
return self.functions.__setitem__(key, value)
def __delitem__(self, key):
return self.functions.__delitem__(key)
def __contains__(self, item):
return self.functions.__contains__(item)
def is_calculated(self, key):
"""Return if a helper function has already been executed.
Necessary as get_value() can return None if the helper function hasn't
been executed or if it returned None."""
return key in self.values
def get_value(self, key):
return self.values.get(key).value
def set_value(self, key, item):
self.values[key] = item
def define(self, name):
"""Decorator for adding helper functions to a "dictionary".
This behaves like a function decorating a function,
not a class decorating a function"""
def wrapper(metafeature_class):
instance = metafeature_class()
self.__setitem__(name, instance)
return instance
return wrapper
class OrderedSet:
def __init__(self, items=None):
self.d = OrderedDict()
self.update(items)
def update(self, items):
if items is not None:
for item in items:
self.d[item] = 1
def __iter__(self):
return self.d.__iter__()
def __contains__(self, key):
return self.d.__contains__(key)
def __delitem__(self, key):
return self.d.__delitem__(key)
def __len__(self):
return self.d.__len__()
def add(x):
return update(self, [x])
def discard(self, x):
if self.__contains__(x):
return self.__del__(x)
def remove(self, x):
if not self.__contains__(x):
raise KeyError
return self.__del__(x)
def _format_op(self, op):
if hasattr(op, 'name'):
return op.name
return str(op)
def __repr__(self):
if not self:
return '%s()' % (self.__class__.__name__, )
return '{%r}' % (','.join([self._format_op(op) for op in self]), )
class OrderedSet:
def __init__(self, items=None):
self.d = OrderedDict()
self.update(items)
def update(self, items):
if items is not None:
for item in items:
self.d[item] = 1
def __iter__(self):
return self.d.__iter__()
def __contains__(self, key):
return self.d.__contains__(key)
def __delitem__(self, key):
return self.d.__delitem__(key)
def __len__(self):
return self.d.__len__()
def add(self, x):
return update(self, [x])
def discard(self, x):
if self.__contains__(x):
return self.__del__(x)
def remove(self, x):
if not self.__contains__(x):
raise KeyError
return self.__del__(x)
def _format_op(self, op):
if hasattr(op, 'name'):
return op.name
return str(op)
def __repr__(self):
if not self:
return '%s()' % (self.__class__.__name__,)
return '{%r}' % (','.join([self._format_op(op) for op in self]),)
def create_index():
## stopwords from nltk.corpus
stopwords = ['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', 'her', 'hers', 'herself', 'it', 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don', 'should', 'now']
print "creating index.."
start = time.clock()
path1 = ".//Extracted_text_files"
dirlist = os.listdir(path1) ## list all the extracted text files
index = OrderedDict()
porter = nltk.PorterStemmer()
for input_file in dirlist: ## for each text file
count = 0
docid = input_file.strip()
with open(path1 + "//" + input_file,"r") as input_line: ## open the text file in read mode
lines = input_line.read().strip()
words = re.findall(r'\w+', lines,flags = re.UNICODE | re.LOCALE) ## extract words from the text
for word in words: ## for each word
count +=1 ## count is calculated to get the word position, hence count indicates the word's position
word = word.lower().strip() ## lower-case the word
word = porter.stem(word) ## stemming the words before inserting into the index
if index.__contains__(word): ## check if index contains the word
if index[word].__contains__(docid): ## check if there is a doc containing this word
index[word][docid].append(count) ## if yes, append the new position to previous positions
else: ## else create an entry for this new doc containing this word
index[word][docid] = [] ## initialize the value for this docid and
index[word][docid].append(count) ## insert the position into the index entry against this docid
else: ## if word not in index, create an index entry for this word
index[word]= OrderedDict()
index[word][docid] = []
index[word][docid].append(count) ## insert the position against the current docid
for word in stopwords: ## removing stop-words from the index
word = porter.stem(word)
if word in index.keys():
del index[word]
## pickle the index dictionary for querying and ranking
pickle.dump(index,open("index","wb"))
elapsed = (time.clock() - start)
print elapsed
class ArgumentsGroup(BaseType):
def __init__(self, *args, **kwargs):
arguments = to_arguments(*args, **kwargs)
self.arguments = OrderedDict([(arg.name, arg) for arg in arguments])
def internal_type(self, schema):
return OrderedDict([(arg.name, schema.T(arg))
for arg in self.arguments.values()])
def __len__(self):
return len(self.arguments)
def __iter__(self):
return iter(self.arguments)
def __contains__(self, *args):
return self.arguments.__contains__(*args)
def __getitem__(self, *args):
return self.arguments.__getitem__(*args)
def __contains__(self, key):
"""
Checks if given key exists in by first trying to find an exact match.
If no exact match is found then this method iterates trhough keys
and tries to check if given key is either:
1. A subclass of one of the keys
2. An instance of one of the keys
The first check has the time complexity of O(1) whereas the second
check has O(n).
Example::
class A(object):
pass
class B(object):
pass
class A2(A):
pass
class_map = ClassMap({A: 1, B: 2})
assert B in class_map
assert A in class_map
assert A2 in class_map
assert B() in class_map
assert A() in class_map
assert A2() in class_map
"""
if OrderedDict.__contains__(self, key):
return True
test_func = issubclass if isclass(key) else isinstance
return any(test_func(key, class_) for class_ in self)
class redict(object):
'''
@summary: Class to store regexes as dictionary keys
'''
def __init__(self):
self._dict = OrderedDict()
def __setitem__(self, key, value):
self._dict[key] = [re.compile(key[0], key[1]), value]
def __getitem__(self, key):
return self._dict[key]
def __delitem__(self, key):
self._dict.pop(key)
def __contains__(self, k):
return k in self._dict.__contains__(k)
has_key = __contains__
def __iter__(self):
for k in self._dict.iterkeys():
yield k
iterkeys = __iter__
def iteritems(self):
for (k,v) in self._dict.iteritems():
yield (k, v)
def itervalues(self):
for v in self._dict.itervalues():
yield v
# Extended functions
def iter_regex(self):
for (v1,v2) in self._dict.itervalues():
yield (v1, v2)
def OPTIMO(wlf,n):
pages=[]
# Read pages from file.
f = open(wlf)
pages= f.read().splitlines()
f.close()
cache_list = OrderedDict()
page_fault=0
count=0
# Insertion of pages to memory and calculation of page faults.
for page in pages:
tam_cache=len(cache_list)
lista=[]
count+=1
if not cache_list.__contains__(page):
if tam_cache < n:
cache_list[page]=''
page_fault += 1
else:
lista=[pages[x] for x in range(count+1,len(pages))]
list_inters = set(lista).intersection(cache_list)
if(len(list_inters)==len(cache_list)):
cache_list.popitem()
cache_list[page]=''
page_fault += 1
else:
lista_comp=set(list_inters).symmetric_difference(cache_list)
elem=lista_comp.pop()
cache_list.__delitem__(elem)
cache_list[page]=''
page_fault += 1
return page_fault
print "Resultados: "
print "Miss rate: ", ' '+str(round((float(page_fault*100)/(3721736)),3))+'%'
print 'Miss rate (warm cache): ', ' '+str(round((float(page_fault*100)/(3721736-n)),3))+'%'
with open("log.csv", "a") as output:
output.write('OPTIMO,'+str(page_fault)+','+str(n)+'\n')
output.close()
class Pipeline:
def __init__(self,name,**args):
self.name = name
self.dict = OrderedDict(args)
def __getitem__(self,key): return self.dict.__getitem__(key)
def __setitem__(self,key,val): return self.dict.__setitem__(key,val)
def has_key(self,key): return self.dict.__contains__(key)
def __repr__(self):
return 'Pipeline(%s,%r)' % (self.name,', '.join('%s=%s' % (key,val) for (key,val) in self.dict.items()))
def flush(self):
self.dict.clear()
def stall(self, needed):
stall = 0
for x in needed:
stall = max(stall, self.dict.get(x,0))
return stall
def conflicts(self, needed):
conflict = OrderedDict()
for x in needed:
if x in self.dict:
conflict[x] = self.dict[x]
return conflict
def retire(self, cycles=1):
dict_retire(self.dict, cycles)
def __contains__(self, key):
"""Override __contains__"""
return OrderedDict.__contains__(self, key.lower())
class DotMap(OrderedDict):
def __init__(self, *args, **kwargs):
self._map = OrderedDict()
self._dynamic = True # mettendo False non funzionano più i test di default. E' normale in quanto si aspettano la creazione dinamica dei figli
# ===================================
if LORETO:
global MY_DICT_TYPES # global var per la classe
self._dynamic = False # mettendo False non funzionano più i test di default. E' normale in quanto si aspettano la creazione dinamica dei figli
MY_DICT_TYPES = [dict, DotMap] # by Loreto (DEFAULT dictionary)
# ===================================
if kwargs:
if '_dynamic' in kwargs:
self._dynamic = kwargs['_dynamic']
if args:
d = args[0]
if isinstance(d, dict):
for k,v in self.__call_items(d):
if type(v) is dict:
v = DotMap(v, _dynamic=self._dynamic)
if type(v) is list:
l = []
for i in v:
n = i
if type(i) is dict:
n = DotMap(i, _dynamic=self._dynamic)
l.append(n)
v = l
self._map[k] = v
if kwargs:
for k,v in self.__call_items(kwargs):
if k is not '_dynamic':
self._map[k] = v
def __call_items(self, obj):
if hasattr(obj, 'iteritems') and ismethod(getattr(obj, 'iteritems')):
return obj.iteritems()
else:
return obj.items()
def items(self):
return self.iteritems()
def iteritems(self):
return self.__call_items(self._map)
def __iter__(self):
return self._map.__iter__()
def next(self):
return self._map.next()
def __setitem__(self, k, v):
self._map[k] = v
def __getitem__(self, k):
if k not in self._map and self._dynamic and k != '_ipython_canary_method_should_not_exist_':
# automatically extend to new DotMap
self[k] = DotMap()
return self._map[k]
def __setattr__(self, k, v):
if k in {'_map','_dynamic', '_ipython_canary_method_should_not_exist_'}:
super(DotMap, self).__setattr__(k,v)
else:
self[k] = v
def __getattr__(self, k):
if k == {'_map','_dynamic','_ipython_canary_method_should_not_exist_'}:
super(DotMap, self).__getattr__(k)
else:
return self[k]
def __delattr__(self, key):
return self._map.__delitem__(key)
def __contains__(self, k):
return self._map.__contains__(k)
def __str__(self):
items = []
for k,v in self.__call_items(self._map):
# bizarre recursive assignment situation (why someone would do this is beyond me)
if id(v) == id(self):
items.append('{0}=DotMap(...)'.format(k))
else:
items.append('{0}={1}'.format(k, repr(v)))
out = 'DotMap({0})'.format(', '.join(items))
return out
def __repr__(self):
return str(self)
def toDict(self):
d = {}
for k,v in self.items():
if type(v) is DotMap:
# bizarre recursive assignment support
if id(v) == id(self):
v = d
else:
v = v.toDict()
elif type(v) is list:
l = []
for i in v:
n = i
if type(i) is DotMap:
n = i.toDict()
l.append(n)
v = l
d[k] = v
return d
def pprint(self):
pprint(self.toDict())
# ===================================
if LORETO:
# MY_DICT_TYPES = [dict, DotMap]
def Ptr(self, listOfQualifiers, create=False):
ptr = self
for item in listOfQualifiers:
if item in ptr:
ptr = ptr[item]
else:
if create:
ptr[item] = DotMap()
ptr = ptr[item]
else:
return None
return ptr
def KeyTree(self, fPRINT=False):
return DictToList.KeyTree(self, myDictTYPES=MY_DICT_TYPES, fPRINT=fPRINT)
def KeyList(self):
return DictToList.KeyList(self, myDictTYPES=MY_DICT_TYPES)
def PrintTree(self, fEXIT=False, MaxLevel=10, header=None, printTYPE='LTKV', stackLevel=1):
PrintDictionaryTree.PrintDictionary(self, myDictTYPES=MY_DICT_TYPES, printTYPE=printTYPE, fEXIT=fEXIT, MaxLevel=MaxLevel, header=header, stackLevel=stackLevel+1)
printDict = PrintTree
printTree = PrintTree
def GetValue(self, listOfQualifiers=[], fPRINT=False):
return DictToList.getValue(self, listOfQualifiers=listOfQualifiers, myDictTYPES=MY_DICT_TYPES, fPRINT=fPRINT)
# ===================================
def empty(self):
return (not any(self))
# proper dict subclassing
def values(self):
return self._map.values()
# ipython support
def __dir__(self):
return self.keys()
@classmethod
def parseOther(self, other):
if type(other) is DotMap:
return other._map
else:
return other
def __cmp__(self, other):
other = DotMap.parseOther(other)
return self._map.__cmp__(other)
def __eq__(self, other):
other = DotMap.parseOther(other)
if not isinstance(other, dict):
return False
return self._map.__eq__(other)
def __ge__(self, other):
other = DotMap.parseOther(other)
return self._map.__ge__(other)
def __gt__(self, other):
other = DotMap.parseOther(other)
return self._map.__gt__(other)
def __le__(self, other):
other = DotMap.parseOther(other)
return self._map.__le__(other)
def __lt__(self, other):
other = DotMap.parseOther(other)
return self._map.__lt__(other)
def __ne__(self, other):
other = DotMap.parseOther(other)
return self._map.__ne__(other)
def __delitem__(self, key):
return self._map.__delitem__(key)
def __len__(self):
return self._map.__len__()
def clear(self):
self._map.clear()
def copy(self):
return DotMap(self.toDict())
def get(self, key, default=None):
return self._map.get(key, default)
def has_key(self, key):
return key in self._map
def iterkeys(self):
return self._map.iterkeys()
def itervalues(self):
return self._map.itervalues()
def keys(self):
return self._map.keys()
def pop(self, key, default=None):
return self._map.pop(key, default)
def popitem(self):
return self._map.popitem()
def setdefault(self, key, default=None):
self._map.setdefault(key, default)
def update(self, *args, **kwargs):
if len(args) != 0:
self._map.update(*args)
self._map.update(kwargs)
def viewitems(self):
return self._map.viewitems()
def viewkeys(self):
return self._map.viewkeys()
def viewvalues(self):
return self._map.viewvalues()
@classmethod
def fromkeys(cls, seq, value=None):
d = DotMap()
d._map = OrderedDict.fromkeys(seq, value)
return d
def __getstate__(self): return self.__dict__
def __setstate__(self, d): self.__dict__.update(d)
def __contains__(self, key):
if OrderedDict.__contains__(self, key):
return True
return self.parent and key in self.parent
class DotMap(OrderedDict):
def __init__(self, *args, **kwargs):
self._map = OrderedDict()
if args:
d = args[0]
if type(d) is dict:
for k,v in self.__call_items(d):
if type(v) is dict:
v = DotMap(v)
self._map[k] = v
if kwargs:
for k,v in self.__call_items(kwargs):
self._map[k] = v
def __call_items(self, obj):
if hasattr(obj, 'iteritems') and ismethod(getattr(obj, 'iteritems')):
return obj.iteritems()
else:
return obj.items()
def items(self):
return self.iteritems()
def iteritems(self):
return self.__call_items(self._map)
def __iter__(self):
return self._map.__iter__()
def next(self):
return self._map.next()
def __setitem__(self, k, v):
self._map[k] = v
def __getitem__(self, k):
if k not in self._map:
# automatically extend to new DotMap
self[k] = DotMap()
return self._map[k]
def __setattr__(self, k, v):
if k == '_map':
super(DotMap, self).__setattr__(k,v)
else:
self[k] = v
def __getattr__(self, k):
if k == '_map':
super(DotMap, self).__getattr__(k)
else:
return self[k]
def __delattr__(self, key):
return self._map.__delitem__(key)
def __contains__(self, k):
return self._map.__contains__(k)
def __str__(self):
items = []
for k,v in self.__call_items(self._map):
items.append('{0}={1}'.format(k, repr(v)))
out = 'DotMap({0})'.format(', '.join(items))
return out
def __repr__(self):
return str(self)
def toDict(self):
d = {}
for k,v in self.items():
if type(v) is DotMap:
v = v.toDict()
d[k] = v
return d
def pprint(self):
pprint(self.toDict())
# proper dict subclassing
def values(self):
return self._map.values()
@classmethod
def parseOther(self, other):
if type(other) is DotMap:
return other._map
else:
return other
def __cmp__(self, other):
other = DotMap.parseOther(other)
return self._map.__cmp__(other)
def __eq__(self, other):
other = DotMap.parseOther(other)
if not isinstance(other, dict):
return False
return self._map.__eq__(other)
def __ge__(self, other):
other = DotMap.parseOther(other)
return self._map.__ge__(other)
def __gt__(self, other):
other = DotMap.parseOther(other)
return self._map.__gt__(other)
def __le__(self, other):
other = DotMap.parseOther(other)
return self._map.__le__(other)
def __lt__(self, other):
other = DotMap.parseOther(other)
return self._map.__lt__(other)
def __ne__(self, other):
other = DotMap.parseOther(other)
return self._map.__ne__(other)
def __delitem__(self, key):
return self._map.__delitem__(key)
def __len__(self):
return self._map.__len__()
def clear(self):
self._map.clear()
def copy(self):
return self
def get(self, key, default=None):
return self._map.get(key, default)
def has_key(self, key):
return key in self._map
def iterkeys(self):
return self._map.iterkeys()
def itervalues(self):
return self._map.itervalues()
def keys(self):
return self._map.keys()
def pop(self, key, default=None):
return self._map.pop(key, default)
def popitem(self):
return self._map.popitem()
def setdefault(self, key, default=None):
self._map.setdefault(key, default)
def update(self, *args, **kwargs):
if len(args) != 0:
self._map.update(*args)
self._map.update(kwargs)
def viewitems(self):
return self._map.viewitems()
def viewkeys(self):
return self._map.viewkeys()
def viewvalues(self):
return self._map.viewvalues()
@classmethod
def fromkeys(cls, seq, value=None):
d = DotMap()
d._map = OrderedDict.fromkeys(seq, value)
return d
class DotMap(OrderedDict):
def __init__(self, *args, **kwargs):
self._map = OrderedDict()
self._dynamic = True
if kwargs:
if '_dynamic' in kwargs:
self._dynamic = kwargs['_dynamic']
if args:
d = args[0]
if isinstance(d, dict):
for k,v in self.__call_items(d):
if type(v) is dict:
v = DotMap(v, _dynamic=self._dynamic)
if type(v) is list:
l = []
for i in v:
n = i
if type(i) is dict:
n = DotMap(i, _dynamic=self._dynamic)
l.append(n)
v = l
self._map[k] = v
if kwargs:
for k,v in self.__call_items(kwargs):
if k is not '_dynamic':
self._map[k] = v
def __call_items(self, obj):
if hasattr(obj, 'iteritems') and ismethod(getattr(obj, 'iteritems')):
return obj.iteritems()
else:
return obj.items()
def items(self):
return self.iteritems()
def iteritems(self):
return self.__call_items(self._map)
def __iter__(self):
return self._map.__iter__()
def next(self):
return self._map.next()
def __setitem__(self, k, v):
self._map[k] = v
def __getitem__(self, k):
if k not in self._map and self._dynamic and k != '_ipython_canary_method_should_not_exist_':
# automatically extend to new DotMap
self[k] = DotMap()
return self._map[k]
def __setattr__(self, k, v):
if k in {'_map','_dynamic', '_ipython_canary_method_should_not_exist_'}:
super(DotMap, self).__setattr__(k,v)
else:
self[k] = v
def __getattr__(self, k):
if k == {'_map','_dynamic','_ipython_canary_method_should_not_exist_'}:
super(DotMap, self).__getattr__(k)
else:
return self[k]
def __delattr__(self, key):
return self._map.__delitem__(key)
def __contains__(self, k):
return self._map.__contains__(k)
def __str__(self):
items = []
for k,v in self.__call_items(self._map):
# bizarre recursive assignment situation (why someone would do this is beyond me)
if id(v) == id(self):
items.append('{0}=DotMap(...)'.format(k))
else:
items.append('{0}={1}'.format(k, repr(v)))
out = 'DotMap({0})'.format(', '.join(items))
return out
def __repr__(self):
return str(self)
def toDict(self):
d = {}
for k,v in self.items():
if type(v) is DotMap:
# bizarre recursive assignment support
if id(v) == id(self):
v = d
else:
v = v.toDict()
elif type(v) is list:
l = []
for i in v:
n = i
if type(i) is DotMap:
n = i.toDict()
l.append(n)
v = l
d[k] = v
return d
def pprint(self):
pprint(self.toDict())
def empty(self):
return (not any(self))
# proper dict subclassing
def values(self):
return self._map.values()
# ipython support
def __dir__(self):
return self.keys()
@classmethod
def parseOther(self, other):
if type(other) is DotMap:
return other._map
else:
return other
def __cmp__(self, other):
other = DotMap.parseOther(other)
return self._map.__cmp__(other)
def __eq__(self, other):
other = DotMap.parseOther(other)
if not isinstance(other, dict):
return False
return self._map.__eq__(other)
def __ge__(self, other):
other = DotMap.parseOther(other)
return self._map.__ge__(other)
def __gt__(self, other):
other = DotMap.parseOther(other)
return self._map.__gt__(other)
def __le__(self, other):
other = DotMap.parseOther(other)
return self._map.__le__(other)
def __lt__(self, other):
other = DotMap.parseOther(other)
return self._map.__lt__(other)
def __ne__(self, other):
other = DotMap.parseOther(other)
return self._map.__ne__(other)
def __delitem__(self, key):
return self._map.__delitem__(key)
def __len__(self):
return self._map.__len__()
def clear(self):
self._map.clear()
def copy(self):
return DotMap(self.toDict())
def get(self, key, default=None):
return self._map.get(key, default)
def has_key(self, key):
return key in self._map
def iterkeys(self):
return self._map.iterkeys()
def itervalues(self):
return self._map.itervalues()
def keys(self):
return self._map.keys()
def pop(self, key, default=None):
return self._map.pop(key, default)
def popitem(self):
return self._map.popitem()
def setdefault(self, key, default=None):
self._map.setdefault(key, default)
def update(self, *args, **kwargs):
if len(args) != 0:
self._map.update(*args)
self._map.update(kwargs)
def viewitems(self):
return self._map.viewitems()
def viewkeys(self):
return self._map.viewkeys()
def viewvalues(self):
return self._map.viewvalues()
@classmethod
def fromkeys(cls, seq, value=None):
d = DotMap()
d._map = OrderedDict.fromkeys(seq, value)
return d
def __getstate__(self): return self.__dict__
def __setstate__(self, d): self.__dict__.update(d)
class Stack(object):
""" Base class to store pandas objects, with special operations to
return as 3d data (eg panel) and to apply functions itemwise. Items are
stored in an ordered dict."""
itemlabel = 'Item'
_magic=['__len__',
'__iter__',
'__reversed__',
'__contains__',
]
def __init__(self, data, keys=None, name='', sort_items=False):
self.name = name
# Dictionary input
if isinstance(data, dict):
logger.debug('Initializing "%s" from dictionary.' % self.full_name)
if sort_items:
logger.debug('Sorting keys')
self._data=OrderedDict(sorted(data.keys(), key=lambda t: t[0]))
else:
self._data=OrderedDict(data)
else:
if not isinstance(data, Iterable):
logger.info('%s constructed from non-iterable... converting '
'data to an iterable' % self.full_name)
data=[data]
if keys:
if not isinstance(keys, Iterable):
logger.info('%s constructed from non-iterable... converting '
'keys to an iterable' % self.full_name)
keys = [keys]
if len(keys) != len(data):
raise ValueError('Length mistmatch: keys and data (%s,%s)'\
% (len(keys), len(data)))
# If keys not passed, generate them
else:
# Zipped data ((key, df), (key, df))
try:
keys, data = zip(*data)
except Exception:
keys=self._gen_keys(len(data))
if len(keys) > 1:
logger.warn("Generating keys %s-%s" % (keys[0], keys[-1]))
else:
logger.warn("Generating key %s" % keys[0])
self._data=OrderedDict( [ (key, data[i]) for (i, key)
in enumerate(keys) ])
@property
def _address(self):
""" Property to make easily accesible by multicanvas """
return mem_address(super(Stack, self).__repr__())
def _gen_keys(self, length):
""" Return a list of itemlables (item0, item1 etc...) using
self.itemlabel and a length"""
logger.debug('Items not found on %s: generating item list' % self.full_name)
return [self.itemlabel+str(i) for i in range(length)]
# --------------------
# Dictionary Interface
def __getitem__(self, keyslice):
""" If single name, used dict interface. If slice or integer, uses
list interface. All results parameterized to key, data pairs, passed
directly into a new Stack.
"""
# Slice as list of strings or int [0, 'foo', 2, 'bar']
if hasattr(keyslice, '__iter__'):
tuples_out = []
for item in keyslice:
if isinstance(item, str):
item = self._data.keys().index(item)
tuples_out.append(self._data.items()[item])
else:
if isinstance(keyslice, int) or isinstance(keyslice, slice):
tuples_out = self._data.items()[keyslice]
else:
tuples_out = [(keyslice, self._data[keyslice])] #keyslice is name
# If single item, return TimeSpectra, else, return new Stack
# Canonical slicing implementaiton; don't change unless good reason
# Because len() wonky with nested tuples (eg (x,y) and [(x1,y1),(x2,y2)]
# are both length two, this will work:
if sum(1 for x in tuples_out) == 2:
return tuples_out[1] #Return timespectra
else:
return self.__class__(tuples_out)
def __delitem__(self, keyslice):
""" Delete a single name, or a keyslice from names/canvas """
if isinstance(keyslice, str):
idx = self.names.index(keyslice)
self.pop(idx)
else:
raise NotImplementedError("Deletion only supports single entry")
def __setitem__(self, name, canvas):
""" """
if name in self.names:
idx = self.names.index(name)
self.pop(idx)
self.insert(idx, name, canvas)
else:
self.names.append(name)
def __getattr__(self, attr):
""" If attribute not found, try attribute lookup in dictionary. If
that is not found, try finding attribute on self._data.
For example, self.keys() will first look for self['keys']. Since
this isn't found, it calls self._data.keys(). But if I do
self.Item1, then it returns self['Item1']. The very rare conflict
case that a user has named the items a method that may already exist
in the dictionary (eg items=['a','b','keys'] is addressed.
"""
if attr in self._data.keys():
if hasattr(self._data, attr):
raise AttributeError('"%s attribute" found in both the items\
and as a method of the underlying dictionary object.'%(attr))
else:
return self[attr]
return getattr(self._data, attr)
# Attributes deferred to self.data /dictionary
def __len__(self):
return self._data.__len__()
def __iter__(self):
return self._data.__iter__()
def __reversed__(self):
return self._data.__reversed__()
def __contains__(self):
return self._data.__contains__()
def as_3d(self):
""" Return 3d structure of data. Default is panel."""
raise Panel(data=self._data)
### Data types without labels
#Is this realy necessary? See pyparty.ParticleManger for possibly more consistent implementation
def get_all(self, attr, astype=tuple):
"""Generator/tuple etc.. of (item, attribute) pairs. """
return put._parse_generator(
((item[0], getattr(item[1], attr)) for item in self.items()), astype)
def _get_unique(self, attr):
""" Inspects Stack itemwise for an attribute for unique values.
If non-unique value for the attributes are found, returns
"mixed".
"""
unique = set(self.get_all(attr, astype=dict).values())
if len(unique) > 1:
return 'mixed'
else:
return tuple(unique)[0] #set doesn't support indexing
def set_all(self, attr, val, inplace=False):
""" Set attributes itemwise.
If not inplace, returns new instance of self"""
if inplace:
for (key, item) in self.items():
try:
setattr(item, attr, val)
except Exception as E:
raise Exception('Could not set %s in "%s". Received the following \
exception:\n "%s"'%(attr, key, E))
else:
out=deepcopy(self._data) #DEEPCOPY
for item in out:
setattr(out[item], attr, val)
return self.__class__(out)
def apply(self, func, *args, **kwargs):
""" Applies a user-passed function, or calls an instance method itemwise.
Parameters:
-----------
func: str or function
If string, must correspond to a method on the object stored
itemwise in the stack. If a function, appliked itemwise to
objects stored.
inplace: False
Special kwarg. If true, self._data modified inplace,
otherwise new specstack is returned.
*args, **kwargs:
func arguments.
Returns:
--------
If not inplace, returns SpecStack after itemwise application.
"""
inplace=kwargs.pop('inplace', False)
if isinstance(func, basestring):
if inplace:
for item in self:
self[item] = getattr(self[item], func)(*args, **kwargs)
else:
return self.__class__(OrderedDict([(k, getattr(v, func)(*args, \
**kwargs)) for k,v in self.items()]))
# function, numpyfunction etc...
else:
if inplace:
for item in self:
self[item] = self[item].apply(func)(*args, **kwargs)
else:
return self.__class__(OrderedDict([(k, v.apply(func, *args, \
**kwargs)) for k,v in self.items()]))
@property
def full_name(self):
""" Timespectra:name or Timespectra:unnamed. Useful for scripts mostly """
outname = getattr(self, 'name', 'unnamed')
return '%s:%s' % (self.__class__.__name__, self.name)
from collections import OrderedDict
d = OrderedDict()
for tc in range(int(input())):
key = input()
if d.__contains__(key): d[key] += 1
else: d[key] = 1
print(len(d))
print(" ".join(map(lambda x: str(d[x]),d)))
# If page in list is the same as the page that is going to be inserted, return i.
if page_in_list == page:
return i
# If we reached the end of the length of the arguments, return i.
if (i == len_of_remaining_page):
return i
#
# Authored by Juan Mite
# LRU algorithm
#
if policy.upper() == 'LRU':
print 'Evaluando una caché LRU con '+ str(cache_tam) +' entradas, tiempo estimado 25 segundos...'
with open(workload_file) as f:
for line in f:
if (cache.__contains__(line)):
# HIT
cache.__delitem__(line)
cache[line] = ''
hites = hites + 1
else:
# MISS
misses = misses + 1
cache[line] = ''
if len(cache) > cache_tam :
cache.popitem(last=False)
references = hites + misses
print "Resultados: "
print "Miss rate: ", ' '+str(round((float(misses)/(references)),3))+'% ('+str(misses)+' misses out of '+str(references)+' references)'
print 'Miss rate (warm cache): ', ' '+str(round((float(misses)/(references-cache_tam)),3))+'% ('+str(hites)+' misses out of '+ str(references-cache_tam)+' references)'
print 'Efficiency: '+str(round(float(hites)/(references),3))
def __contains__(self, key):
l = _lower(key)
return OrderedDict.__contains__(self, l)
class ChainSet(object):
"""
Base class for various methods to rename chains
Contains _chains, which maps from the renamed chain to a tuple with the
original (object,state,chain). All dict-like accessors work on ChainSets,
e.g.
chain_set["A"] -> ("obj",1,"A")
"""
def __init__(self):
# Use an OrderedDict in Python >= 1.7 for better printing
if _orderedDict:
self._chains = OrderedDict()
else:
self._chains = dict()
def map_chain(self, obj, state, origChain ):
"""
map_chain(string obj,int state, string chain]]) -> string
Maps a chain letter to a unique chainID. Results are unique within each
instance, and can be used as keys on this chain set.
"""
raise NotImplementedError("Base class")
# delegate most methods to _chains
def __getattr__(self,at):
if at in "pop popitem update setdefault".split():
raise AttributeError("type object '%s' has no attribute '%s'"%(type(self),at))
return getattr(self._chains,at)
def __cmp__(self,other): return self._chains.__cmp__(other)
def __eq__(self,other): return self._chains.__eq__(other)
def __ge__(self,other): return self._chains.__ge__(other)
def __gt__(self,other): return self._chains.__gt__(other)
def __le__(self,other): return self._chains.__le__(other)
def __lt__(self,other): return self._chains.__lt__(other)
def __ne__(self,other): return self._chains.__ne__(other)
def __len__(self): return self._chains.__len__()
def __contains__(self,key): return self._chains.__contains__(key)
def __getitem__(self,key): return self._chains.__getitem__(key)
def __iter__(self): return self._chains.__iter__()
def __str__(self): return str(self._chains)
@staticmethod
def _int_to_chain(i,base=_default_base):
"""
_int_to_chain(int,int) -> str
Converts a positive integer to a chain ID. Chain IDs include uppercase
characters, numbers, and optionally lowercase letters.
i = a positive integer to convert
base = the alphabet size to include. Typically 36 or 62.
"""
if i < 0:
raise ValueError("positive integers only")
if base < 0 or 62 < base:
raise ValueError("Invalid base")
quot = int(i)//base
rem = i%base
if rem < 26:
letter = chr( ord("A") + rem)
elif rem < 36:
letter = str( rem-26)
else:
letter = chr( ord("a") + rem - 36)
if quot == 0:
return letter
else:
return ChainSet._int_to_chain(quot-1,base) + letter
def __contains__(self, key):
if self._allow_contains_default:
if key in getattr(self, "defaults_", {}):
return True
return OrderedDict.__contains__(self, key)
class Cache(object):
"""Base caching class
It uses a least recently used (LRU) algorithm to determine which
entries to delete when the cache gets filled.
"""
def __init__(self, size=0):
"""Zero or negative values for size will create an unlimited cache"""
self.maxSize = size
self.dict = OrderedDict()
self.lock = RLock()
self.logger = logging.getLogger("Cache")
self.logger.setLevel(logging.DEBUG)
# If you want to see debug messages change level here
self.logger.setLevel(logging.WARNING)
def __getitem__(self, key):
with self.lock:
# This curious code just puts the item at
# the end of the OrderedDict to enforce the
# LRU caching algorithm
retval = self.dict.pop(key)
self.dict[key] = retval
return retval
def __setitem__(self, key, value):
with self.lock:
self._trim()
self.dict[key] = value
def __delitem__(self, key):
with self.lock:
del self.dict[key]
def __contains__(self, key):
return self.dict.__contains__(key)
def values(self):
"""Returns all the cached values"""
return self.dict.values()
def keys(self):
"""Returns the keys of all the cached values"""
return self.dict.keys()
def touch(self, key):
"""Accesses a cached item so it will not be removed from the cache"""
self.__getitem__(key)
def setMaxSize(self, size):
"""Sets the maximum size of the cache and removes any elements that do not fit"""
self.maxSize = size
self._trim()
def _trim(self):
"""Remove elements that exceed the maximum cache size"""
with self.lock:
if(self.maxSize > 0):
while len(self.dict) >= self.maxSize:
(k, _) = self.dict.popitem(last=False) # FIFO pop
self.logger.debug("Removing %d from cache (maxSize=%i)", k, self.maxSize)
class SortedDotDict(object):
def __init__(self, *args, **kwargs):
super(SortedDotDict, self).__init__(*args, **kwargs)
self._dict = SortedDict()
def __contains__(self, *args, **kwargs):
return self._dict.__contains__(*args, **kwargs)
def __eq__(self, *args, **kwargs):
return self._dict.__eq__(*args, **kwargs)
def __format__(self, *args, **kwargs):
return self._dict.__format__(*args, **kwargs)
def __ge__(self, *args, **kwargs):
return self._dict.__ge__(*args, **kwargs)
def __getattr__(self, key):
try:
return self._dict[key]
except:
raise AttributeError(key)
def __iter__(self):
vals = list(self.values())
for k in vals:
yield k
def __getitem__(self, key):
return self._dict[key]
def __setitem__(self, key, value):
self._dict[key] = value
def __delitem__(self, key):
del self._dict[key]
def keys(self):
return list(self._dict.keys())
def values(self):
vals = list(self._dict.values())
vals = [v for v in vals if isinstance(v, (ConfigurationGroup, Value))]
vals.sort()
return vals
def items(self):
return list(self._dict.items())
def iterkeys(self):
return iter(self._dict.keys())
def itervalues(self):
return iter(self._dict.values())
def iteritems(self):
return iter(self._dict.items())
def get(self, *args, **kwargs):
return self._dict.get(*args, **kwargs)
def clear(self):
return self._dict.clear()
def copy(self):
s = SortedDotDict()
s._dict = self._dict.copy()
return s
def fromkeys(self):
return self._dict.fromkeys()
def has_key(self, key):
return key in self._dict
def pop(self, *args, **kwargs):
return self._dict.pop(*args, **kwargs)
def popitem(self, *args, **kwargs):
return self._dict.popitem(*args, **kwargs)
def setdefault(self, key, default):
return self._dict.setdefault(key, default)
def update(self, d):
return self._dict.update(d)
def viewitems(self, *args, **kwargs):
return self._dict.viewitems(*args, **kwargs)
def viewvalues(self, *args, **kwargs):
return self._dict.viewvalues(*args, **kwargs)
def __contains__(self, name):
name = name.lower()
return OrderedDict.__contains__(self, name)
