def codename2server(codename):
aliases = DictTool.filter(
lambda k, v: v, {
"en": h_codename2aliases_en.get(codename),
"ko": h_codename2aliases_ko.get(codename),
})
culture = {
Server.Field.CODENAME: codename,
Server.Field.ALIASES: aliases,
}
return DictTool.filter(lambda k, v: v, culture)
def codename2culture(codename):
aliases = DictTool.filter(
lambda k, v: v, {
"en": h_codename2aliases_en.get(codename),
"ko": h_codename2aliases_ko.get(codename),
})
culture = {
Culture.Field.CODENAME: codename,
Culture.Field.ALIASES: aliases,
Culture.Field.PREFERS: h_codename2prefers.get(codename) or [],
}
return DictTool.filter(lambda k, v: v, culture)
def codename2port(codename):
aliases = DictTool.filter(lambda k, v: v,
{"en": h_codename2aliases_en.get(codename),
"ko": h_codename2aliases_ko.get(codename),
})
tradegoodtype = h_codename2tradegoodtype.get(codename)
port = DictTool.filter(lambda k, v: bool(v),
{Tradegood.Field.CODENAME: codename,
Tradegood.Field.TRADEGOODTYPE: tradegoodtype,
Tradegood.Field.ALIASES: aliases,
}
)
return DictTool.filter(lambda k, v: v, port)
def codename2port(codename):
aliases = DictTool.filter(
lambda k, v: v, {
"en": h_codename2aliases_en.get(codename),
"ko": h_codename2aliases_ko.get(codename),
})
category = h_codename2category.get(codename)
port = DictTool.filter(
lambda k, v: bool(v), {
Tradegoodtype.Field.CODENAME: codename,
Tradegoodtype.Field.CATEGORY: category,
Tradegoodtype.Field.ALIASES: aliases,
})
return DictTool.filter(lambda k, v: v, port)
def server2datetime_nanban(
cls,
server_codename,
dt_pivot,
):
server_doc = ServerDoc.codename2doc(server_codename)
if not server_doc:
return None
nanban_prev = ServerDoc.doc2nanban(server_doc)
nanban_datetime_prev = ServerNanban.nanban2datetime(nanban_prev)
# dt_nanban_raw
# utc_now = datetime.now(tz=pytz.utc)
nanban_datetime_this = DatetimeTool.from_pivot_period2next(
nanban_datetime_prev, dt_pivot, NanbanTimedelta.period())
if nanban_datetime_this != nanban_datetime_prev:
nanban = DictTool.filter(
lambda k, v: v is not None, {
ServerNanban.Field.DATETIME:
nanban_datetime_this,
ServerNanban.Field.COMMAND_IN:
ServerNanban.nanban2command_in(nanban_prev),
})
doc = {
ServerDoc.Field.CODENAME: server_codename,
ServerDoc.Field.NANBAN: nanban,
}
ServerDoc.docs2upsert([doc])
return nanban_datetime_this
def index_init_unittest(cls, settings=None, mappings=None):
logger = FoxylibLogger.func_level2logger(cls.index_init_unittest, logging.DEBUG)
client = FoxylibElasticsearch.client()
index = UnittestIndex.env2name(FoxylibEnv.env())
logger.debug({"index_unittest": index})
# ElasticsearchTool.delete_or_skip(client, index_unittest)
# settings = UnittestIndex.settings()
# mappings = UnittestIndex.mappings()
body = DictTool.filter(lambda k, v: bool(v),
{"settings": settings if settings else None,
"mappings": mappings if mappings else None,
}
)
ElasticsearchTool.create_or_skip(client, index, body=body)
query_all = {"match_all": {}}
client.delete_by_query(index, {"query": query_all}, refresh=True)
result = client.search(index, body={"query": query_all, })
assert_false(ElasticsearchTool.j_result2j_hit_list(result))
return client, index
def norm(cls, match):
keys = [
cls.Field.CHARSPAN_REF,
cls.Field.MATCHSTRING_REF,
cls.Field.CHARSPAN_HYP,
cls.Field.MATCHSTRING_HYP,
]
return DictTool.exclude_keys(match, keys)
def codename2port(codename):
aliases = DictTool.filter(lambda k, v: v,
{"en": h_codename2aliases_en.get(codename),
"ko": h_codename2aliases_ko.get(codename),
})
comments = DictTool.filter(lambda k, v: v,
{"ko": h_codename2comments_ko.get(codename),
})
port = {Port.Field.CODENAME: codename,
Port.Field.CULTURE: h_codename2culture[codename],
Port.Field.ALIASES: aliases,
Port.Field.PRODUCTS: h_codename2product_list.get(codename),
Port.Field.COMMENTS: comments,
}
return DictTool.filter(lambda k, v: v, port)
def match2matchstring_hyp(cls, match):
""" lazy implementation """
def _match2matchstring_hyp(_match):
fulltext_hyp = cls.match2fulltext_hyp(_match)
charspan_hyp = cls.match2charspan_hyp(_match)
return StringTool.str_span2substr(fulltext_hyp, charspan_hyp)
return DictTool.lazyget(match, cls.Field.MATCHSTRING_HYP,
partial(_match2matchstring_hyp, match))
def match2matchstring_ref(cls, match):
""" lazy implementation """
def _match2matchstring_ref(_match):
fulltext_ref = cls.match2fulltext_ref(_match)
charspan_ref = cls.match2charspan_ref(_match)
return StringTool.str_span2substr(fulltext_ref, charspan_ref)
return DictTool.lazyget(match, cls.Field.MATCHSTRING_REF,
partial(_match2matchstring_ref, match))
def row2product(row):
port, tradegood = row[0], row[1]
price = row[2] if len(row) >= 3 else None
raw = {Product.Field.PORT: port,
Product.Field.TRADEGOOD: tradegood,
Product.Field.PRICE: price,
}
product = DictTool.filter(lambda k,v:v, raw)
return product
def method2hideout(cls, owner, method):
"""
owner needs to be passed because classmethod has no "__self__" before decorator
"""
dict_method2hideout = AttributeTool.get_or_init(
owner, CacheManager.Constant.ATTRIBUTE_NAME, {})
hideout = DictTool.get_or_init(dict_method2hideout,
FunctionTool.func2name(method), {})
return hideout
def codename2chatroomuser(codename):
aliases = h_codename2aliases.get(codename) or []
comments = h_codename2comments.get(codename)
chatroomuser = {
Chatroomuser.Field.CODENAME: codename,
Chatroomuser.Field.COMMENTS: comments,
Chatroomuser.Field.ALIASES: aliases,
}
return DictTool.filter(lambda k, v: v, chatroomuser)
def match2charspan_ref(cls, match):
""" lazy implementation """
def _match2charspan_ref(_match):
token_list_ref = cls.match2tokens_ref(_match)
tokenspan_ref = cls.match2tokenspan_ref(_match)
charspan_ref = SubphraseMatcher.Token.token_list_span2charspan(
token_list_ref, tokenspan_ref)
return charspan_ref
return DictTool.lazyget(match, cls.Field.CHARSPAN_REF,
partial(_match2charspan_ref, match))
def test_03(self):
dict_value2texts = DictTool.append_key2values({"ReD": ["scarleTT", "radish"]})
gazetteer = GazetteerMatcher(dict_value2texts)
span_value_list = list(gazetteer.text2span_value_iter("ReD scarlett blue radish"))
hyp = span_value_list
ref = [((0, 3), 'ReD'), ((18, 24), 'ReD')]
# pprint(hyp)
self.assertEqual(hyp, ref)
def test_01(self):
dict_value2texts = DictTool.append_key2values(
{"ReD": ["scarleTT", "radish"]})
matcher = FulltextMatcher(dict_value2texts,
config={"normalizer": str2lower})
hyp1 = list(matcher.text2values("scarlett"))
self.assertEqual(hyp1, ["ReD"])
hyp2 = list(matcher.text2values("red scarlett"))
self.assertEqual(hyp2, [])
def item2doc(item):
port_codename = JsonTool.down(item, ["_id", cls.Field.PORT])
tradegood_codename = JsonTool.down(item, ["_id", cls.Field.TRADEGOOD])
price = merge_dicts([DictTool.keys2filtered(item, fields_others, ),
{cls.Field.PORT: port_codename,
cls.Field.TRADEGOOD: tradegood_codename,
},
], vwrite=vwrite_no_duplicate_key
)
return price
def attach_cachedmethod(
cls,
func=None,
cachedmethod=None,
self2cache=None,
key=None,
lock=None,
):
logger = FoxylibLogger.func_level2logger(cls.attach_cachedmethod,
logging.DEBUG)
assert_is_not_none(self2cache)
cachedmethod = cachedmethod or cachetools.cachedmethod
config = DictTool.filter(
lambda k, v: v is not None, {
cls.Config.Field.SELF2CACHE: self2cache,
cls.Config.Field.KEY: key or hashkey,
cls.Config.Field.LOCK: lock,
})
kwargs = cls.Config.config2kwargs(config)
def wrapper(f):
# types_valid = {CallableTool.Type.INSTANCEMETHOD,CallableTool.Type.CLASSMETHOD_BEFORE_DECORATOR,}
# assert_in(CallableTool.callable2type(f), types_valid,
# "For functions, use attach_cached() instead")
assert_false(hasattr(f, cls.Constant.ATTRIBUTE_NAME))
setattr(f, cls.Constant.ATTRIBUTE_NAME, config)
_self2cache = cls.Config.config2self2cache(config)
@wraps(f)
def wrapped(self, *_, **__):
# config = cls.callable2config(f)
# _self2cache = cls.Config.config2cache(config)
# f.__self__ doesn't work here because classmethod() is not called yet
hideout = cls.Hideout.method2hideout(self, f)
cache = cls.Hideout.get_or_lazyinit_cache(
hideout, lambda: _self2cache(self))
# logger.debug({"f":f,"cache":cache})
f_with_cache = cachedmethod(lambda x: cache, **kwargs)(f)
result = f_with_cache(self, *_, **__)
# raise Exception({"self":self,"result":result,"cache":cache})
return result
return wrapped
return wrapper(func) if func else wrapper
def test_01(self):
dict_value2texts = DictTool.append_key2values({"ReD": ["scarleTT", "radish"]})
gazetteer = GazetteerMatcher(dict_value2texts, config={"normalizer":str2lower})
span_value_list = list(gazetteer.text2span_value_iter("red scarlett blue radish"))
hyp = span_value_list
ref = [((0, 3), 'ReD'),
((4, 12), 'ReD'),
((18, 24), 'ReD')]
# pprint(hyp)
self.assertEqual(hyp, ref)
def match2charspan_hyp(cls, match):
""" lazy implementation """
def _match2charspan_hyp(_match):
token_list_hyp = cls.match2tokens_hyp(_match)
tokenspan_hyp = cls.match2tokenspan_hyp(_match)
# logger.debug({"token_list_hyp":token_list_hyp, "tokenspan_hyp":tokenspan_hyp})
charspan_hyp = SubphraseMatcher.Token.token_list_span2charspan(
token_list_hyp, tokenspan_hyp)
return charspan_hyp
return DictTool.lazyget(match, cls.Field.CHARSPAN_HYP,
partial(_match2charspan_hyp, match))
def docs_pair2j_diff(cls, docs_pair, keys=None):
docs1, docs2 = docs_pair
doc_list_1, doc_list_2 = list(docs1), list(docs2)
n1, n2 = len(doc_list_1), len(doc_list_2)
if keys is None:
keys = DictTool.dicts2keys(chain(doc_list_1, doc_list_2))
h2key = lambda h: tuple(h[k] for k in keys)
h1 = merge_dicts([{
h2key(h): h
} for h in doc_list_1],
vwrite=vwrite_no_duplicate_key)
key_list_1 = lmap(h2key, doc_list_1)
key_list_2 = lmap(h2key, doc_list_2)
key_set_1 = set(key_list_1)
key_set_2 = set(key_list_2)
def sheet_ranges2data_lll(cls, credentials, spreadsheet_id, ranges=None):
logger = FoxylibLogger.func_level2logger(cls.sheet_ranges2data_lll,
logging.DEBUG)
logger.debug({"spreadsheet_id": spreadsheet_id, "ranges": ranges})
# service = build('sheets', 'v4', http=credentials.authorize(Http()))
service = build('sheets', 'v4', credentials=credentials)
h = DictTool.filter(lambda k, v: v, {
"spreadsheetId": spreadsheet_id,
"ranges": ranges,
})
result = service.spreadsheets().values().batchGet(**h).execute()
# pprint({"result":result})
values = lmap(lambda h: h.get("values") or [],
result.get('valueRanges', []))
logger.debug({"len(values)": len(values)})
return values
class Merge:
@classmethod
def merge2dict(
cls,
h_to,
h_from,
vwrite=None,
):
if vwrite is None:
vwrite = DictTool.VWrite.overwrite
vwrite_out = CSSTool.VWrite.is_attr_appendable2vwrite(
CSSTool.VWrite.attr2is_appendable,
vwrite,
)
return DictTool.Merge.merge2dict(
h_to,
h_from,
vwrite=vwrite_out,
)
merge_dicts = DictTool.f_binary2f_iter(merge2dict, default={})
def attach_cached(
cls,
func=None,
cached=None,
cache=None,
key=None,
lock=None,
):
logger = FoxylibLogger.func_level2logger(cls.attach_cached,
logging.DEBUG)
assert_is_not_none(cache)
cached = cached or cachetools.cached
config = DictTool.filter(
lambda k, v: v is not None, {
cls.Config.Field.CACHE: cache,
cls.Config.Field.KEY: key or hashkey,
cls.Config.Field.LOCK: lock,
})
kwargs = cls.Config.config2kwargs(config)
def wrapper(f):
"""
CLASSMETHOD can used cached() too.
But for simplicity of the system, CLASSMETHOD is forced to use cachedmethod
"""
# types_valid = {CallableTool.Type.FUNCTION, }
# assert_in(CallableTool.callable2type(f), types_valid,
# "For instancemethod, use attach_cachedmethod() instead")
assert_false(hasattr(f, cls.Constant.ATTRIBUTE_NAME))
setattr(f, cls.Constant.ATTRIBUTE_NAME, config)
f_cached = cached(cache, **kwargs)(f)
return f_cached
return wrapper(func) if func else wrapper
def i2entity(i):
entity = entity_list_in[i]
assert_equal(FoxylibEntity.entity2type(entity),
TimeEntity.entity_type())
if i not in h_i2j:
return entity_list_in[i]
j = h_i2j[i]
m_ampm = m_list_ampm[j]
span = (span_list_in[i][0], span_list_ampm[j][1])
v_entity = FoxylibEntity.entity2value(entity)
hour, minute, second = TimeEntity.Value.value2hms(v_entity)
ampm = AMPM.match2value(m_ampm)
hour_adjusted, ampm_adjusted = AMPM.hour_ampm2normalized(
hour, ampm)
# logger.debug({"hour":hour, "ampm":ampm,
# "hour_adjusted":hour_adjusted, "ampm_adjusted":ampm_adjusted})
value = DictTool.filter(
lambda k, v: v is not None, {
TimeEntity.Value.Field.HOUR: hour_adjusted,
TimeEntity.Value.Field.MINUTE: minute,
TimeEntity.Value.Field.SECOND: second,
TimeEntity.Value.Field.AMPM: ampm_adjusted,
})
entity = {
FoxylibEntity.Field.TYPE: FoxylibEntity.entity2type(entity),
FoxylibEntity.Field.FULLTEXT: text_in,
FoxylibEntity.Field.SPAN: span,
FoxylibEntity.Field.VALUE: value,
}
return entity
def test_02(self):
cls = self.__class__
cache = LRUCache(maxsize=200, )
@cached(cache=cache)
def f(x, y=None):
return 2
cache[hashkey("a", )] = 3
cache[hashkey("b", )] = 4
cache[hashkey("d", y="y")] = 5
self.assertEqual(f("a"), 3)
self.assertEqual(f("b"), 4)
self.assertEqual(f("d", y="y"), 5)
self.assertEqual(f("c"), 2)
del cache[hashkey("a", )]
self.assertEqual(f("a"), 2)
self.assertEqual(f("b"), 4)
self.assertEqual(DictTool.pop(cache, hashkey("b", )), 4)
self.assertEqual(f("b"), 2)
def form2j_form(cls, form):
if not form:
return form
return DictTool.filter(lambda k, v: v, form.patch_data)
def dict_codename2comments(cls):
data_ll = PortGooglesheets.sheetname2data_ll(cls.NAME)
h = merge_dicts([{row[0]: lfilter(bool,row[1:])} for row in data_ll[1:]],
vwrite=vwrite_no_duplicate_key)
return DictTool.filter(lambda k,v:bool(v), h)
def doc2norm_unittest(cls, doc):
return DictTool.keys2excluded(doc, [cls.Field.CREATED_AT])
def config2normalizer(cls, config):
return DictTool.lookup(config, cls.Key.NORMALIZER)