def __init__(self, schemas, options):
defaultOptions = {
'fuzzyLink': True,
'fuzzyEmail': True,
'fuzzyIp': False
}
# DON'T try to make PRs with changes. Extend TLDs with LinkifyIt.tlds() instead
tlds_default = [
'biz', 'com', 'edu', 'gov', 'net', 'org', 'web', 'xxx', 'aero',
'asia', 'coop', 'info', 'miseum', 'name', 'shop', 'рф'
]
self.schemas = schemas
self.options = options
self._opts = merge_dict(defaultOptions, options)
# Cache last tested result. Used to skip repeating steps on next `match` call.
self._index = -1
self._last_index = -1
self._schema = ''
self._text_cache = ''
self._schemas = merge_dict(defaultOptions, schemas)
self._compiled = {}
self._tlds = tlds_default
self._tlds_replaced = False
self.re = {}
# todo
compile(self)
def test_merge_dict(self):
bdict = {
'CherryPy': '3.2.4',
'Django': '1.4.13',
'MySQL-python': '1.2.3',
'Pillow': '2.1.0',
'MarkupSafe': '0.18'
}
mdict = {
'CherryPy': '3.2.0',
'Django': '1.4.14',
'MySQL-python': '1.2.3',
'Pillow': '2.1.0',
'MarkupSafe': '0.18',
'SQLAlchemy': '0.5.8'
}
res_dict = merge_dict(bdict, mdict)
merged_dict = {
'Django': '1.4.14',
'MarkupSafe': '0.18',
'MySQL-python': '1.2.3',
'Pillow': '2.1.0',
'SQLAlchemy': '0.5.8',
'CherryPy': '3.2.4'
}
self.assertDictEqual(
merge_dict(bdict, mdict),
merged_dict,
'test_merge_dict'
)
def __init__(self, task_config, global_config):
merged = merge_dict(merge_dict(
dict_without_key(task_config, 'sqoop_args'),
dict_without_key(global_config, 'sqoop_args')),
TaskConfig.DEFAULT)
for key in ['source', 'hive', 'skip_tables',
'map_types', 'import_tables']:
setattr(self, key, merged[key])
self._config = task_config
self._global_config = global_config
def __init__(self, task_config, global_config):
merged = merge_dict(
merge_dict(dict_without_key(task_config, 'sqoop_args'),
dict_without_key(global_config, 'sqoop_args')),
TaskConfig.DEFAULT)
for key in [
'script', 'source', 'jvmargs', 'hive', 'skip_tables',
'map_types', 'import_tables'
]:
setattr(self, key, merged[key])
self._config = task_config
self._global_config = global_config
def process_deaths(self):
"""
Processes the global confirmed in-memory records
"""
logger = logging.getLogger("Deaths")
csse_handler_global = CSSEGISandData(logger, USFileType=False)
global_deaths_gps_data = csse_handler_global.parse_csv_file_contents(
self.global_deaths_dataset)
self.date_keys_sanity_check(csse_handler_global.date_keys)
# The header of the US file has the dates start at offset 12
# perhaps we should validate this never changes, or the data will
# be out of sync
csse_handler_us = CSSEGISandData(logger,
USFileType=True,
offset_dates=12)
us_deaths_gps_data = csse_handler_us.parse_csv_file_contents(
self.us_deaths_dataset)
self.date_keys_sanity_check(csse_handler_us.date_keys)
deaths_gps_data = utils.merge_dict(global_deaths_gps_data,
us_deaths_gps_data)
utils.write_to_file(
"data/deaths.json",
csse_handler_global.generate_globe_json_string(
deaths_gps_data, self.global_population_dataset,
self.global_population))
def merge_course(courses, module):
courses = [i[1] for i in courses]
# credits
details = module['details']
credits = 0
credits_ = [
i for i in details if i['title'] in ["Credit Points", "CP", "Credits"]
]
if len(credits_) > 0:
try:
credits = int(credits_[0]["details"].split(",")[0])
details = [
i for i in details
if not i['title'] in ["Credit Points", "CP", "Credits"]
]
except:
pass
content = {
i["title"]: {k: v
for k, v in i.items() if k != "modules"}
for i in courses
}
return utils.merge_dict(module, {
'content': content,
'details': details,
'credits': credits
})
def get_tucan_page(title_url):
title, url = title_url
# if the url list was stored in a previous session,
# we need to replace the outdated session key in the url with the new one:
soup = tucan_browser.getcached(TUCAN_URL +
url) # tucan_browser / inferno_browser
# print("\n=-=-=-=-=-=-=-= BEGIN",
# "\nwget --no-cookies --header \"Cookie: cnsc="+ inferno_browser.get_cookiejar().get('cnsc') +"\" \"" + TUCAN_URL + url + "\" -O test.html",
# "\n" + re.sub("[ ]+", " ", soup.text),
# "\n=-=-=-=-=-=-=-= END")
blame = utils.blame
dates = blame("no dates for '" + title + "'",
lambda: extract_tucan_dates(soup)) or []
uedates = blame("no uedates for '" + title + "'",
lambda: extract_tucan_uedates(soup, title)) or []
details = blame("no details for '" + title + "'",
lambda: extract_tucan_details(soup)) or {}
modules = blame("no modules for '" + title + "'",
lambda: extract_tucan_course_modules(soup)) or []
return utils.merge_dict(details, {
'title': title,
'dates': dates,
'uedates': uedates,
'modules': modules
}) # 'link':url,
def put_review_annotation(annotation_id):
data = request.get_json()
basket = data['basket']
try:
annotation = Annotation.objects().get(id=annotation_id)
except Exception:
return Response('404', status=404)
try:
annotation_review = AnnotationReview.objects().get(annotation=annotation, user=g.user)
except AnnotationReview.DoesNotExist:
annotation_review = AnnotationReview(annotation=annotation, user=g.user)
review_basket = dict()
for key in basket:
if '-review' in key:
review_basket[key] = basket[key]
annotation_review.doc = annotation.doc
annotation_review.ip = request.remote_addr
annotation_review.basket = review_basket
annotation_review.updated_at = datetime.datetime.now
annotation_review.save()
annotation.basket = utils.merge_dict(annotation.basket, review_basket)
return json.dumps({
'annotation': annotation.dump(),
})
def get_candle(df, up_color=None, down_color=None, theme=None, **kwargs):
ohlc = ["open", "high", "low", "close"]
if not theme:
theme = auth.get_config_file()["theme"]
layout = getLayout(theme=theme)
c_dir = _ohlc_dict(df)
args = [df[c_dir[_]] for _ in ohlc]
args.append(df.index)
fig = py.plotly.tools.FigureFactory.create_candlestick(*args, **kwargs)
candle = Figure()
candle["data"] = fig["data"]
candle["layout"] = layout
data = candle["data"]
def update_color(n, color):
data[n]["fillcolor"] = normalize(color)
data[n]["line"].update(color=normalize(color))
if up_color:
update_color(0, up_color)
if down_color:
update_color(1, down_color)
candle["layout"]["hovermode"] = "closest"
layout = getLayout(theme=theme)
candle["layout"] = merge_dict(layout, candle["layout"])
return candle
def test_gender_allowed_validation(self):
data = utils.merge_dict(valid_material_params(), {'gender': 'invalid'})
r, status = self.post('/materials', data=data)
self.assertValidationErrorStatus(status)
self.assertValidationError(r, {'gender': 'unallowed value invalid'})
def test_verify_ownership_422_missing_owner_id(self):
materials_data = utils.merge_dict(valid_material_params(), {'owner_id': 'abc'})
r, _ = self.post('/materials', data=materials_data)
data = {'materials': [r['_id']]}
r, status = self.post('/materials/verify_ownership', data=data)
self.assert422(status)
def test_verify_ownership_materials_dont_belong_to_ownerid(self):
abc_materials_data = utils.merge_dict(valid_material_params(), {'owner_id': 'abc'})
xyz_materials_data = utils.merge_dict(valid_material_params(), {'owner_id': 'xyz'})
r1, _ = self.post('/materials', data=abc_materials_data)
r2, _ = self.post('/materials', data=abc_materials_data)
r3, _ = self.post('/materials', data=xyz_materials_data)
abc_materials = [r1['_id'], r2['_id']]
xyz_materials = [r3['_id']]
data = {'owner_id': 'xyz', 'materials': abc_materials + xyz_materials}
r, status = self.post('/materials/verify_ownership', data=data)
self.assert403(status)
self.assertEqual(len(r['_issues']), 2)
def test_meta_allows_unknown(self):
data = utils.merge_dict(valid_material_params(), {'meta': {'allows': 'unknown'}})
r, status = self.post('/materials', data=data)
self.assert201(status)
r, status = self.get('materials', '', r['_id'])
self.assertEqual(r['meta']['allows'], 'unknown')
def extract_mentions(path, trim=True):
"""
"""
entities = merge_dict(path, trim=trim)
mentions = list(entities.keys())
print(len(mentions))
write_to_file("mention_list.txt", mentions)
pass
def test_submitter_id_is_set(self):
submitter_id = '*****@*****.**'
materials_data = utils.merge_dict(valid_material_params(), {'submitter_id': submitter_id})
r, status = self.post('/materials/', data=materials_data)
self.assert201(status)
r, status = self.get('materials', '', r['_id'])
self.assertEqual(r['submitter_id'], submitter_id)
def __get_count_people_by_role_name():
count_people_by_role_name = dict()
for person in People.people:
count_people_by_role_name = merge_dict(
count_people_by_role_name,
person.get_count_people_by_role_name())
return count_people_by_role_name
def get_inferno_page(module_id):
soup = state.inferno_br.getcached(INFERNO_PREFIX + module_id + "?lang=de")
details = extract_inferno_module(soup) or {}
# TODO get title
regulations = [i['details']
for i in details['details']
if i['title'] == "Studiengangsordnungen"]
regulations = regulations[0] if regulations else []
return utils.merge_dict(details, {'module_id':module_id, 'regulations':regulations})
def test_verify_ownership_materials_belong_to_owner_id(self):
materials_data = utils.merge_dict(valid_material_params(), {'owner_id': 'abc'})
r1, _ = self.post('/materials', data=materials_data)
r2, _ = self.post('/materials', data=materials_data)
r3, _ = self.post('/materials', data=materials_data)
data = {'owner_id': 'abc', 'materials': [r1['_id'], r2['_id'], r3['_id']]}
r, status = self.post('/materials/verify_ownership', data=data)
self.assert200(status)
def render(self, template_name, root=None):
# 添加方法到模板
kwargs = merge_dict(root, all_funcs)
# 移除self
if kwargs.has_key('self'):
del kwargs['self']
# 生成html
html = self.render_string(template_name, **kwargs)
html += self.request_time_info()
self.write(html)
return html
def render(self, template_name, root=None):
# 添加方法到模板
kwargs = merge_dict(root, all_funcs)
# 移除self
if kwargs.has_key('self'):
del kwargs['self']
# 生成html
html = self.render_string(template_name, **kwargs)
html += self.request_time_info()
self.write(html)
return html
def __add__(self, other):
if not isinstance(other, Turn):
raise TypeError(f"unsupported operand type: {type(other)}")
return Turn(
text=self.text + " " + other.text,
speaker=self.speaker,
goal=self.goal | other.goal,
state=self.state | other.state,
asr={k: self.asr.get(k, 0) + other.asr.get(k, 0)
for k in set(self.asr.keys()) | set(other.asr.keys())},
meta=utils.merge_dict(self.meta, other.meta)
)
def __init__(self, name='Player 1', configuration={}):
self.playfield = list([[None] * 10 for i in range(40)])
self.configuration = merge_dict(default_configuration, configuration)
self.controller = Controller(self)
self.drop = Drop(self)
self.hold = Hold(self)
self.calc = ScoreCalc(self)
self.last_clear = []
self.name = name
self.opponent = None
self.garbage_amount = 0
self.finished = False
self.statistics = Statistics(self)
def toJSON(self):
d = utils.merge_dict(vars(self), None)
d["dst_data"]["begin_date"] = utils.x10_year_day_to_string(self.dst_data["begin_year_day"])
d["dst_data"]["end_date"] = utils.x10_year_day_to_string(self.dst_data["end_year_day"])
def processDawnDusk(l):
da = {"rise": utils.x10_time_to_string(l["start_double_hour"], l["start_min"]), \
"set": utils.x10_time_to_string(l["stop_double_hour"], l["stop_min"])}
utils.merge_dict_if_not_default(l, da, "reserved_1", 0)
utils.merge_dict_if_not_default(l, da, "reserved_2", 0)
return da
sunrise_sunset_list = [processDawnDusk(l) for l in d["sunrise_sunset_times"]]
d["sunrise_sunset_times"] = sunrise_sunset_list
return d
def toJSON(self):
l = utils.merge_dict(vars(self), None, "delay_secs", "id")
utils.convert_key_to_hex_string(l, "id")
# l["reserved"] = utils.to_binary_string(self.reserved_1)
elements = l["elements"] = []
for d in self.elements:
units = d["unit_bitmap_hi"], d["unit_bitmap_lo"]
d["units"] = [k for k,v in codec.DEVICECODE_MASK.decode(units).iteritems() if v == 1]
del d["unit_bitmap_lo"]
del d["unit_bitmap_hi"]
d["function_code"] = codec.FUNCTIONS_ZERO_OFFSET[d["function_code"]]
d["house_code"] = codec.VALUE_TO_HOUSECODE_MAP[d["house_code"]]
elements.append(d)
return l
def toJSON(self):
t = vars(self)
tijson = utils.merge_dict(t, None, "start_macro_ptr", "stop_macro_ptr", "start_macro_id", "stop_macro_id")
utils.convert_key_to_hex_string(tijson, "start_macro_ptr", "stop_macro_ptr", "start_macro_id", "stop_macro_id")
if utils.merge_dict_if_not_default(t, tijson, "begin_year_day", 0):
tijson["begin_date"] = utils.x10_year_day_to_string(t["begin_year_day"])
if utils.merge_dict_if_not_default(t, tijson, "end_year_day", 367):
tijson["end_date"] = utils.x10_year_day_to_string(t["end_year_day"])
utils.merge_dict_if_not_default(t, tijson, "start_security", 0)
utils.merge_dict_if_not_default(t, tijson, "stop_security", 0)
tijson["start_time"] = utils.x10_time_to_string(t["start_double_hour"], t["start_min"])
tijson["stop_time"] = utils.x10_time_to_string(t["stop_double_hour"], t["stop_min"])
tijson["week_day_mask"] = utils.week_mask_to_string(t["week_day_mask"])
return tijson
def test_bulk_search_materials(self):
query = {'owner_id': 'abc'}
query_empty = {'owner_id': 'xyz'}
abc_materials_data = utils.merge_dict(valid_material_params(), query)
r1, _ = self.post('/materials', data=abc_materials_data)
r, status = self.post('/materials/search', data={'where': query})
self.assert200(status)
self.assertEqual(len(r['_items']), 1)
r, status = self.post('/materials/search', data={'where': query_empty})
self.assert200(status)
self.assertEqual(len(r['_items']), 0)
def active_threads(self, query_params={}):
if not self.course_id:
raise utils.CommentClientRequestError("Must provide course_id when retrieving active threads for the user")
url = _url_for_user_active_threads(self.id)
params = {'course_id': text_type(self.course_id)}
params = utils.merge_dict(params, query_params)
response = utils.perform_request(
'get',
url,
params,
metric_action='user.active_threads',
metric_tags=self._metric_tags,
paged_results=True,
)
return response.get('collection', []), response.get('page', 1), response.get('num_pages', 1)
def test_merge_gps_records(self):
csse_handler = self.test_parse_header()
lhs_data = self.test_parse_data()
second_data_array = "Province,Country,10,5,1,5".split(",")
gps_key, parsed_daily_data = csse_handler.parse_data_line(
second_data_array)
rhs_data = dict()
rhs_data[gps_key] = parsed_daily_data
merged_records = utils.merge_dict(lhs_data, rhs_data)
first_data = merged_records.get("0,80,Country - Province")
self.assertIsNotNone(first_data)
self.assertEqual(first_data, lhs_data.get("0,80,Country - Province"))
second_data = merged_records.get("10,5,Country - Province")
self.assertIsNotNone(second_data)
self.assertEqual(second_data, rhs_data.get("10,5,Country - Province"))
def restore_attr(self,
attr_name: str,
alt_name: str = None,
convert: Sequence[Tuple[type, Callable]] = None,
max_depth: int = 0,
filter_items: Sequence[str] = None,
filter_keys: Sequence[str] = None):
"""
Restore the specified attribute from disk, if any saved state exists.
Arguments:
attr_name: The name of the attribute to restore, as per `getattr(self, attr_name)`.
convert: Optional Sequence of (type, Callable) tuples. If any loaded item matches a type in the list,
return the result of Callable(item) instead of the item itself.
max_depth: The max depth the attribute was saved with as passed to :meth:`save_state`, representing the
lowest depth at which dict items are split into subdirectories.
filter_items: If not None, only these lowest-depth dict items will be restored and others will be ignored.
filter_keys: If not None, only these top-level dict keys will be restored and other will be ignored.
"""
load_name = alt_name if alt_name else attr_name
values = utils.io.load_split(load_name,
config['state_path'],
convert=convert,
max_depth=max_depth,
filter_items=filter_items,
filter_keys=filter_keys)
if values is not None:
orig = getattr(self, attr_name)
if isinstance(orig, dict):
orig = utils.merge_dict(orig, values)
setattr(self, attr_name, orig)
else:
setattr(self, attr_name, values)
self.log.debug("Restored '{}' from file(s).", attr_name)
self.log.debug("Restored '{}' value:\n{}",
attr_name,
values,
verbosity=2)
else:
self.log.debug("No state for '{}' found in file(s).",
attr_name,
verbosity=1)
def args(self):
""" Get arguments of the Sqoop command for this task as dictionary"""
basic_args = {
'connect': self._arg_jdbc_url(),
'username': self.task.source['user'],
'table': self.table.name,
'hive-import': True,
'map-column-hive': self._arg_column_map('hive'),
'map-column-java': self._arg_column_map('java'),
'hive-table': self._arg_hive_tablename()
}
if 'password' in self.task.source:
basic_args['password'] = "******" + self.task.source['password'] + "'"
else:
basic_args['password-file'] = self.task.source['password_file']
custom_args = self.task.sqoop_args(self.table)
return merge_dict(custom_args, basic_args)
def recognize_sentences(corpus,
keywords_path,
mode,
trim=True,
label=False,
output=None,
thread=None,
limit=None):
"""
Arguments:
corpus(str): Path to the corpus file.
keywords_path(str): Path to where keywords dictionaries are.
thread(int): Number of thread to process.
output(str): Path to the output file.
"""
# output name
if output is None:
output = corpus[:-4] + "_sentence.tsv"
# Decompose corpus to sentences and each as one datum
# Load corpus (skip first line for PubMed smaller version)
raw_data = readlines(corpus, begin=1, limit=limit)
# Load tokenizer
tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')
# Threading
param = (tokenizer, )
context = generic_threading(thread // 2, raw_data, parallel_split, param)
context = list(chain.from_iterable(context))
del raw_data
print()
print("Recognize mentions in sentences (mode: {:s})".format(mode))
# Load all mentions
entity = merge_dict(keywords_path, trim=trim)
# Threading
keywords = list(entity.keys())
param = (keywords, mode)
result = generic_threading(thread, context, keyword_in_sentences, param)
# write all result to file
write_to_file(output, result)
def json2sql(jfile,dir_name='../data/nvd.db'):
"""
解析json，入库
"""
with codecs.open(jfile,'r',encoding='utf-8') as f:
fjson=json.load(f)
so=SQLite(dir_name)
ct=dict2createdb()
so.execute(ct)
j=0
cve={}
all_content={}
for i in dict_generator(fjson):
tp='.'.join(i[0:-1]).replace('.','_')
# 过滤json文件头
if tp in ['CVE_data_type','CVE_data_format','CVE_data_version','CVE_data_numberOfCVEs','CVE_data_timestamp']:
continue
value=i[-1]
# 合并json中的列表数据结构
cve_now={}
cve_now[tp]=str(value)
cve=merge_dict(cve_now,cve)
# 一条完整的CVE数据
if tp =='CVE_Items_lastModifiedDate':
cve=padding(cve,cve_tags)
sql=d2sql(cve)
if sql:
j=j+1
all_content[j]=tuple(cve.values())
cve={}
# 批量插入sqlite3
try:
so.executemany(sql,all_content.values())
print('[INSERT] number:%d' %j)
except Exception as e:
logging.error("[sql]: %s %s" % (sql, str(e)))
return True
def args(self):
""" Get arguments of the Sqoop command for this task as dictionary"""
basic_args = {
'connect': self._arg_jdbc_url(),
'username': self.task.source['user'],
'table': self.table.name,
'hive-import': True,
'map-column-hive': self._arg_column_map('hive'),
'map-column-java': self._arg_column_map('java'),
'hive-table': self._arg_hive_tablename()
}
if 'password' in self.task.source:
basic_args['password'] = "******" + self.task.source['password'] + "'"
else:
basic_args['password-file'] = self.task.source['password_file']
custom_args = self.task.sqoop_args(self.table)
return merge_dict(custom_args, basic_args)
def process_confirmed(self):
"""
Processes the global confirmed in-memory records
"""
logger = logging.getLogger("Confirmed")
csse_handler_global = CSSEGISandData(logger, USFileType=False)
global_confirmed_gps_data = csse_handler_global.parse_csv_file_contents(
self.global_confirmed_dataset)
self.date_keys_sanity_check(csse_handler_global.date_keys)
csse_handler_us = CSSEGISandData(logger, USFileType=True)
us_confirmed_gps_data = csse_handler_us.parse_csv_file_contents(
self.us_confirmed_dataset)
self.date_keys_sanity_check(csse_handler_us.date_keys)
confirmed_gps_data = utils.merge_dict(global_confirmed_gps_data,
us_confirmed_gps_data)
utils.write_to_file(
"data/confirmed.json",
csse_handler_global.generate_globe_json_string(
confirmed_gps_data, self.global_population_dataset,
self.global_population))
def get_ohlc(df, up_color=None, down_color=None, theme=None, **kwargs):
ohlc = ["open", "high", "low", "close"]
if not theme:
theme = auth.get_config_file()["theme"]
c_dir = _ohlc_dict(df)
args = [df[c_dir[_]] for _ in ohlc]
args.append(df.index)
fig = py.plotly.tools.FigureFactory.create_ohlc(*args, **kwargs)
ohlc_bars = Figure()
ohlc_bars["data"] = fig["data"]
ohlc_bars["layout"] = fig["layout"]
data = ohlc_bars["data"]
if up_color:
data[0]["line"].update(color=normalize(up_color))
if down_color:
data[1]["line"].update(color=normalize(down_color))
ohlc_bars["layout"]["hovermode"] = "closest"
layout = getLayout(theme=theme)
ohlc_bars["layout"] = merge_dict(layout, ohlc_bars["layout"])
return ohlc_bars
def subscribed_threads(self, query_params={}):
if not self.course_id:
raise utils.CommentClientRequestError("Must provide course_id when retrieving subscribed threads for the user")
url = _url_for_user_subscribed_threads(self.id)
params = {'course_id': text_type(self.course_id)}
params = utils.merge_dict(params, query_params)
response = utils.perform_request(
'get',
url,
params,
metric_action='user.subscribed_threads',
metric_tags=self._metric_tags,
paged_results=True
)
return utils.CommentClientPaginatedResult(
collection=response.get('collection', []),
page=response.get('page', 1),
num_pages=response.get('num_pages', 1),
thread_count=response.get('thread_count', 0)
)
def get_review_annotation(user_id, doc_id):
try:
doc = Doc.objects.get(id=doc_id)
user = User.objects.get(id=user_id)
except Exception as e:
return redirect('/404')
annotations = Annotation.objects(doc=doc, user=user)
data = []
for annotation in annotations:
try:
annotation_review = AnnotationReview.objects().get(annotation=annotation, user=g.user)
annotation.basket = utils.merge_dict(annotation.basket, annotation_review.basket)
except AnnotationReview.DoesNotExist:
pass
data.append(annotation.dump())
return json.dumps({
'annotations': data,
})
def __init__(self,
x_train,
y_train,
discrete_feature_val,
cont_col,
unrankable,
auto_discrete=True,
allocation_book=None):
"""
Features types: discrete rankable; discrete unrankable; continuous
unrankable should be a list specifying the columns for discrete unrankable features
x_train does not need to be categorized before passing in
if auto_discrete is False, you should pass in allocation_book whose keys are cont columns and values are num of classes
"""
self.x_train = x_train # x_train is numpy 2d array
self.y_train = y_train # y_train is numpy 1d array
self.cont_col = cont_col
self.unrankable = unrankable # a list
self.auto_discrete = auto_discrete
self.allocation_book = allocation_book
if self.unrankable != None: # get the indices of features used in comonotonicity
self.como = list(
set([i for i in range(len(x_train[0]))]) - set(unrankable))
else:
self.como = [i for i in range(len(x_train[0]))]
cont_feature_val = {}
for i in range(len(x_train.T)):
if i in cont_col:
if auto_discrete == True:
cont_feature_val[i] = 8
else: # in this way, use the allocation_book to discretize
cont_feature_val[i] = allocation_book[i]
if discrete_feature_val != None:
self.feature_val = utils.merge_dict(cont_feature_val,
discrete_feature_val)
else:
self.feature_val = cont_feature_val.copy()
def train(self, dataloader: td.DataLoader, val_dataloader: td.DataLoader
) -> Record:
stats = None
for eidx in range(1, self.num_epochs + 1):
stats = self(dataloader)
stats["epoch"] = torch.tensor(eidx)
if eidx % self.validate_every == 0:
with torch.no_grad():
fval_stats = self.fin_valid(val_dataloader)
with torch.no_grad():
samples, gval_stats = \
self.gen_valid(val_dataloader.dataset.data)
sample = random.choice(samples)
self.log_dialog("gval-input-sample", sample.input)
self.log_dialog("gval-gen-sample", sample.output)
with torch.no_grad():
samples, _ = self.gen_valid(dataloader.dataset.data, 3)
for i, sample in enumerate(samples, 1):
self.log_dialog(f"gtrain-input-sample-{i}", sample.input)
self.log_dialog(f"gtrain-gen-sample-{i}", sample.output)
val_stats = utils.merge_dict(fval_stats, gval_stats)
self.log_stats(f"e{eidx}-val-summary", val_stats, prefix="val")
stats.update({f"val-{k}": v for k, v in val_stats.items()})
if self.early_stop and self.check_early_stop(stats):
break
if self.save_every is not None and eidx % self.save_every == 0:
self.save_snapshot(self.state_dict(), f"e{eidx}")
if stats is not None and self._best_record is None:
self.make_record(stats)
if self.early_stop and self._best_record is not None:
self.model.load_state_dict(self._best_record.state_dict)
if self._best_record is not None:
self._logger.info(f"final summary: "
f"{pprint.pformat(self._best_record.to_json())}")
self.save_snapshot(self.state_dict(), "final")
return self._best_record
def update(self, values):
self.status = utils.merge_dict(self.status, values)
self.handler.router.notify('status', values)
def set_listitem_details(self, cur_listitem, content_type, prefix):
'''set the window properties based on the current listitem'''
self.bgtasks += 1
try:
if cur_listitem in self.listitem_details:
# data already in memory
all_props = self.listitem_details[cur_listitem]
else:
# prefer listitem's contenttype over container's contenttype
dbtype = xbmc.getInfoLabel("%sListItem.DBTYPE" % prefix)
if not dbtype:
dbtype = xbmc.getInfoLabel("%sListItem.Property(DBTYPE)" % prefix)
if dbtype:
content_type = dbtype + "s"
# collect all details from listitem
details = self.get_listitem_details(content_type, prefix)
if prefix and cur_listitem == self.last_listitem:
# for widgets we immediately set all normal properties as window prop
self.set_win_props(prepare_win_props(details))
# if another lookup for the same listitem already in progress... wait for it to complete
while self.lookup_busy.get(cur_listitem):
xbmc.sleep(250)
if self.exit:
return
self.lookup_busy[cur_listitem] = True
# music content
if content_type in ["albums", "artists", "songs"] and self.enable_musicart:
details = extend_dict(details, self.metadatautils.get_music_artwork(
details["artist"], details["album"], details["title"], details["discnumber"]))
# moviesets
elif details["path"].startswith("videodb://movies/sets/") and details["dbid"]:
details = extend_dict(
details, self.metadatautils.get_moviesetdetails(
details["title"], details["dbid"]))
content_type = "sets"
# video content
elif content_type in ["movies", "setmovies", "tvshows", "seasons", "episodes", "musicvideos"]:
# get imdb and tvdbid
details["imdbnumber"], tvdbid = self.metadatautils.get_imdbtvdb_id(
details["title"], content_type,
details["year"], details["imdbnumber"], details["tvshowtitle"])
# generic video properties (studio, streamdetails, omdb, top250)
details = merge_dict(details,
self.get_directors_writers(details["director"], details["writer"]))
if self.enable_extrafanart:
if not details["filenameandpath"]:
details["filenameandpath"] = details["path"]
if "videodb://" not in details["filenameandpath"]:
details = merge_dict(details,
self.metadatautils.get_extrafanart(details["filenameandpath"]))
details = merge_dict(details, self.metadatautils.get_duration(details["duration"]))
details = merge_dict(details, self.get_genres(details["genre"]))
details = merge_dict(details, self.metadatautils.get_studio_logo(details["studio"]))
details = merge_dict(details, self.metadatautils.get_omdb_info(details["imdbnumber"]))
details = merge_dict(details, self.get_streamdetails(details["dbid"], details["path"], content_type))
if self.exit:
return
details = merge_dict(details, self.metadatautils.get_top250_rating(details["imdbnumber"]))
if self.exit:
return
# tvshows-only properties (tvdb)
if content_type in ["tvshows", "seasons", "episodes"]:
details = merge_dict(details, self.metadatautils.get_tvdb_details(details["imdbnumber"], tvdbid))
if self.exit:
return
# movies-only properties (tmdb, animated art)
if content_type in ["movies", "setmovies"]:
details = merge_dict(details, self.metadatautils.get_tmdb_details(details["imdbnumber"]))
if details["imdbnumber"] and self.enable_animatedart:
details = extend_dict(
details, self.metadatautils.get_animated_artwork(
details["imdbnumber"]))
if self.exit:
return
# extended art
if self.enable_extendedart:
tmdbid = details.get("tmdb_id", "")
details = extend_dict(
details, self.metadatautils.get_extended_artwork(
details["imdbnumber"], tvdbid, tmdbid, content_type), [
"posters", "clearlogos", "banners"])
if self.exit:
return
# monitor listitem props when PVR is active
elif content_type in ["tvchannels", "tvrecordings", "channels", "recordings", "timers", "tvtimers"]:
details = self.get_pvr_artwork(details, prefix)
# process all properties
all_props = prepare_win_props(details)
if content_type not in ["weathers", "systeminfos", "sets"]:
self.listitem_details[cur_listitem] = all_props
self.lookup_busy.pop(cur_listitem, None)
if cur_listitem == self.last_listitem:
self.set_win_props(all_props)
except Exception as exc:
log_exception(__name__, exc)
self.bgtasks -= 1
def sqoop_args(self, table):
return merge_dict(
self._config.get('sqoop_args', {}).get(table.name, {}),
self._global_config.get('sqoop_args', {}))
