def test_0():
countries_metadata = get_metadata()
nm = get_hs_db_name(countries_metadata, 161)
ts_dir = TRAINING_SETS_MXNET_DIR + nm + '/'
feat = load_features(ts_dir, 0)
net = get_nn_for_training(1024)
print('TEST: ' + str(net(feat).shape))
def user_window_0():
qd = QDialog()
qd.setWindowTitle(SOFTWARE_NAME + ' ' + SOFTWARE_VERSION)
qd.setGeometry(10, 10, WINDOW_WIDTH, WINDOW_HEIGHT)
qd.countries_metadata = get_metadata()
cn = countries_names(qd.countries_metadata)
cs = country_selector_0(cn, init_index=161)
qd.countries_combo = cs[1][0]
csp = grid_group_0(cs, 'Country selector')
dp = description_panel_0()
qd.query_edit = dp[1][0]
dpp = grid_group_0(dp, 'Commodity description')
sp = search_panel_0()
qd.search_button = sp[0][0]
qd.clear_query_button = sp[0][1]
qd.clear_results_button = sp[0][2]
spp = grid_group_0(sp, 'Search tools')
rp = result_panel_0()
qd.results_label = rp[0][0]
qd.table = rp[1][0]
rpp = grid_group_0(rp, 'Results')
widgets_0 = [[csp, teggs_logo_0()], [dpp], [spp]]
group_0 = grid_group_0(widgets_0)
layout = QGridLayout()
layout.addWidget(group_0, 0, 0)
layout.addWidget(rpp, 1, 0)
qd.setLayout(layout)
qd.query_edit.setFocus()
qd.search_button.clicked.connect(partial(search_0, qd))
qd.clear_query_button.clicked.connect(partial(clear_description, qd))
qd.clear_results_button.clicked.connect(partial(clear_results, qd))
return qd
def create_corpus():
md = get_metadata()
countries_indices = select_databases_for_corpus(md)
corp_pfn = HG_HS_CORPUS_DIR + CORPUS_FILENAME
safely_remove_file(corp_pfn)
for i in countries_indices:
corp = linesep + get_corpus_single_db(md, i)
append_str_to_txt_file(corp_pfn, corp)
if VERBOSITY > 0:
print('Corpus has been generated.')
def select_hscodes():
hsc = []
m = get_metadata()
dbn = get_database_filename(m, COUNTRY_INDEX)
n = get_number_of_records(dbn)
indices = [i for i in range(n)]
shuffle(indices)
indices = indices[0:NUMBER_OF_PAIRS]
for i in indices:
hsc.append(get_hs_code(dbn, i))
df = pd.DataFrame(hsc)
save(df, "~/Documents/hscodes_for_evaluation.json")
def select_records():
records = []
m = get_metadata()
db_fn = get_database_filename(m, COUNTRY_INDEX)
sdf = get_dataframe(db_fn)
n = get_number_of_records(db_fn)
indices = [i for i in range(n)]
shuffle(indices)
indices = indices[0:NUMBER_OF_RECORDS]
for i in indices:
records.append(sdf.iloc[[i]])
df = pd.concat(records, ignore_index=True)
df.to_excel("~/Documents/experimental_database.xlsx")
def test(rec_index):
country_index = 37
md = get_metadata()
dbf = get_database_filename(md, country_index)
desc = get_hs_description(dbf, rec_index)
print(desc)
from hsr.dev_tools.training_tools_0 import training_tool_1
from hsr.evaluation.evaluator_0 import evaluate_1, alpha_score, epsilon_score
from hsr.hsdb.metadata import get_metadata
from hsr.preprocessing.preprocessors import batch_preprocess_step_0, \
batch_preprocess_step_1
from hsr.user_interface.data_formatting import float_to_percent_str
# COUNTRIES_INDICES = [37, 144, 161, 188] # [China, Poland, Singapore, USA]
COUNTRIES_INDICES = [161] # [Singapore]
PREPROCESSING_STEP_0 = 0
PREPROCESSING_STEP_1 = 0
TRAINING = 1
countries_metadata = get_metadata()
if PREPROCESSING_STEP_0:
batch_preprocess_step_0(countries_metadata, COUNTRIES_INDICES)
if PREPROCESSING_STEP_1:
batch_preprocess_step_1(countries_metadata, COUNTRIES_INDICES)
if TRAINING:
training_tool_1(countries_metadata, COUNTRIES_INDICES)
alpha = float_to_percent_str(evaluate_1(COUNTRIES_INDICES, alpha_score))
epsilon = float_to_percent_str(evaluate_1(COUNTRIES_INDICES, epsilon_score))
print("alpha-score: " + alpha + "%")
print("epsilon-score: " + epsilon + "%")
def search_hs_codes(query, country_index):
hs_search(get_metadata(), query, country_index)
return mean_score
def evaluate_0(country_index, evaluation_function):
(nr, _) = EVAL_SET.shape
return sum(scores(country_index, evaluation_function)) / nr
def read_eval_set():
return pd.read_excel(EVAL_SET_FILENAME,
header=EVAL_SET_HEADER,
dtype='object')
EVAL_SET = read_eval_set()
METADATA = get_metadata()
def hs_search_e(query, country_index):
return hs_search(METADATA, query, country_index, return_results=True)
def search_results(eval_set_record_index, country_index):
return hs_search_e(EVAL_SET.iat[eval_set_record_index, 0], country_index)
def hs_compare(x, y):
x = x[0:LENGTH_OF_HSCODES_COMMON_PART]
y = y[0:LENGTH_OF_HSCODES_COMMON_PART]
return x == y