def test_causality(): # test Granger- and instantaneous causality
if debug_mode:
if "causality" not in to_test:
return
else:
print("\n\nCAUSALITY", end="")
for ds in datasets:
for dt in dt_s_list:
if debug_mode:
print("\n" + dt_s_tup_to_string(dt) + ": ", end="")
err_msg_g_p = build_err_msg(ds, dt, "GRANGER CAUS. - p-VALUE")
err_msg_g_t = build_err_msg(ds, dt, "GRANGER CAUS. - TEST STAT.")
err_msg_i_p = build_err_msg(ds, dt, "INSTANT. CAUS. - p-VALUE")
err_msg_i_t = build_err_msg(ds, dt, "INSTANT. CAUS. - TEST STAT.")
v_ind = range(len(ds.variable_names))
for causing_ind in sublists(v_ind, 1, len(v_ind)-1):
causing_names = ["y" + str(i+1) for i in causing_ind]
causing_key = tuple(ds.variable_names[i] for i in causing_ind)
caused_ind = [i for i in v_ind if i not in causing_ind]
caused_names = ["y" + str(i+1) for i in caused_ind]
caused_key = tuple(ds.variable_names[i] for i in caused_ind)
# test Granger-causality ######################################
granger_sm_ind = results_sm[ds][
dt].test_causality(caused_ind, causing_ind)
granger_sm_str = results_sm[ds][
dt].test_causality(caused_names, causing_names)
# test test-statistic for Granger non-causality:
g_t_obt = granger_sm_ind.test_statistic
g_t_des = results_ref[ds][dt]["granger_caus"][
"test_stat"][(causing_key, caused_key)]
yield assert_allclose, g_t_obt, g_t_des, rtol, atol, \
False, err_msg_g_t
# check whether string sequences as args work in the same way:
g_t_obt_str = granger_sm_str.test_statistic
yield assert_allclose, g_t_obt_str, g_t_obt, 1e-07, 0, False, \
err_msg_g_t + " - sequences of integers and ".upper() + \
"strings as arguments don't yield the same result!".upper()
# check if int (e.g. 0) as index and list of int ([0]) yield
# the same result:
if len(causing_ind) == 1 or len(caused_ind) == 1:
ci = causing_ind[0] if len(causing_ind)==1 else causing_ind
ce = caused_ind[0] if len(caused_ind) == 1 else caused_ind
granger_sm_single_ind = results_sm[ds][
dt].test_causality(ce, ci)
g_t_obt_single = granger_sm_single_ind.test_statistic
yield assert_allclose, g_t_obt_single, g_t_obt, 1e-07, 0, \
False, \
err_msg_g_t + " - list of int and int as ".upper() + \
"argument don't yield the same result!".upper()
# test p-value for Granger non-causality:
g_p_obt = granger_sm_ind.pvalue
g_p_des = results_ref[ds][dt]["granger_caus"]["p"][(
causing_key, caused_key)]
yield assert_allclose, g_p_obt, g_p_des, rtol, atol, \
False, err_msg_g_p
# check whether string sequences as args work in the same way:
g_p_obt_str = granger_sm_str.pvalue
yield assert_allclose, g_p_obt_str, g_p_obt, 1e-07, 0, False, \
err_msg_g_t + " - sequences of integers and ".upper() + \
"strings as arguments don't yield the same result!".upper()
# check if int (e.g. 0) as index and list of int ([0]) yield
# the same result:
if len(causing_ind) == 1:
g_p_obt_single = granger_sm_single_ind.pvalue
yield assert_allclose, g_p_obt_single, g_p_obt, 1e-07, 0, \
False, \
err_msg_g_t + " - list of int and int as ".upper() + \
"argument don't yield the same result!".upper()
# test instantaneous causality ################################
inst_sm_ind = results_sm[ds][dt].test_inst_causality(
causing_ind)
inst_sm_str = results_sm[ds][dt].test_inst_causality(
causing_names)
# test test-statistic for instantaneous non-causality
t_obt = inst_sm_ind.test_statistic
t_des = results_ref[ds][dt]["inst_caus"][
"test_stat"][(causing_key, caused_key)]
yield assert_allclose, t_obt, t_des, rtol, atol, False, \
err_msg_i_t
# check whether string sequences as args work in the same way:
t_obt_str = inst_sm_str.test_statistic
yield assert_allclose, t_obt_str, t_obt, 1e-07, 0, False, \
err_msg_i_t + " - sequences of integers and ".upper() + \
"strings as arguments don't yield the same result!".upper()
# check if int (e.g. 0) as index and list of int ([0]) yield
# the same result:
if len(causing_ind) == 1:
inst_sm_single_ind = results_sm[ds][
dt].test_inst_causality(causing_ind[0])
t_obt_single = inst_sm_single_ind.test_statistic
yield assert_allclose, t_obt_single, t_obt, 1e-07, 0, \
False, \
err_msg_i_t + " - list of int and int as ".upper() + \
"argument don't yield the same result!".upper()
# test p-value for instantaneous non-causality
p_obt = results_sm[ds][dt].test_inst_causality(
causing_ind).pvalue
p_des = results_ref[ds][dt]["inst_caus"]["p"][(
causing_key, caused_key)]
yield assert_allclose, p_obt, p_des, rtol, atol, False, \
err_msg_i_p
# check whether string sequences as args work in the same way:
p_obt_str = inst_sm_str.pvalue
yield assert_allclose, p_obt_str, p_obt, 1e-07, 0, False, \
err_msg_i_p + " - sequences of integers and ".upper() + \
"strings as arguments don't yield the same result!".upper()
# check if int (e.g. 0) as index and list of int ([0]) yield
# the same result:
if len(causing_ind) == 1:
inst_sm_single_ind = results_sm[ds][
dt].test_inst_causality(causing_ind[0])
p_obt_single = inst_sm_single_ind.pvalue
yield assert_allclose, p_obt_single, p_obt, 1e-07, 0, \
False, \
err_msg_i_p + " - list of int and int as ".upper() + \
"argument don't yield the same result!".upper()
def test_causality(): # test Granger- and instantaneous causality
if debug_mode:
if "causality" not in to_test:
return
else:
print("\n\nCAUSALITY", end="")
for ds in datasets:
for dt in dt_s_list:
if debug_mode:
print("\n" + dt_s_tup_to_string(dt) + ": ", end="")
err_msg_g_p = build_err_msg(ds, dt, "GRANGER CAUS. - p-VALUE")
err_msg_g_t = build_err_msg(ds, dt, "GRANGER CAUS. - TEST STAT.")
err_msg_i_p = build_err_msg(ds, dt, "INSTANT. CAUS. - p-VALUE")
err_msg_i_t = build_err_msg(ds, dt, "INSTANT. CAUS. - TEST STAT.")
v_ind = range(len(ds.variable_names))
for causing_ind in sublists(v_ind, 1, len(v_ind)-1):
causing_names = ["y" + str(i+1) for i in causing_ind]
causing_key = tuple(ds.variable_names[i] for i in causing_ind)
caused_ind = [i for i in v_ind if i not in causing_ind]
caused_names = ["y" + str(i+1) for i in caused_ind]
caused_key = tuple(ds.variable_names[i] for i in caused_ind)
# test Granger-causality ######################################
granger_sm_ind = results_sm[ds][
dt].test_causality(caused_ind, causing_ind)
granger_sm_str = results_sm[ds][
dt].test_causality(caused_names, causing_names)
# test test-statistic for Granger non-causality:
g_t_obt = granger_sm_ind.test_statistic
g_t_des = results_ref[ds][dt]["granger_caus"][
"test_stat"][(causing_key, caused_key)]
assert_allclose(g_t_obt, g_t_des, rtol, atol, False, err_msg_g_t)
# check whether string sequences as args work in the same way:
g_t_obt_str = granger_sm_str.test_statistic
assert_allclose(g_t_obt_str, g_t_obt, 1e-07, 0, False,
err_msg_g_t + " - sequences of integers and ".upper() +
"strings as arguments don't yield the same result!".upper())
# check if int (e.g. 0) as index and list of int ([0]) yield
# the same result:
if len(causing_ind) == 1 or len(caused_ind) == 1:
ci = causing_ind[0] if len(causing_ind)==1 else causing_ind
ce = caused_ind[0] if len(caused_ind) == 1 else caused_ind
granger_sm_single_ind = results_sm[ds][
dt].test_causality(ce, ci)
g_t_obt_single = granger_sm_single_ind.test_statistic
assert_allclose(g_t_obt_single, g_t_obt, 1e-07, 0, False,
err_msg_g_t + " - list of int and int as ".upper() +
"argument don't yield the same result!".upper())
# test p-value for Granger non-causality:
g_p_obt = granger_sm_ind.pvalue
g_p_des = results_ref[ds][dt]["granger_caus"]["p"][(
causing_key, caused_key)]
assert_allclose(g_p_obt, g_p_des, rtol, atol, False, err_msg_g_p)
# check whether string sequences as args work in the same way:
g_p_obt_str = granger_sm_str.pvalue
assert_allclose(g_p_obt_str, g_p_obt, 1e-07, 0, False,
err_msg_g_t + " - sequences of integers and ".upper() +
"strings as arguments don't yield the same result!".upper())
# check if int (e.g. 0) as index and list of int ([0]) yield
# the same result:
if len(causing_ind) == 1:
g_p_obt_single = granger_sm_single_ind.pvalue
assert_allclose(g_p_obt_single, g_p_obt, 1e-07, 0, False,
err_msg_g_t + " - list of int and int as ".upper() + \
"argument don't yield the same result!".upper())
# test instantaneous causality ################################
inst_sm_ind = results_sm[ds][dt].test_inst_causality(
causing_ind)
inst_sm_str = results_sm[ds][dt].test_inst_causality(
causing_names)
# test test-statistic for instantaneous non-causality
t_obt = inst_sm_ind.test_statistic
t_des = results_ref[ds][dt]["inst_caus"][
"test_stat"][(causing_key, caused_key)]
assert_allclose(t_obt, t_des, rtol, atol, False, err_msg_i_t)
# check whether string sequences as args work in the same way:
t_obt_str = inst_sm_str.test_statistic
assert_allclose(t_obt_str, t_obt, 1e-07, 0, False,
err_msg_i_t + " - sequences of integers and ".upper() +
"strings as arguments don't yield the same result!".upper())
# check if int (e.g. 0) as index and list of int ([0]) yield
# the same result:
if len(causing_ind) == 1:
inst_sm_single_ind = results_sm[ds][
dt].test_inst_causality(causing_ind[0])
t_obt_single = inst_sm_single_ind.test_statistic
assert_allclose(t_obt_single, t_obt, 1e-07, 0, False,
err_msg_i_t + " - list of int and int as ".upper() +
"argument don't yield the same result!".upper())
# test p-value for instantaneous non-causality
p_obt = results_sm[ds][dt].test_inst_causality(
causing_ind).pvalue
p_des = results_ref[ds][dt]["inst_caus"]["p"][(
causing_key, caused_key)]
assert_allclose(p_obt, p_des, rtol, atol, False, err_msg_i_p)
# check whether string sequences as args work in the same way:
p_obt_str = inst_sm_str.pvalue
assert_allclose(p_obt_str, p_obt, 1e-07, 0, False,
err_msg_i_p + " - sequences of integers and ".upper() +
"strings as arguments don't yield the same result!".upper())
# check if int (e.g. 0) as index and list of int ([0]) yield
# the same result:
if len(causing_ind) == 1:
inst_sm_single_ind = results_sm[ds][
dt].test_inst_causality(causing_ind[0])
p_obt_single = inst_sm_single_ind.pvalue
assert_allclose(p_obt_single, p_obt, 1e-07, 0, False,
err_msg_i_p + " - list of int and int as ".upper() +
"argument don't yield the same result!".upper())
def load_results_jmulti(dataset, dt_s_list):
"""
Parameters
----------
dataset : module
A data module in the statsmodels/datasets directory that defines a
__str__() method returning the dataset's name.
dt_s_list : list
A list of strings where each string represents a combination of
deterministic terms.
Returns
-------
result : dict
A dict (keys: tuples of deterministic terms and seasonal terms)
of dicts (keys: strings "est" (for estimators),
"se" (for standard errors),
"t" (for t-values),
"p" (for p-values))
of dicts (keys: strings "alpha", "beta", "Gamma" and other results)
"""
source = "jmulti"
results_dict_per_det_terms = dict.fromkeys(dt_s_list)
for dt_s in dt_s_list:
dt_string = dt_s_tup_to_string(dt_s)
params_file = dataset.__str__(
) + "_" + source + "_" + dt_string + ".txt"
params_file = os.path.join(here, params_file)
# sections in jmulti output:
section_headers = [
"Lagged endogenous term", # parameter matrices
"Deterministic term"
] # c, s, ct
if dt_string == "nc":
del section_headers[-1]
results = dict()
results["est"] = dict.fromkeys(section_headers)
results["se"] = dict.fromkeys(section_headers)
results["t"] = dict.fromkeys(section_headers)
results["p"] = dict.fromkeys(section_headers)
result = []
result_se = []
result_t = []
result_p = []
rows = 0
started_reading_section = False
start_end_mark = "-----"
# ---------------------------------------------------------------------
# parse information about \alpha, \beta, \Gamma, deterministic of VECM
# and A_i and deterministic of corresponding VAR:
section = -1
params_file = open(params_file, encoding='latin_1')
for line in params_file:
if section == -1 and section_headers[section + 1] not in line:
continue
if section < len(section_headers)-1 \
and section_headers[section+1] in line: # new section
section += 1
continue
if not started_reading_section:
if line.startswith(start_end_mark):
started_reading_section = True
continue
if started_reading_section:
if line.startswith(start_end_mark):
if result == []: # no values collected in section "Legend"
started_reading_section = False
continue
results["est"][section_headers[section]] = np.column_stack(
result)
result = []
results["se"][section_headers[section]] = np.column_stack(
result_se)
result_se = []
results["t"][section_headers[section]] = np.column_stack(
result_t)
result_t = []
results["p"][section_headers[section]] = np.column_stack(
result_p)
result_p = []
started_reading_section = False
continue
str_number = r"-?\d+\.\d{3}"
regex_est = re.compile(str_number + r"[^\)\]\}]")
est_col = re.findall(regex_est, line)
# standard errors in parantheses in JMulTi output:
regex_se = re.compile(r"\(" + str_number + r"\)")
se_col = re.findall(regex_se, line)
# t-values in brackets in JMulTi output:
regex_t_value = re.compile(r"\[" + str_number + r"\]")
t_col = re.findall(regex_t_value, line)
# p-values in braces in JMulTi output:
regex_p_value = re.compile(r"\{" + str_number + r"\}")
p_col = re.findall(regex_p_value, line)
if result == [] and est_col != []:
rows = len(est_col)
if est_col != []:
est_col = [float(el) for el in est_col]
result.append(est_col)
elif se_col != []:
for i in range(rows):
se_col[i] = se_col[i].replace("(", "").replace(")", "")
se_col = [float(el) for el in se_col]
result_se.append(se_col)
elif t_col != []:
for i in range(rows):
t_col[i] = t_col[i].replace("[", "").replace("]", "")
t_col = [float(el) for el in t_col]
result_t.append(t_col)
elif p_col != []:
for i in range(rows):
p_col[i] = p_col[i].replace("{", "").replace("}", "")
p_col = [float(el) for el in p_col]
result_p.append(p_col)
params_file.close()
# ---------------------------------------------------------------------
# parse information regarding \Sigma_u
sigmau_file = dataset.__str__() + "_" + source + "_" + dt_string \
+ "_Sigmau" + ".txt"
sigmau_file = os.path.join(here, sigmau_file)
rows_to_parse = 0
# all numbers of Sigma_u in notation with e (e.g. 2.283862e-05)
regex_est = re.compile(r"\s+\S+e\S+")
sigmau_section_reached = False
sigmau_file = open(sigmau_file, encoding='latin_1')
for line in sigmau_file:
if line.startswith("Log Likelihood:"):
line = line[len("Log Likelihood:"):]
results["log_like"] = float(re.findall(regex_est, line)[0])
continue
if not sigmau_section_reached and "Covariance:" not in line:
continue
if "Covariance:" in line:
sigmau_section_reached = True
row = re.findall(regex_est, line)
rows_to_parse = len(row) # Sigma_u quadratic ==> #rows==#cols
sigma_u = np.empty((rows_to_parse, rows_to_parse))
row = re.findall(regex_est, line)
rows_to_parse -= 1
sigma_u[rows_to_parse] = row # rows are added in reverse order...
if rows_to_parse == 0:
break
sigmau_file.close()
results["est"]["Sigma_u"] = sigma_u[::-1] # ...and reversed again here
# ---------------------------------------------------------------------
# parse forecast related output:
fc_file = dataset.__str__() + "_" + source + "_" + dt_string \
+ "_fc5" + ".txt"
fc_file = os.path.join(here, fc_file)
fc, lower, upper, plu_min = [], [], [], []
fc_file = open(fc_file, encoding='latin_1')
for line in fc_file:
str_number = r"(\s+-?\d+\.\d{3}\s*)"
regex_number = re.compile(str_number)
numbers = re.findall(regex_number, line)
if numbers == []:
continue
fc.append(float(numbers[0]))
lower.append(float(numbers[1]))
upper.append(float(numbers[2]))
plu_min.append(float(numbers[3]))
fc_file.close()
neqs = len(results["est"]["Sigma_u"])
fc = np.hstack(np.vsplit(np.array(fc)[:, None], neqs))
lower = np.hstack(np.vsplit(np.array(lower)[:, None], neqs))
upper = np.hstack(np.vsplit(np.array(upper)[:, None], neqs))
results["fc"] = dict.fromkeys(["fc", "lower", "upper"])
results["fc"]["fc"] = fc
results["fc"]["lower"] = lower
results["fc"]["upper"] = upper
# ---------------------------------------------------------------------
# parse output related to Granger-caus. and instantaneous causality:
results["granger_caus"] = dict.fromkeys(["p", "test_stat"])
results["granger_caus"]["p"] = dict()
results["granger_caus"]["test_stat"] = dict()
results["inst_caus"] = dict.fromkeys(["p", "test_stat"])
results["inst_caus"]["p"] = dict()
results["inst_caus"]["test_stat"] = dict()
vn = dataset.variable_names
# all possible combinations of potentially causing variables
# (at least 1 variable and not all variables together):
var_combs = sublists(vn, 1, len(vn) - 1)
if debug_mode:
print("\n\n\n" + dt_string)
for causing in var_combs:
caused = tuple(name for name in vn if name not in causing)
causality_file = dataset.__str__() + "_" + source + "_" \
+ dt_string + "_granger_causality_" \
+ stringify_var_names(causing, "_") + ".txt"
causality_file = os.path.join(here, causality_file)
causality_file = open(causality_file)
causality_results = []
for line in causality_file:
str_number = r"\d+\.\d{4}"
regex_number = re.compile(str_number)
number = re.search(regex_number, line)
if number is None:
continue
number = float(number.group(0))
causality_results.append(number)
causality_file.close()
results["granger_caus"]["test_stat"][(causing, caused)] = \
causality_results[0]
results["granger_caus"]["p"][(causing, caused)] =\
causality_results[1]
results["inst_caus"]["test_stat"][(causing, caused)] = \
causality_results[2]
results["inst_caus"]["p"][(causing, caused)] = \
causality_results[3]
# ---------------------------------------------------------------------
# parse output related to impulse-response analysis:
ir_file = dataset.__str__() + "_" + source + "_" + dt_string \
+ "_ir" + ".txt"
ir_file = os.path.join(here, ir_file)
ir_file = open(ir_file, encoding='latin_1')
causing = None
caused = None
data = None
regex_vars = re.compile(r"\w+")
regex_vals = re.compile(r"-?\d+\.\d{4}")
line_start_causing = "time"
data_line_indicator = "point estimate"
data_rows_read = 0
for line in ir_file:
if causing is None and not line.startswith(line_start_causing):
continue # no relevant info in the header
if line.startswith(line_start_causing):
line = line[4:]
causing = re.findall(regex_vars, line)
# 21 periods shown in JMulTi output
data = np.empty((21, len(causing)))
continue
if caused is None:
caused = re.findall(regex_vars, line)
continue
# now start collecting the values:
if data_line_indicator not in line:
continue
start = line.find(data_line_indicator) + len(data_line_indicator)
line = line[start:]
data[data_rows_read] = re.findall(regex_vals, line)
data_rows_read += 1
ir_file.close()
results["ir"] = data
# ---------------------------------------------------------------------
# parse output related to lag order selection:
lagorder_file = dataset.__str__() + "_" + source + "_" + dt_string \
+ "_lagorder" + ".txt"
lagorder_file = os.path.join(here, lagorder_file)
lagorder_file = open(lagorder_file, encoding='latin_1')
results["lagorder"] = dict()
aic_start = "Akaike Info Criterion:"
fpe_start = "Final Prediction Error:"
hqic_start = "Hannan-Quinn Criterion:"
bic_start = "Schwarz Criterion:"
for line in lagorder_file:
if line.startswith(aic_start):
results["lagorder"]["aic"] = int(line[len(aic_start):])
elif line.startswith(fpe_start):
results["lagorder"]["fpe"] = int(line[len(fpe_start):])
elif line.startswith(hqic_start):
results["lagorder"]["hqic"] = int(line[len(hqic_start):])
elif line.startswith(bic_start):
results["lagorder"]["bic"] = int(line[len(bic_start):])
lagorder_file.close()
# ---------------------------------------------------------------------
# parse output related to non-normality-test:
test_norm_file = dataset.__str__() + "_" + source + "_" + dt_string \
+ "_diag" + ".txt"
test_norm_file = os.path.join(here, test_norm_file)
test_norm_file = open(test_norm_file, encoding='latin_1')
results["test_norm"] = dict()
section_start_marker = "TESTS FOR NONNORMALITY"
section_reached = False
subsection_start_marker = "Introduction to Multiple Time Series A"
subsection_reached = False
line_start_statistic = "joint test statistic:"
line_start_pvalue = " p-value:"
for line in test_norm_file:
if not section_reached:
if section_start_marker in line:
section_reached = True # section w/ relevant results found
continue
if not subsection_reached:
if subsection_start_marker in line:
subsection_reached = True
continue
if "joint_pvalue" in results["test_norm"].keys():
break
if line.startswith(line_start_statistic):
line_end = line[len(line_start_statistic):]
results["test_norm"]["joint_test_statistic"] = float(line_end)
if line.startswith(line_start_pvalue):
line_end = line[len(line_start_pvalue):]
results["test_norm"]["joint_pvalue"] = float(line_end)
test_norm_file.close()
# ---------------------------------------------------------------------
# parse output related to testing the whiteness of the residuals:
whiteness_file = dataset.__str__() + "_" + source + "_" + dt_string \
+ "_diag" + ".txt"
whiteness_file = os.path.join(here, whiteness_file)
whiteness_file = open(whiteness_file, encoding='latin_1')
results["whiteness"] = dict()
section_start_marker = "PORTMANTEAU TEST"
order_start = "tested order:"
statistic_start = "test statistic:"
p_start = " p-value:"
adj_statistic_start = "adjusted test statistic:"
unadjusted_finished = False
in_section = False
for line in whiteness_file:
if not in_section and section_start_marker not in line:
continue
if not in_section and section_start_marker in line:
in_section = True
continue
if line.startswith(order_start):
results["whiteness"]["tested order"] = int(
line[len(order_start):])
continue
if line.startswith(statistic_start):
results["whiteness"]["test statistic"] = float(
line[len(statistic_start):])
continue
if line.startswith(adj_statistic_start):
results["whiteness"]["test statistic adj."] = float(
line[len(adj_statistic_start):])
continue
if line.startswith(p_start): # same for unadjusted and adjusted
if not unadjusted_finished:
results["whiteness"]["p-value"] = \
float(line[len(p_start):])
unadjusted_finished = True
else:
results["whiteness"]["p-value adjusted"] = \
float(line[len(p_start):])
break
whiteness_file.close()
# ---------------------------------------------------------------------
if debug_mode:
print_debug_output(results, dt_string)
results_dict_per_det_terms[dt_s] = results
return results_dict_per_det_terms
def load_results_jmulti(dataset, dt_s_list):
"""
Parameters
----------
dataset : module
A data module in the statsmodels/datasets directory that defines a
__str__() method returning the dataset's name.
dt_s_list : list
A list of strings where each string represents a combination of
deterministic terms.
Returns
-------
result : dict
A dict (keys: tuples of deterministic terms and seasonal terms)
of dicts (keys: strings "est" (for estimators),
"se" (for standard errors),
"t" (for t-values),
"p" (for p-values))
of dicts (keys: strings "alpha", "beta", "Gamma" and other results)
"""
source = "jmulti"
results_dict_per_det_terms = dict.fromkeys(dt_s_list)
for dt_s in dt_s_list:
dt_string = dt_s_tup_to_string(dt_s)
params_file = dataset.__str__()+"_"+source+"_"+dt_string+".txt"
params_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
params_file)
# sections in jmulti output:
section_headers = ["Lagged endogenous term", # parameter matrices
"Deterministic term"] # c, s, ct
if dt_string == "nc":
del section_headers[-1]
results = dict()
results["est"] = dict.fromkeys(section_headers)
results["se"] = dict.fromkeys(section_headers)
results["t"] = dict.fromkeys(section_headers)
results["p"] = dict.fromkeys(section_headers)
result = []
result_se = []
result_t = []
result_p = []
rows = 0
started_reading_section = False
start_end_mark = "-----"
# ---------------------------------------------------------------------
# parse information about \alpha, \beta, \Gamma, deterministic of VECM
# and A_i and deterministic of corresponding VAR:
section = -1
params_file = open(params_file, encoding='latin_1')
for line in params_file:
if section == -1 and section_headers[section+1] not in line:
continue
if section < len(section_headers)-1 \
and section_headers[section+1] in line: # new section
section += 1
continue
if not started_reading_section:
if line.startswith(start_end_mark):
started_reading_section = True
continue
if started_reading_section:
if line.startswith(start_end_mark):
if result == []: # no values collected in section "Legend"
started_reading_section = False
continue
results["est"][section_headers[section]] = np.column_stack(
result)
result = []
results["se"][section_headers[section]] = np.column_stack(
result_se)
result_se = []
results["t"][section_headers[section]] = np.column_stack(
result_t)
result_t = []
results["p"][section_headers[section]] = np.column_stack(
result_p)
result_p = []
started_reading_section = False
continue
str_number = r"-?\d+\.\d{3}"
regex_est = re.compile(str_number + r"[^\)\]\}]")
est_col = re.findall(regex_est, line)
# standard errors in parantheses in JMulTi output:
regex_se = re.compile(r"\(" + str_number + r"\)")
se_col = re.findall(regex_se, line)
# t-values in brackets in JMulTi output:
regex_t_value = re.compile(r"\[" + str_number + r"\]")
t_col = re.findall(regex_t_value, line)
# p-values in braces in JMulTi output:
regex_p_value = re.compile(r"\{" + str_number + r"\}")
p_col = re.findall(regex_p_value, line)
if result == [] and est_col != []:
rows = len(est_col)
if est_col != []:
est_col = [float(el) for el in est_col]
result.append(est_col)
elif se_col != []:
for i in range(rows):
se_col[i] = se_col[i].replace("(", "").replace(")", "")
se_col = [float(el) for el in se_col]
result_se.append(se_col)
elif t_col != []:
for i in range(rows):
t_col[i] = t_col[i].replace("[", "").replace("]", "")
t_col = [float(el) for el in t_col]
result_t.append(t_col)
elif p_col != []:
for i in range(rows):
p_col[i] = p_col[i].replace("{", "").replace("}", "")
p_col = [float(el) for el in p_col]
result_p.append(p_col)
params_file.close()
# ---------------------------------------------------------------------
# parse information regarding \Sigma_u
sigmau_file = dataset.__str__() + "_" + source + "_" + dt_string \
+ "_Sigmau" + ".txt"
sigmau_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
sigmau_file)
rows_to_parse = 0
# all numbers of Sigma_u in notation with e (e.g. 2.283862e-05)
regex_est = re.compile(r"\s+\S+e\S+")
sigmau_section_reached = False
sigmau_file = open(sigmau_file, encoding='latin_1')
for line in sigmau_file:
if line.startswith("Log Likelihood:"):
line = line[len("Log Likelihood:"):]
results["log_like"] = float(re.findall(regex_est, line)[0])
continue
if not sigmau_section_reached and "Covariance:" not in line:
continue
if "Covariance:" in line:
sigmau_section_reached = True
row = re.findall(regex_est, line)
rows_to_parse = len(row) # Sigma_u quadratic ==> #rows==#cols
sigma_u = np.empty((rows_to_parse, rows_to_parse))
row = re.findall(regex_est, line)
rows_to_parse -= 1
sigma_u[rows_to_parse] = row # rows are added in reverse order...
if rows_to_parse == 0:
break
sigmau_file.close()
results["est"]["Sigma_u"] = sigma_u[::-1] # ...and reversed again here
# ---------------------------------------------------------------------
# parse forecast related output:
fc_file = dataset.__str__() + "_" + source + "_" + dt_string \
+ "_fc5" + ".txt"
fc_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
fc_file)
fc, lower, upper, plu_min = [], [], [], []
fc_file = open(fc_file, encoding='latin_1')
for line in fc_file:
str_number = r"(\s+-?\d+\.\d{3}\s*)"
regex_number = re.compile(str_number)
numbers = re.findall(regex_number, line)
if numbers == []:
continue
fc.append(float(numbers[0]))
lower.append(float(numbers[1]))
upper.append(float(numbers[2]))
plu_min.append(float(numbers[3]))
fc_file.close()
neqs = len(results["est"]["Sigma_u"])
fc = np.hstack(np.vsplit(np.array(fc)[:, None], neqs))
lower = np.hstack(np.vsplit(np.array(lower)[:, None], neqs))
upper = np.hstack(np.vsplit(np.array(upper)[:, None], neqs))
results["fc"] = dict.fromkeys(["fc", "lower", "upper"])
results["fc"]["fc"] = fc
results["fc"]["lower"] = lower
results["fc"]["upper"] = upper
# ---------------------------------------------------------------------
# parse output related to Granger-caus. and instantaneous causality:
results["granger_caus"] = dict.fromkeys(["p", "test_stat"])
results["granger_caus"]["p"] = dict()
results["granger_caus"]["test_stat"] = dict()
results["inst_caus"] = dict.fromkeys(["p", "test_stat"])
results["inst_caus"]["p"] = dict()
results["inst_caus"]["test_stat"] = dict()
vn = dataset.variable_names
# all possible combinations of potentially causing variables
# (at least 1 variable and not all variables together):
var_combs = sublists(vn, 1, len(vn)-1)
if debug_mode:
print("\n\n\n" + dt_string)
for causing in var_combs:
caused = tuple(name for name in vn if name not in causing)
causality_file = dataset.__str__() + "_" + source + "_" \
+ dt_string + "_granger_causality_" \
+ stringify_var_names(causing, "_") + ".txt"
causality_file = os.path.join(os.path.dirname(
os.path.realpath(__file__)), causality_file)
causality_file = open(causality_file)
causality_results = []
for line in causality_file:
str_number = r"\d+\.\d{4}"
regex_number = re.compile(str_number)
number = re.search(regex_number, line)
if number is None:
continue
number = float(number.group(0))
causality_results.append(number)
causality_file.close()
results["granger_caus"]["test_stat"][(causing, caused)] = \
causality_results[0]
results["granger_caus"]["p"][(causing, caused)] =\
causality_results[1]
results["inst_caus"]["test_stat"][(causing, caused)] = \
causality_results[2]
results["inst_caus"]["p"][(causing, caused)] = \
causality_results[3]
# ---------------------------------------------------------------------
# parse output related to impulse-response analysis:
ir_file = dataset.__str__() + "_" + source + "_" + dt_string \
+ "_ir" + ".txt"
ir_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
ir_file)
ir_file = open(ir_file, encoding='latin_1')
causing = None
caused = None
data = None
regex_vars = re.compile(r"\w+")
regex_vals = re.compile(r"-?\d+\.\d{4}")
line_start_causing = "time"
data_line_indicator = "point estimate"
data_rows_read = 0
for line in ir_file:
if causing is None and not line.startswith(line_start_causing):
continue # no relevant info in the header
if line.startswith(line_start_causing):
line = line[4:]
causing = re.findall(regex_vars, line)
# 21 periods shown in JMulTi output
data = np.empty((21, len(causing)))
continue
if caused is None:
caused = re.findall(regex_vars, line)
continue
# now start collecting the values:
if data_line_indicator not in line:
continue
start = line.find(data_line_indicator) + len(data_line_indicator)
line = line[start:]
data[data_rows_read] = re.findall(regex_vals, line)
data_rows_read += 1
ir_file.close()
results["ir"] = data
# ---------------------------------------------------------------------
# parse output related to lag order selection:
lagorder_file = dataset.__str__() + "_" + source + "_" + dt_string \
+ "_lagorder" + ".txt"
lagorder_file = os.path.join(os.path.dirname(
os.path.realpath(__file__)), lagorder_file)
lagorder_file = open(lagorder_file, encoding='latin_1')
results["lagorder"] = dict()
aic_start = "Akaike Info Criterion:"
fpe_start = "Final Prediction Error:"
hqic_start = "Hannan-Quinn Criterion:"
bic_start = "Schwarz Criterion:"
for line in lagorder_file:
if line.startswith(aic_start):
results["lagorder"]["aic"] = int(line[len(aic_start):])
elif line.startswith(fpe_start):
results["lagorder"]["fpe"] = int(line[len(fpe_start):])
elif line.startswith(hqic_start):
results["lagorder"]["hqic"] = int(line[len(hqic_start):])
elif line.startswith(bic_start):
results["lagorder"]["bic"] = int(line[len(bic_start):])
lagorder_file.close()
# ---------------------------------------------------------------------
# parse output related to non-normality-test:
test_norm_file = dataset.__str__() + "_" + source + "_" + dt_string \
+ "_diag" + ".txt"
test_norm_file = os.path.join(os.path.dirname(
os.path.realpath(__file__)), test_norm_file)
test_norm_file = open(test_norm_file, encoding='latin_1')
results["test_norm"] = dict()
section_start_marker = "TESTS FOR NONNORMALITY"
section_reached = False
subsection_start_marker = "Introduction to Multiple Time Series A"
subsection_reached = False
line_start_statistic = "joint test statistic:"
line_start_pvalue = " p-value:"
for line in test_norm_file:
if not section_reached:
if section_start_marker in line:
section_reached = True # section w/ relevant results found
continue
if not subsection_reached:
if subsection_start_marker in line:
subsection_reached = True
continue
if "joint_pvalue" in results["test_norm"].keys():
break
if line.startswith(line_start_statistic):
line_end = line[len(line_start_statistic):]
results["test_norm"]["joint_test_statistic"] = float(line_end)
if line.startswith(line_start_pvalue):
line_end = line[len(line_start_pvalue):]
results["test_norm"]["joint_pvalue"] = float(line_end)
test_norm_file.close()
# ---------------------------------------------------------------------
# parse output related to testing the whiteness of the residuals:
whiteness_file = dataset.__str__() + "_" + source + "_" + dt_string \
+ "_diag" + ".txt"
whiteness_file = os.path.join(os.path.dirname(
os.path.realpath(__file__)), whiteness_file)
whiteness_file = open(whiteness_file, encoding='latin_1')
results["whiteness"] = dict()
section_start_marker = "PORTMANTEAU TEST"
order_start = "tested order:"
statistic_start = "test statistic:"
p_start = " p-value:"
adj_statistic_start = "adjusted test statistic:"
unadjusted_finished = False
in_section = False
for line in whiteness_file:
if not in_section and section_start_marker not in line:
continue
if not in_section and section_start_marker in line:
in_section = True
continue
if line.startswith(order_start):
results["whiteness"]["tested order"] = int(
line[len(order_start):])
continue
if line.startswith(statistic_start):
results["whiteness"]["test statistic"] = float(
line[len(statistic_start):])
continue
if line.startswith(adj_statistic_start):
results["whiteness"]["test statistic adj."] = float(
line[len(adj_statistic_start):])
continue
if line.startswith(p_start): # same for unadjusted and adjusted
if not unadjusted_finished:
results["whiteness"]["p-value"] = \
float(line[len(p_start):])
unadjusted_finished = True
else:
results["whiteness"]["p-value adjusted"] = \
float(line[len(p_start):])
break
whiteness_file.close()
# ---------------------------------------------------------------------
if debug_mode:
print_debug_output(results, dt_string)
results_dict_per_det_terms[dt_s] = results
return results_dict_per_det_terms