def run_recipe(self, session):
"""
Recipe session
:param Session session: the session of the import
:rtype BaseRecipe
"""
if session.get_recipe()['name'] not in self.registry:
raise RecipeValidationException("Recipe '" + session.get_recipe()['name'] + "' not found; "
"if it's a custom recipe, please put it in the "
"'$RMANHOME/share/rasdaman/wcst_import/recipes_custom' folder.")
else:
recipe = self.registry[session.get_recipe()['name']](session)
log.title("Initialization")
log.info("Collected files: " + str(map(lambda f: str(f), session.get_files()[:10])) + "...")
log.title("\nValidation")
recipe.validate()
recipe.describe()
if not session.is_automated():
raw_input("Press Enter to Continue...: ")
t = Thread(target=run_status, args=(recipe,))
t.daemon = True
log.title("\nRunning")
t.start()
recipe.run()
t.join()
log.success("Recipe executed successfully")
def run_recipe(self, session):
"""
Recipe session
:param Session session: the session of the import
:rtype BaseRecipe
"""
recipe_name = session.get_recipe()['name']
if recipe_name not in self.registry:
raise RecipeValidationException(
"Recipe '" + session.get_recipe()['name'] + "' not found; "
"if it's a custom recipe, please put it in the "
"'$RMANHOME/share/rasdaman/wcst_import/recipes_custom' folder."
)
else:
recipe = self.registry[session.get_recipe()['name']](session)
log.title("Initialization")
if recipe_name != virtual_coverage_recipe.RECIPE_NAME:
number_of_files = len(session.get_files())
if number_of_files > 10:
number_of_files = 10
log.info("Collected first " + str(number_of_files) +
" files: " +
str([str(f)
for f in session.get_files()[:10]]) + "...")
log.title("\nValidation")
recipe.validate()
# Show what recipe and coverage are imported only once
super(recipe.__class__, recipe).describe()
if session.blocking is True:
# Default blocking import mode (analyze all files -> import)
self.__run_recipe(session, recipe)
else:
# Non blocking import mode (analyze 1 file -> import then next file)
files = list(session.get_files())
for file in files:
session.files = [file]
self.__run_recipe(session, recipe)
log.success("Recipe executed successfully")
def print_stats(elapsed):
write_lines(FAILED_TESTS, FAILED_TESTS_LOG)
write_lines(IGNORED_TESTS, IGNORED_TESTS_LOG)
log.title("\nTest summary")
log.title(" Tests executed : %d", TOTAL_TESTS_COUNT)
log.title(" Queries executed : %d", TOTAL_QUERIES_COUNT)
log.title(" Failed tests : %d", len(FAILED_TESTS))
log.title(" Ignored / skipped : %d\n", len(IGNORED_TESTS))
if TOTAL_QUERIES_COUNT > 0:
log.title(" Time / query (ms) : %s",
(elapsed * 1000.0 / TOTAL_QUERIES_COUNT))
log.title(" Total time (m) : %s\n", elapsed / 60.0)
log.title(" Failed tests log : %s", FAILED_TESTS_LOG)
log.title(" Ignored tests log : %s\n", IGNORED_TESTS_LOG)
status_msg = " Status : %s"
if len(FAILED_TESTS) == 0:
log.success(status_msg, TEST_PASSED)
else:
log.error(status_msg, TEST_FAILED)
def evaluate_tests(tests_file, outdir, expdir, separator):
"""
Evaluates the tests in tests_file (rendered output from rasqte.py). The
outputs are saved in outdir, and then compared to the expected output in
expdir.
"""
ret = True
if not os.path.exists(outdir):
os.makedirs(outdir)
log.title("Evaluating tests in '%s'...", tests_file)
with open(tests_file, "r") as f:
outdir = append_slash(outdir)
expdir = append_slash(expdir)
while True:
test = read_next_test(f, separator)
if test is None:
break
if not test.validate():
log.error("Invalid test case, skipping evaluation:\n%s\n%s",
test, separator)
ret = False
continue
if not test.queries:
log.error("Test case specifies no queries, skipping.")
ret = False
continue
ret = evaluate_test(test, outdir, expdir, separator) and ret
if ret:
log.success("Done, %s %s.", tests_file, TEST_PASSED)
else:
log.error("Done, %s %s.", tests_file, TEST_FAILED)
return ret
def main():
args = parse_cmdline()
separator = args.separator
template = args.template
outdir = args.outdir + ("/" if args.outdir[-1] != "/" else "")
# additional values to the globals
globals_dic = get_globals()
globals_dic['separator'] = separator
# print globals and exit
if args.globals:
for k, v in sorted(globals_dic.iteritems(), key=lambda item: item[0]):
val = str(v)
if not "jinja2" in val:
log.info(" - %s: %s\n", k, v)
sys.exit(0)
# create out directory if it doesn't exist
if args.outdir != ".":
try:
os.makedirs(args.outdir)
except OSError as exc:
if exc.errno == errno.EEXIST and os.path.isdir(args.outdir):
pass
else:
raise
# get output file
template_name = os.path.basename(template)
template_name = os.path.splitext(template_name)[0]
outfile = outdir + template_name
log.title("Rendering templates")
log.info(""" template: %s
output: %s
dimensions: %s
cell types: %s
separator: %s""", template, outfile, globals_dic['dimension_list'],
globals_dic['cell_type_name_list'], separator)
if os.path.isfile(outfile):
log.warn("Output file '" + outfile + "' exists, removing.")
os.remove(outfile)
#
# render template
#
file_loader = FileSystemLoader('')
env = Environment(loader=file_loader)
env.globals.update(globals_dic)
template_opener = env.get_template(template)
# render the template for each dimension and cell type
with open(outfile, 'a') as f:
for dimension in env.globals["dimension_list"]:
env.globals["dimension"] = dimension
test_id_prefix = template_name + "_" + str(dimension) + "d_"
for cell_type in env.globals["cell_type_name_list"]:
test_id = test_id_prefix + cell_type
key = (dimension, cell_type)
output = template_opener.render(
dimension=dimension,
cell_type_name=cell_type,
cell_type_suffix=env.globals["cell_type_suffix_dic"][cell_type],
cell_type_max=env.globals["cell_type_max_dic"][cell_type],
cell_type_min=env.globals["cell_type_min_dic"][cell_type],
cell_type_val=env.globals["cell_type_val_dic"][cell_type],
cell_type_zero=env.globals["cell_type_zero_dic"][cell_type],
cell_type_size=env.globals["cell_type_size_dic"][cell_type],
cell_type_signed=env.globals["cell_type_signed_dic"][cell_type],
cell_type_components=env.globals["cell_type_components_dic"][cell_type],
coll_name=env.globals["coll_name_dic"][key],
coll_type_name=env.globals["coll_type_name_dic"][key],
mdd_type_name=env.globals["mdd_type_name_dic"][key],
mdd_constant_extents=env.globals["mdd_constant_extents_list"][dimension],
mdd_constant_sdom=env.globals["mdd_constant_sdom_list"][dimension],
mdd_constant_subsets=env.globals["mdd_constant_subsets_list"][dimension],
mdd_constant_cell_values=env.globals["mdd_constant_cell_values_dic"][cell_type],
mdd_constant=env.globals["mdd_constant_dic"][key],
test_id=test_id,
template_name=template_name,
)
f.write(output)
log.success("Done.")