def __run_recipe(self, session, recipe):
"""
Run recipe
:param Session session:
:param BaseRecipe recipe:
"""
if session.before_hooks:
log.info(make_bold("Executing before ingestion hook(s)..."))
self.__run_hooks(session, session.before_hooks, False)
recipe.describe()
if session.blocking and not session.is_automated():
input("Press Enter to Continue...: ")
log.title("\nRunning")
if session.blocking:
# It needs to display progress bar (how many files imported)
t = Thread(target=run_status, args=(recipe,))
t.daemon = True
t.start()
recipe.run()
t.join()
else:
# Non-blocking mode, only 1 file is importing, no need to show progress bar (it takes longer time to join threads)
recipe.run()
recipe.importer = None
if session.after_hooks:
log.info(make_bold("Executing after ingestion hook(s)..."))
self.__run_hooks(session, session.after_hooks, True)
def main():
args = parse_cmdline()
global RMANHOME, RMANHOME_BIN
RMANHOME = args.rmanhome
if not RMANHOME:
log.warn(
"RMANHOME has not been set; consider specifying it with the --rmanhome option."
)
else:
RMANHOME = append_slash(RMANHOME)
RMANHOME_BIN = RMANHOME + "bin/"
RASQL[0] = RMANHOME_BIN + RASQL[0]
log.info("rasql: %s", RASQL)
log.title("Running rasql systemtest.")
# queries and oracles directory are quite big, so they are distributed
# as compressed archives, which potentially need to be uncompressed (for a first run)
for d in [QUERIES_DIR, ORACLES_DIR]:
uncompress_directory(d)
all_tests = None
if args.testsfile:
all_tests = [SETUP_TESTS, args.testsfile]
elif args.drop:
all_tests = []
else:
all_tests = [SETUP_TESTS] + sorted([QUERIES_DIR + f
for f in os.listdir(QUERIES_DIR) \
if not f in SPECIAL_FILES and
not f.endswith('.bak')])
all_tests = all_tests + [TEARDOWN_TESTS]
log.info("Tests files to execute: %s", all_tests)
# run tests
ret = True
start_time = timer()
for tests_file in all_tests:
ret = evaluate_existing_tests(tests_file) and ret
end_time = timer()
print_stats(end_time - start_time)
return ret
def run_recipe(self, session):
"""
Recipe session
:param Session session: the session of the import
:rtype BaseRecipe
"""
recipe_name = session.get_recipe()['name']
if recipe_name not in self.registry:
raise RecipeValidationException(
"Recipe '" + session.get_recipe()['name'] + "' not found; "
"if it's a custom recipe, please put it in the "
"'$RMANHOME/share/rasdaman/wcst_import/recipes_custom' folder."
)
else:
recipe = self.registry[session.get_recipe()['name']](session)
log.title("Initialization")
if recipe_name != virtual_coverage_recipe.RECIPE_NAME:
number_of_files = len(session.get_files())
if number_of_files > 10:
number_of_files = 10
log.info("Collected first " + str(number_of_files) +
" files: " +
str([str(f)
for f in session.get_files()[:10]]) + "...")
log.title("\nValidation")
recipe.validate()
# Show what recipe and coverage are imported only once
super(recipe.__class__, recipe).describe()
if session.blocking is True:
# Default blocking import mode (analyze all files -> import)
self.__run_recipe(session, recipe)
else:
# Non blocking import mode (analyze 1 file -> import then next file)
files = list(session.get_files())
for file in files:
session.files = [file]
self.__run_recipe(session, recipe)
log.success("Recipe executed successfully")
def run_recipe(self, session):
"""
Recipe session
:param Session session: the session of the import
:rtype BaseRecipe
"""
if session.get_recipe()['name'] not in self.registry:
raise RecipeValidationException("Recipe '" + session.get_recipe()['name'] + "' not found; "
"if it's a custom recipe, please put it in the "
"'$RMANHOME/share/rasdaman/wcst_import/recipes_custom' folder.")
else:
recipe = self.registry[session.get_recipe()['name']](session)
log.title("Initialization")
log.info("Collected files: " + str(map(lambda f: str(f), session.get_files()[:10])) + "...")
log.title("\nValidation")
recipe.validate()
recipe.describe()
if not session.is_automated():
raw_input("Press Enter to Continue...: ")
t = Thread(target=run_status, args=(recipe,))
t.daemon = True
log.title("\nRunning")
t.start()
recipe.run()
t.join()
log.success("Recipe executed successfully")
def evaluate_tests(tests_file, outdir, expdir, separator):
"""
Evaluates the tests in tests_file (rendered output from rasqte.py). The
outputs are saved in outdir, and then compared to the expected output in
expdir.
"""
ret = True
if not os.path.exists(outdir):
os.makedirs(outdir)
log.title("Evaluating tests in '%s'...", tests_file)
with open(tests_file, "r") as f:
outdir = append_slash(outdir)
expdir = append_slash(expdir)
while True:
test = read_next_test(f, separator)
if test is None:
break
if not test.validate():
log.error("Invalid test case, skipping evaluation:\n%s\n%s",
test, separator)
ret = False
continue
if not test.queries:
log.error("Test case specifies no queries, skipping.")
ret = False
continue
ret = evaluate_test(test, outdir, expdir, separator) and ret
if ret:
log.success("Done, %s %s.", tests_file, TEST_PASSED)
else:
log.error("Done, %s %s.", tests_file, TEST_FAILED)
return ret
def print_usage():
"""
Prints the usage of the program
"""
log.title("NAME")
log.info("\tWCST Import - imports georeferenced files to a WCS service that supports the Transactional extension.")
log.title("\nSYNOPSIS")
log.info("\twcst_import.py ingredients.json")
log.title("\nDESCRIPTION")
log.info("\tThe WCST Import utility imports georeferenced files supported by GDAL into a WCS service that supports "
"Transactional extension.\n\tThis utility is based on a recipe (custom code for a specific scenario) being"
" used to transform the ingredients file (input files and configuration options)."
"\n\n\tThe following recipes are supported as now:"
"\n\t * 2D Mosaic - Given a set of 2D GDAL files, the recipe will produce a 2D mosaic of the given files."
"Ingredients example under ingredients/map_mosaic.json"
"\n\t * Regular Timeseries - Given a set of 2D GDAL files, the recipe will build a timeseries out of "
"them. The initial time of the series and the step for each additional file must be provided. "
"Ingredients example under ingredients/time_series_regular"
"\n\t * Irregular - Given a set of 2D GDAL files, the recipe will produce a timeseries out of "
"them. The initial time of the series and the step for each additional file must be provided. "
"Ingredients example under ingredients/time_series_irregular"
)
def print_usage():
"""
Prints the usage of the program
"""
log.title("NAME")
log.info("\tWCST Import - imports georeferenced files to a WCS service that supports the Transactional extension.")
log.title("\nSYNOPSIS")
log.info("\twcst_import.py ingredients.json")
log.title("\nDESCRIPTION")
log.info("\tThe WCST Import utility imports georeferenced files supported by GDAL into a WCS service that supports "
"Transactional extension.\n\tThis utility is based on a recipe (custom code for a specific scenario) being"
" used to transform the ingredients file (input files and configuration options)."
"\n\n\tThe following recipes are supported as now:"
"\n\t * 2D Mosaic - Given a set of 2D GDAL files, the recipe will produce a 2D mosaic of the given files."
"Ingredients example under ingredients/map_mosaic.json"
"\n\t * Regular Timeseries - Given a set of 2D GDAL files, the recipe will build a timeseries out of "
"them. The initial time of the series and the step for each additional file must be provided. "
"Ingredients example under ingredients/time_series_regular"
"\n\t * Irregular - Given a set of 2D GDAL files, the recipe will produce a timeseries out of "
"them. The initial time of the series and the step for each additional file must be provided. "
"Ingredients example under ingredients/time_series_irregular"
)
def main():
args = parse_cmdline()
separator = args.separator
template = args.template
outdir = args.outdir + ("/" if args.outdir[-1] != "/" else "")
# additional values to the globals
globals_dic = get_globals()
globals_dic['separator'] = separator
# print globals and exit
if args.globals:
for k, v in sorted(globals_dic.iteritems(), key=lambda item: item[0]):
val = str(v)
if not "jinja2" in val:
log.info(" - %s: %s\n", k, v)
sys.exit(0)
# create out directory if it doesn't exist
if args.outdir != ".":
try:
os.makedirs(args.outdir)
except OSError as exc:
if exc.errno == errno.EEXIST and os.path.isdir(args.outdir):
pass
else:
raise
# get output file
template_name = os.path.basename(template)
template_name = os.path.splitext(template_name)[0]
outfile = outdir + template_name
log.title("Rendering templates")
log.info(""" template: %s
output: %s
dimensions: %s
cell types: %s
separator: %s""", template, outfile, globals_dic['dimension_list'],
globals_dic['cell_type_name_list'], separator)
if os.path.isfile(outfile):
log.warn("Output file '" + outfile + "' exists, removing.")
os.remove(outfile)
#
# render template
#
file_loader = FileSystemLoader('')
env = Environment(loader=file_loader)
env.globals.update(globals_dic)
template_opener = env.get_template(template)
# render the template for each dimension and cell type
with open(outfile, 'a') as f:
for dimension in env.globals["dimension_list"]:
env.globals["dimension"] = dimension
test_id_prefix = template_name + "_" + str(dimension) + "d_"
for cell_type in env.globals["cell_type_name_list"]:
test_id = test_id_prefix + cell_type
key = (dimension, cell_type)
output = template_opener.render(
dimension=dimension,
cell_type_name=cell_type,
cell_type_suffix=env.globals["cell_type_suffix_dic"][cell_type],
cell_type_max=env.globals["cell_type_max_dic"][cell_type],
cell_type_min=env.globals["cell_type_min_dic"][cell_type],
cell_type_val=env.globals["cell_type_val_dic"][cell_type],
cell_type_zero=env.globals["cell_type_zero_dic"][cell_type],
cell_type_size=env.globals["cell_type_size_dic"][cell_type],
cell_type_signed=env.globals["cell_type_signed_dic"][cell_type],
cell_type_components=env.globals["cell_type_components_dic"][cell_type],
coll_name=env.globals["coll_name_dic"][key],
coll_type_name=env.globals["coll_type_name_dic"][key],
mdd_type_name=env.globals["mdd_type_name_dic"][key],
mdd_constant_extents=env.globals["mdd_constant_extents_list"][dimension],
mdd_constant_sdom=env.globals["mdd_constant_sdom_list"][dimension],
mdd_constant_subsets=env.globals["mdd_constant_subsets_list"][dimension],
mdd_constant_cell_values=env.globals["mdd_constant_cell_values_dic"][cell_type],
mdd_constant=env.globals["mdd_constant_dic"][key],
test_id=test_id,
template_name=template_name,
)
f.write(output)
log.success("Done.")
def print_stats(elapsed):
write_lines(FAILED_TESTS, FAILED_TESTS_LOG)
write_lines(IGNORED_TESTS, IGNORED_TESTS_LOG)
log.title("\nTest summary")
log.title(" Tests executed : %d", TOTAL_TESTS_COUNT)
log.title(" Queries executed : %d", TOTAL_QUERIES_COUNT)
log.title(" Failed tests : %d", len(FAILED_TESTS))
log.title(" Ignored / skipped : %d\n", len(IGNORED_TESTS))
if TOTAL_QUERIES_COUNT > 0:
log.title(" Time / query (ms) : %s",
(elapsed * 1000.0 / TOTAL_QUERIES_COUNT))
log.title(" Total time (m) : %s\n", elapsed / 60.0)
log.title(" Failed tests log : %s", FAILED_TESTS_LOG)
log.title(" Ignored tests log : %s\n", IGNORED_TESTS_LOG)
status_msg = " Status : %s"
if len(FAILED_TESTS) == 0:
log.success(status_msg, TEST_PASSED)
else:
log.error(status_msg, TEST_FAILED)
