def __run_recipe(self, session, recipe): """ Run recipe :param Session session: :param BaseRecipe recipe: """ if session.before_hooks: log.info(make_bold("Executing before ingestion hook(s)...")) self.__run_hooks(session, session.before_hooks, False) recipe.describe() if session.blocking and not session.is_automated(): input("Press Enter to Continue...: ") log.title("\nRunning") if session.blocking: # It needs to display progress bar (how many files imported) t = Thread(target=run_status, args=(recipe,)) t.daemon = True t.start() recipe.run() t.join() else: # Non-blocking mode, only 1 file is importing, no need to show progress bar (it takes longer time to join threads) recipe.run() recipe.importer = None if session.after_hooks: log.info(make_bold("Executing after ingestion hook(s)...")) self.__run_hooks(session, session.after_hooks, True)
def main(): args = parse_cmdline() global RMANHOME, RMANHOME_BIN RMANHOME = args.rmanhome if not RMANHOME: log.warn( "RMANHOME has not been set; consider specifying it with the --rmanhome option." ) else: RMANHOME = append_slash(RMANHOME) RMANHOME_BIN = RMANHOME + "bin/" RASQL[0] = RMANHOME_BIN + RASQL[0] log.info("rasql: %s", RASQL) log.title("Running rasql systemtest.") # queries and oracles directory are quite big, so they are distributed # as compressed archives, which potentially need to be uncompressed (for a first run) for d in [QUERIES_DIR, ORACLES_DIR]: uncompress_directory(d) all_tests = None if args.testsfile: all_tests = [SETUP_TESTS, args.testsfile] elif args.drop: all_tests = [] else: all_tests = [SETUP_TESTS] + sorted([QUERIES_DIR + f for f in os.listdir(QUERIES_DIR) \ if not f in SPECIAL_FILES and not f.endswith('.bak')]) all_tests = all_tests + [TEARDOWN_TESTS] log.info("Tests files to execute: %s", all_tests) # run tests ret = True start_time = timer() for tests_file in all_tests: ret = evaluate_existing_tests(tests_file) and ret end_time = timer() print_stats(end_time - start_time) return ret
def run_recipe(self, session): """ Recipe session :param Session session: the session of the import :rtype BaseRecipe """ recipe_name = session.get_recipe()['name'] if recipe_name not in self.registry: raise RecipeValidationException( "Recipe '" + session.get_recipe()['name'] + "' not found; " "if it's a custom recipe, please put it in the " "'$RMANHOME/share/rasdaman/wcst_import/recipes_custom' folder." ) else: recipe = self.registry[session.get_recipe()['name']](session) log.title("Initialization") if recipe_name != virtual_coverage_recipe.RECIPE_NAME: number_of_files = len(session.get_files()) if number_of_files > 10: number_of_files = 10 log.info("Collected first " + str(number_of_files) + " files: " + str([str(f) for f in session.get_files()[:10]]) + "...") log.title("\nValidation") recipe.validate() # Show what recipe and coverage are imported only once super(recipe.__class__, recipe).describe() if session.blocking is True: # Default blocking import mode (analyze all files -> import) self.__run_recipe(session, recipe) else: # Non blocking import mode (analyze 1 file -> import then next file) files = list(session.get_files()) for file in files: session.files = [file] self.__run_recipe(session, recipe) log.success("Recipe executed successfully")
def run_recipe(self, session): """ Recipe session :param Session session: the session of the import :rtype BaseRecipe """ if session.get_recipe()['name'] not in self.registry: raise RecipeValidationException("Recipe '" + session.get_recipe()['name'] + "' not found; " "if it's a custom recipe, please put it in the " "'$RMANHOME/share/rasdaman/wcst_import/recipes_custom' folder.") else: recipe = self.registry[session.get_recipe()['name']](session) log.title("Initialization") log.info("Collected files: " + str(map(lambda f: str(f), session.get_files()[:10])) + "...") log.title("\nValidation") recipe.validate() recipe.describe() if not session.is_automated(): raw_input("Press Enter to Continue...: ") t = Thread(target=run_status, args=(recipe,)) t.daemon = True log.title("\nRunning") t.start() recipe.run() t.join() log.success("Recipe executed successfully")
def evaluate_tests(tests_file, outdir, expdir, separator): """ Evaluates the tests in tests_file (rendered output from rasqte.py). The outputs are saved in outdir, and then compared to the expected output in expdir. """ ret = True if not os.path.exists(outdir): os.makedirs(outdir) log.title("Evaluating tests in '%s'...", tests_file) with open(tests_file, "r") as f: outdir = append_slash(outdir) expdir = append_slash(expdir) while True: test = read_next_test(f, separator) if test is None: break if not test.validate(): log.error("Invalid test case, skipping evaluation:\n%s\n%s", test, separator) ret = False continue if not test.queries: log.error("Test case specifies no queries, skipping.") ret = False continue ret = evaluate_test(test, outdir, expdir, separator) and ret if ret: log.success("Done, %s %s.", tests_file, TEST_PASSED) else: log.error("Done, %s %s.", tests_file, TEST_FAILED) return ret
def print_usage(): """ Prints the usage of the program """ log.title("NAME") log.info("\tWCST Import - imports georeferenced files to a WCS service that supports the Transactional extension.") log.title("\nSYNOPSIS") log.info("\twcst_import.py ingredients.json") log.title("\nDESCRIPTION") log.info("\tThe WCST Import utility imports georeferenced files supported by GDAL into a WCS service that supports " "Transactional extension.\n\tThis utility is based on a recipe (custom code for a specific scenario) being" " used to transform the ingredients file (input files and configuration options)." "\n\n\tThe following recipes are supported as now:" "\n\t * 2D Mosaic - Given a set of 2D GDAL files, the recipe will produce a 2D mosaic of the given files." "Ingredients example under ingredients/map_mosaic.json" "\n\t * Regular Timeseries - Given a set of 2D GDAL files, the recipe will build a timeseries out of " "them. The initial time of the series and the step for each additional file must be provided. " "Ingredients example under ingredients/time_series_regular" "\n\t * Irregular - Given a set of 2D GDAL files, the recipe will produce a timeseries out of " "them. The initial time of the series and the step for each additional file must be provided. " "Ingredients example under ingredients/time_series_irregular" )
def main(): args = parse_cmdline() separator = args.separator template = args.template outdir = args.outdir + ("/" if args.outdir[-1] != "/" else "") # additional values to the globals globals_dic = get_globals() globals_dic['separator'] = separator # print globals and exit if args.globals: for k, v in sorted(globals_dic.iteritems(), key=lambda item: item[0]): val = str(v) if not "jinja2" in val: log.info(" - %s: %s\n", k, v) sys.exit(0) # create out directory if it doesn't exist if args.outdir != ".": try: os.makedirs(args.outdir) except OSError as exc: if exc.errno == errno.EEXIST and os.path.isdir(args.outdir): pass else: raise # get output file template_name = os.path.basename(template) template_name = os.path.splitext(template_name)[0] outfile = outdir + template_name log.title("Rendering templates") log.info(""" template: %s output: %s dimensions: %s cell types: %s separator: %s""", template, outfile, globals_dic['dimension_list'], globals_dic['cell_type_name_list'], separator) if os.path.isfile(outfile): log.warn("Output file '" + outfile + "' exists, removing.") os.remove(outfile) # # render template # file_loader = FileSystemLoader('') env = Environment(loader=file_loader) env.globals.update(globals_dic) template_opener = env.get_template(template) # render the template for each dimension and cell type with open(outfile, 'a') as f: for dimension in env.globals["dimension_list"]: env.globals["dimension"] = dimension test_id_prefix = template_name + "_" + str(dimension) + "d_" for cell_type in env.globals["cell_type_name_list"]: test_id = test_id_prefix + cell_type key = (dimension, cell_type) output = template_opener.render( dimension=dimension, cell_type_name=cell_type, cell_type_suffix=env.globals["cell_type_suffix_dic"][cell_type], cell_type_max=env.globals["cell_type_max_dic"][cell_type], cell_type_min=env.globals["cell_type_min_dic"][cell_type], cell_type_val=env.globals["cell_type_val_dic"][cell_type], cell_type_zero=env.globals["cell_type_zero_dic"][cell_type], cell_type_size=env.globals["cell_type_size_dic"][cell_type], cell_type_signed=env.globals["cell_type_signed_dic"][cell_type], cell_type_components=env.globals["cell_type_components_dic"][cell_type], coll_name=env.globals["coll_name_dic"][key], coll_type_name=env.globals["coll_type_name_dic"][key], mdd_type_name=env.globals["mdd_type_name_dic"][key], mdd_constant_extents=env.globals["mdd_constant_extents_list"][dimension], mdd_constant_sdom=env.globals["mdd_constant_sdom_list"][dimension], mdd_constant_subsets=env.globals["mdd_constant_subsets_list"][dimension], mdd_constant_cell_values=env.globals["mdd_constant_cell_values_dic"][cell_type], mdd_constant=env.globals["mdd_constant_dic"][key], test_id=test_id, template_name=template_name, ) f.write(output) log.success("Done.")
def print_stats(elapsed): write_lines(FAILED_TESTS, FAILED_TESTS_LOG) write_lines(IGNORED_TESTS, IGNORED_TESTS_LOG) log.title("\nTest summary") log.title(" Tests executed : %d", TOTAL_TESTS_COUNT) log.title(" Queries executed : %d", TOTAL_QUERIES_COUNT) log.title(" Failed tests : %d", len(FAILED_TESTS)) log.title(" Ignored / skipped : %d\n", len(IGNORED_TESTS)) if TOTAL_QUERIES_COUNT > 0: log.title(" Time / query (ms) : %s", (elapsed * 1000.0 / TOTAL_QUERIES_COUNT)) log.title(" Total time (m) : %s\n", elapsed / 60.0) log.title(" Failed tests log : %s", FAILED_TESTS_LOG) log.title(" Ignored tests log : %s\n", IGNORED_TESTS_LOG) status_msg = " Status : %s" if len(FAILED_TESTS) == 0: log.success(status_msg, TEST_PASSED) else: log.error(status_msg, TEST_FAILED)