def extract_metadata_for_folder(dataset, problem_id=None):
sys.path.append(properties.PYTHON_PROJECTS_HOME)
function_store = get_function_store(dataset)
root_folder = os.path.join(properties.PYTHON_PROJECTS_HOME, dataset)
if problem_id:
root_folder = os.path.join(root_folder, problem_id)
for file_path in cache.list_files(root_folder,
check_nest=True,
is_absolute=True):
file_name = cache.get_file_name(file_path)
if not file_name.startswith(a_consts.GENERATED_PREFIX):
continue
LOGGER.info("Processing '%s' ..." % helper.get_simple_name(file_path))
for func in helper.get_generated_functions(file_path):
function_name = func.__name__
valid, func_key = is_executable_function(dataset, func, False)
print(function_name, func_key, valid)
if valid:
meta_data = {
"name": function_name,
"body": inspect.getsource(func),
"inputKey": func_key,
"filePath": file_path
}
function_store.save_py_metadata(meta_data)
sys.path.remove(properties.PYTHON_PROJECTS_HOME)
def evaluate_function(dataset, file_name, func, is_test=False):
function_store = get_function_store(dataset, is_test=is_test)
simple_name = helper.get_simple_name(file_name)
if isinstance(func, basestring):
func = helper.get_function(file_name, func)
if func is None:
return None
function_name = func.__name__
LOGGER.info("Processing '%s' ..." % function_name)
if function_store.exists_py_function(function_name) and not RE_EVALUATE:
LOGGER.info(
"Function '%s' from '%s' exists. Returning existing value ..." %
(function_name, simple_name))
return function_store.load_py_function(function_name)
if function_store.is_invalid_py_function(
function_name) and not RE_EVALUATE:
LOGGER.info("Function '%s' from '%s' exists and is invalid." %
(function_name, simple_name))
return None
arg_keys = create_arg_keys(dataset, func, is_test)
arg_data = get_function_args(dataset, arg_keys, is_test)
if not arg_data:
LOGGER.warning(
"Arguments not found for key: '%s'. (Function: '%s'; File: '%s')" %
(create_func_key(arg_keys), function_name, simple_name))
return None
is_valid = False
outputs = []
for i, arg in enumerate(arg_data):
if DEBUG:
LOGGER.info("Running for %d of %d args" % (i + 1, len(arg_data)))
result = execute_function(func, arg)
if not is_valid and result.get("return", None) is not None:
is_valid = True
outputs.append(result)
if not is_valid:
LOGGER.warning("Function '%s' from file '%s' is not valid" %
(function_name, simple_name))
function_store.save_failed_py_function({
"name":
function_name,
"fileName":
file_name,
"dataset":
dataset,
"inputKey":
create_func_key(arg_keys)
})
return None
function_data = {
"name": function_name,
"fileName": file_name,
"dataset": dataset,
"inputKey": create_func_key(arg_keys),
"outputs": outputs
}
LOGGER.info("Processed function '%s' ..." % function_name)
function_store.save_py_function(function_data)
return function_data
def execute_problem(dataset, problem_id=None):
root_folder = os.path.join(properties.PYTHON_PROJECTS_HOME, dataset)
if problem_id:
root_folder = os.path.join(root_folder, problem_id)
for file_path in cache.list_files(root_folder,
check_nest=True,
is_absolute=True):
if not cache.get_file_name(file_path).startswith(
a_consts.GENERATED_PREFIX):
continue
LOGGER.info("Processing '%s'" % helper.get_simple_name(file_path))
execute_file(dataset, file_path)
def get_valid_functions_from_folder(dataset, problem_id=None):
total_valid_functions = 0
accessed_keys = set()
root_folder = properties.PYTHON_PROJECTS_HOME
if problem_id:
root_folder = os.path.join(root_folder, problem_id)
for file_path in cache.list_files(root_folder,
check_nest=True,
is_absolute=True):
file_name = cache.get_file_name(file_path)
if not file_name.startswith(a_consts.GENERATED_PREFIX):
continue
LOGGER.info("Processing '%s'" % helper.get_simple_name(file_path))
valid_keys, n_generated_functions = get_valid_function_keys_from_file(
dataset, file_path)
LOGGER.info("Valid Functions: %d / %d\n" %
(len(valid_keys), n_generated_functions))
accessed_keys.update(valid_keys)
total_valid_functions += len(valid_keys)
LOGGER.info("Total valid functions: %d" % total_valid_functions)
print(accessed_keys)