def extract_metadata_for_folder(dataset, problem_id=None):
sys.path.append(properties.PYTHON_PROJECTS_HOME)
function_store = get_function_store(dataset)
root_folder = os.path.join(properties.PYTHON_PROJECTS_HOME, dataset)
if problem_id:
root_folder = os.path.join(root_folder, problem_id)
for file_path in cache.list_files(root_folder,
check_nest=True,
is_absolute=True):
file_name = cache.get_file_name(file_path)
if not file_name.startswith(a_consts.GENERATED_PREFIX):
continue
LOGGER.info("Processing '%s' ..." % helper.get_simple_name(file_path))
for func in helper.get_generated_functions(file_path):
function_name = func.__name__
valid, func_key = is_executable_function(dataset, func, False)
print(function_name, func_key, valid)
if valid:
meta_data = {
"name": function_name,
"body": inspect.getsource(func),
"inputKey": func_key,
"filePath": file_path
}
function_store.save_py_metadata(meta_data)
sys.path.remove(properties.PYTHON_PROJECTS_HOME)
def get_valid_function_keys_from_file(dataset, file_name):
sys.path.append(properties.PYTHON_PROJECTS_HOME)
valid_keys = []
n_generated_functions = 0
for func in helper.get_generated_functions(file_name):
n_generated_functions += 1
valid, func_key = is_executable_function(dataset, func, True)
if valid:
valid_keys.append(func_key)
sys.path.remove(properties.PYTHON_PROJECTS_HOME)
return valid_keys, n_generated_functions
def execute_file(dataset, file_name):
sys.path.append(properties.PYTHON_PROJECTS_HOME)
try:
n_valids, n_totals = 0, 0
for func in helper.get_generated_functions(file_name):
func_name = func.__name__
valid, func_key = is_executable_function(dataset, func, False)
print(func.__name__, valid)
n_totals += 1
if valid:
evaluated = evaluate_function(dataset, file_name, func_name)
if evaluated is not None:
n_valids += 1
LOGGER.info("Valid Functions: %d / %d" % (n_valids, n_totals))
except Exception as e:
LOGGER.critical("Failed to process file: %s. Message: %s" %
(file_name, e.message))
finally:
sys.path.remove(properties.PYTHON_PROJECTS_HOME)
def get_pd_functions(file_path, as_dict=False):
return helper.get_generated_functions(file_path, as_dict=as_dict)