def execute_stmt(stmt, args):
snippet = strip_comments(stmt['snippet'])
rets = get_return_values(snippet)
inp = stmt['variables'][0]
ret_bodies = {}
if rets == props.SELF:
ret_bodies[props.SELF] = ["return %s" % snippet]
elif rets == props.AUTO_RETURN:
ret_bodies[props.AUTO_RETURN] = [snippet]
else:
for ret in rets:
ret_bodies[ret] = [snippet, "return %s" % ret]
ret_results = {}
for ret, body in ret_bodies.items():
func_name, file_path = create_temp_function_file(inp, body)
if func_name is None:
cache.delete_file(file_path)
continue
func = helper.get_function(file_path, func_name, props.PYTHON_SRC)
assert func is not None
results = []
for arg_set in args:
result = execute.execute_function(func, arg_set)
try:
result['return'] = mongo_driver.to_mongo_format(
result['return'])
except Exception as e:
result['return'] = None
result['errorMessage'] = e.message
results.append(result)
cache.delete_file(file_path)
ret_results[mongo_driver.mongo_escape(ret)] = results
return ret_results
def crawl_link(url, key):
LOGGER.info("Fetching URL: %s" % url)
parsed_url = urlparse.urlparse(url)
user_name = re.sub('[^a-zA-Z]+', '',
urlparse.parse_qs(parsed_url.query)['username'][0])
user_name_path = os.path.join(properties.PYTHON_PROJECTS_HOME, DATASET,
key, user_name)
cache.mk_package(user_name_path)
file_handle, _ = urllib.urlretrieve(url)
zip_file_object = zipfile.ZipFile(file_handle, 'r')
for file_name in zip_file_object.namelist():
f = zip_file_object.open(file_name)
file_path = os.path.join(user_name_path, file_name)
file_content = f.read()
cache.write_file(file_path, file_content)
if not cache.is_valid_python_file(file_path):
LOGGER.info("Invalid Python File: %s. Deleting it .... " %
file_path)
cache.delete_file(file_path)
if len(
cache.list_files(user_name_path,
False,
False,
ignores=["__init__.py"])) == 0:
LOGGER.info("Folder '%s' contains no python file. Deleting it")
cache.delete_folder(user_name_path)
def is_valid_function(header_str, function_body, parent_folder):
file_name = helper.generate_py_temp_name()
temp_file = os.path.join(parent_folder, "%s.py" % file_name)
content = "%s\n\n%s" % (header_str, function_body)
cache.write_file(temp_file, content)
validity = helper.is_valid_file(temp_file)
cache.delete_file(temp_file)
return validity
def r_compile(r_file_path, del_compiled=True):
try:
robjects.r('''
library(compiler)
cmpfile('%s')
''' % r_file_path)
if del_compiled:
compiled_file = r_file_path.rsplit(".", 1)[0] + ".Rc"
cache.delete_file(compiled_file)
return True
except Exception as e:
# print("Error while compilation.\n%s" % get_R_error_message(e))
# error_message = get_R_error_message(e)
# return error_message and "import pandas" in error_message
pass
return False
def execute_stmt(stmt, args):
snippet = stmt['snippet'].strip()
ret = get_return_variable(snippet)
inp = stmt['variables'][0]
body = None
if ret == props.AUTO_RETURN:
body = snippet
elif ret is not None:
body = "\n ".join([snippet, ret])
ret_results = {}
if body:
r_func = create_temp_function(inp, body)
results = []
if r_func is not None:
for arg_set in args:
result = functions.execute_R_function(r_func, arg_set)
# if result and result['return'] is not None:
# print(result)
# exit()
result['return'] = mongo_driver.to_mongo_format(result['return'])
results.append(result)
ret_results[mongo_driver.mongo_escape(ret)] = results
cache.delete_file(TEMP_FUNC_PATH)
return ret_results
def create_temp_function_file(variable, body):
headers = "\n".join(
["import numpy", "import pandas", "np, pd = numpy, pandas"])
file_path = os.path.join(
props.MISCONCEPTIONS_HOME,
"temp_func_%s.py" % helper.generate_random_string()[:6])
func_name = helper.generate_function_name()
args = ast.arguments(args=[ast.Name(id=str(variable))],
vararg=None,
kwarg=None,
defaults=[])
func_node = ast.FunctionDef(name=func_name,
args=args,
body=as_nodes(body),
decorator_list=[])
if func_node is None:
return None
func_str = astor.to_source(func_node)
content = "\n\n".join([headers, func_str])
cache.write_file(file_path, content)
if helper.is_valid_file(file_path, props.PYTHON_SRC):
return func_name, file_path
cache.delete_file(file_path)
return None, file_path
def convert_notebook(notebook_path, force=False):
write_folder = cache.get_parent_folder(notebook_path)
file_name = cache.get_file_name(notebook_path)
write_file = os.path.join(write_folder,
"%s.%s" % (file_name, props.TYPE_R))
if not force and cache.file_exists(write_file):
LOGGER.info("'%s' already converted. Moving on ... " % file_name)
return
else:
LOGGER.info("Converting filename '%s' ... " % file_name)
source_code_lines = []
with open(notebook_path) as fh:
json_obj = json.load(fh)
cells = json_obj['cells']
for cell in cells:
if cell['cell_type'] == 'code' and cell['source']:
source_code_lines += cell['source']
source_code = "\n".join(source_code_lines)
with open(write_file, 'w+') as fh:
fh.writelines(source_code.encode('utf-8'))
if not functions.r_compile(write_file, del_compiled=True):
cache.delete_file(write_file)
return False
return True