def execute_stmt(stmt, args): snippet = strip_comments(stmt['snippet']) rets = get_return_values(snippet) inp = stmt['variables'][0] ret_bodies = {} if rets == props.SELF: ret_bodies[props.SELF] = ["return %s" % snippet] elif rets == props.AUTO_RETURN: ret_bodies[props.AUTO_RETURN] = [snippet] else: for ret in rets: ret_bodies[ret] = [snippet, "return %s" % ret] ret_results = {} for ret, body in ret_bodies.items(): func_name, file_path = create_temp_function_file(inp, body) if func_name is None: cache.delete_file(file_path) continue func = helper.get_function(file_path, func_name, props.PYTHON_SRC) assert func is not None results = [] for arg_set in args: result = execute.execute_function(func, arg_set) try: result['return'] = mongo_driver.to_mongo_format( result['return']) except Exception as e: result['return'] = None result['errorMessage'] = e.message results.append(result) cache.delete_file(file_path) ret_results[mongo_driver.mongo_escape(ret)] = results return ret_results
def crawl_link(url, key): LOGGER.info("Fetching URL: %s" % url) parsed_url = urlparse.urlparse(url) user_name = re.sub('[^a-zA-Z]+', '', urlparse.parse_qs(parsed_url.query)['username'][0]) user_name_path = os.path.join(properties.PYTHON_PROJECTS_HOME, DATASET, key, user_name) cache.mk_package(user_name_path) file_handle, _ = urllib.urlretrieve(url) zip_file_object = zipfile.ZipFile(file_handle, 'r') for file_name in zip_file_object.namelist(): f = zip_file_object.open(file_name) file_path = os.path.join(user_name_path, file_name) file_content = f.read() cache.write_file(file_path, file_content) if not cache.is_valid_python_file(file_path): LOGGER.info("Invalid Python File: %s. Deleting it .... " % file_path) cache.delete_file(file_path) if len( cache.list_files(user_name_path, False, False, ignores=["__init__.py"])) == 0: LOGGER.info("Folder '%s' contains no python file. Deleting it") cache.delete_folder(user_name_path)
def is_valid_function(header_str, function_body, parent_folder): file_name = helper.generate_py_temp_name() temp_file = os.path.join(parent_folder, "%s.py" % file_name) content = "%s\n\n%s" % (header_str, function_body) cache.write_file(temp_file, content) validity = helper.is_valid_file(temp_file) cache.delete_file(temp_file) return validity
def r_compile(r_file_path, del_compiled=True): try: robjects.r(''' library(compiler) cmpfile('%s') ''' % r_file_path) if del_compiled: compiled_file = r_file_path.rsplit(".", 1)[0] + ".Rc" cache.delete_file(compiled_file) return True except Exception as e: # print("Error while compilation.\n%s" % get_R_error_message(e)) # error_message = get_R_error_message(e) # return error_message and "import pandas" in error_message pass return False
def execute_stmt(stmt, args): snippet = stmt['snippet'].strip() ret = get_return_variable(snippet) inp = stmt['variables'][0] body = None if ret == props.AUTO_RETURN: body = snippet elif ret is not None: body = "\n ".join([snippet, ret]) ret_results = {} if body: r_func = create_temp_function(inp, body) results = [] if r_func is not None: for arg_set in args: result = functions.execute_R_function(r_func, arg_set) # if result and result['return'] is not None: # print(result) # exit() result['return'] = mongo_driver.to_mongo_format(result['return']) results.append(result) ret_results[mongo_driver.mongo_escape(ret)] = results cache.delete_file(TEMP_FUNC_PATH) return ret_results
def create_temp_function_file(variable, body): headers = "\n".join( ["import numpy", "import pandas", "np, pd = numpy, pandas"]) file_path = os.path.join( props.MISCONCEPTIONS_HOME, "temp_func_%s.py" % helper.generate_random_string()[:6]) func_name = helper.generate_function_name() args = ast.arguments(args=[ast.Name(id=str(variable))], vararg=None, kwarg=None, defaults=[]) func_node = ast.FunctionDef(name=func_name, args=args, body=as_nodes(body), decorator_list=[]) if func_node is None: return None func_str = astor.to_source(func_node) content = "\n\n".join([headers, func_str]) cache.write_file(file_path, content) if helper.is_valid_file(file_path, props.PYTHON_SRC): return func_name, file_path cache.delete_file(file_path) return None, file_path
def convert_notebook(notebook_path, force=False): write_folder = cache.get_parent_folder(notebook_path) file_name = cache.get_file_name(notebook_path) write_file = os.path.join(write_folder, "%s.%s" % (file_name, props.TYPE_R)) if not force and cache.file_exists(write_file): LOGGER.info("'%s' already converted. Moving on ... " % file_name) return else: LOGGER.info("Converting filename '%s' ... " % file_name) source_code_lines = [] with open(notebook_path) as fh: json_obj = json.load(fh) cells = json_obj['cells'] for cell in cells: if cell['cell_type'] == 'code' and cell['source']: source_code_lines += cell['source'] source_code = "\n".join(source_code_lines) with open(write_file, 'w+') as fh: fh.writelines(source_code.encode('utf-8')) if not functions.r_compile(write_file, del_compiled=True): cache.delete_file(write_file) return False return True