def safe_to_file(folder, results): """ Receives a list of results (type :class:`Clazz` or :class:`Function`), and put them into the right files in :var:`folder` :param folder: Where the files should be in. :type folder: str :param results: A list of :class:`Clazz` or :class:`Function` objects, which will be used to calculate the source code. :type results: Union(Clazz, Function) """ functions = [] clazzes = {} # "filepath": [Class, Class, ...] # split results into functions and classes for result in results: assert isinstance(result, (Clazz, Function)) if isinstance(result, Clazz): import_path = get_type_path(result.clazz) import_path = import_path.rstrip(".") file_path = calc_path_and_create_folders(folder, import_path) result.filepath = file_path if file_path not in clazzes: clazzes[file_path] = [] clazzes[file_path].append(result) else: assert isinstance(result, Function) import_path = "pytgbot.bot." file_path = calc_path_and_create_folders(folder, import_path) result.filepath = file_path functions.append(result) # end if # end for bot_template = get_template("bot.template") clazzfile_template = get_template("classfile.template") for path, clazz_list in clazzes.items(): clazz_imports = set() for clazz_ in clazz_list: assert isinstance(clazz_, Clazz) assert isinstance(clazz_.parent_clazz, Type) clazz_imports.add(clazz_.parent_clazz.as_import) # end for clazz_imports = list(clazz_imports) clazz_imports.sort() is_sendable = ("sendable" in path) try: with open(path, "w") as f: result = clazzfile_template.render(clazzes=clazz_list, imports=clazz_imports, is_sendable=is_sendable) result = result.replace("\t", " ") f.write(result) # end with except IOError: raise # lol # end try # end for classes if functions: txt = bot_template.render(functions=functions) with open(functions[0].filepath, "w") as f: f.write(txt)
def safe_to_file(folder, results): """ Receives a list of results (type :class:`Clazz` or :class:`Function`), and put them into the right files in :var:`folder` :param folder: Where the files should be in. :type folder: str :param results: A list of :class:`Clazz` or :class:`Function` objects, which will be used to calculate the source code. :type results: Union(Clazz, Function) """ functions = [] clazzes = {} # "filepath": [Class, Class, ...] # split results into functions and classes for result in results: assert isinstance(result, (Clazz, Function)) if isinstance(result, Clazz): import_path = get_type_path(result.clazz) import_path = import_path.rstrip(".") file_path = calc_path_and_create_folders(folder, import_path) result.filepath = file_path if file_path not in clazzes: clazzes[file_path] = [] clazzes[file_path].append(result) else: assert isinstance(result, Function) import_path = "pytgbot.bot." file_path = calc_path_and_create_folders(folder, import_path) result.filepath = file_path functions.append(result) # end if # end for bot_template = get_template("bot.template") clazzfile_template = get_template("classfile.template") for path, clazz_list in clazzes.items(): clazz_imports = set() for clazz_ in clazz_list: assert isinstance(clazz_, Clazz) assert isinstance(clazz_.parent_clazz, Type) clazz_imports.add(clazz_.parent_clazz.as_import) # end for clazz_imports = list(clazz_imports) clazz_imports.sort() try: with open(path, "w") as f: result = clazzfile_template.render(clazzes=clazz_list, imports=clazz_imports) result = result.replace("\t", " ") f.write(result) # end with except IOError: raise # lol # end try # end for classes if functions: txt = bot_template.render(functions=functions) with open(functions[0].filepath, "w") as f: f.write(txt)
def safe_to_file(folder, results): """ Receives a list of results (type :class:`Clazz` or :class:`Function`), and put them into the right files in :var:`folder` :param folder: Where the files should be in. :type folder: str :param results: A list of :class:`Clazz` or :class:`Function` objects, which will be used to calculate the source code. :type results: Union(Clazz, Function) """ functions = [] message_send_functions = [] clazzes = {} # "filepath": [Class, Class, ...] all_the_clazzes = [] # split results into functions and classes for result in results: assert isinstance(result, (Clazz, Function)) if isinstance(result, Clazz): import_path = get_type_path(result.clazz) import_path = import_path.rstrip(".") file_path = calc_path_and_create_folders(folder, import_path) result.filepath = file_path if file_path not in clazzes: clazzes[file_path] = [] clazzes[file_path].append(result) all_the_clazzes.append(result) else: assert isinstance(result, Function) import_path = "pytgbot.bot." file_path = calc_path_and_create_folders(folder, import_path) result.filepath = file_path functions.append(result) if result.name.startswith('send_'): import_path = "teleflask_messages." file_path = calc_path_and_create_folders(folder, import_path) result2 = safe_eval( repr(result), SAVE_VALUES) # serialize + unserialize = deepcopy result2.filepath = file_path message_send_functions.append(result2) # end if # end if # end for bot_template = get_template("bot.template") clazzfile_template = get_template("classfile.template") teleflask_messages_template = get_template( "teleflask_messages_file.template") typehints_template = get_template("typehintsfile.template") telegram_bot_api_server_funcs_template = get_template( "telegram_bot_api_server/funcs.template") telegram_bot_api_server_class_template = get_template( "telegram_bot_api_server/classes.template") mkdir_p(path_join(folder, 'telegram_bot_api_server', 'generated')) if all_the_clazzes: txt = telegram_bot_api_server_class_template.render( clazzes=all_the_clazzes) render_file_to_disk( path_join(folder, 'telegram_bot_api_server', 'generated', 'models.py'), txt) # end if for path, clazz_list in clazzes.items(): clazz_imports = set() for clazz_ in clazz_list: assert isinstance(clazz_, Clazz) assert isinstance(clazz_.parent_clazz, Type) clazz_imports.add(clazz_.parent_clazz.as_import) # end for clazz_imports = list(clazz_imports) clazz_imports.sort() is_sendable = ("sendable" in path) try: txt = clazzfile_template.render(clazzes=clazz_list, imports=clazz_imports, is_sendable=is_sendable) txt = txt.replace("\t", " ") render_file_to_disk(path, txt) except IOError: raise # lol # end try try: txt = typehints_template.render(clazzes=clazz_list, imports=clazz_imports, is_sendable=is_sendable) txt = txt.replace("\t", " ") render_file_to_disk(path + "i", txt) # "ponies.py" + "i" => "ponies.pyi" except IOError: raise # lol # end try try: txt = typehints_template.render(clazzes=clazz_list, imports=clazz_imports, is_sendable=is_sendable) txt = txt.replace("\t", " ") render_file_to_disk(path + "i", txt) # "ponies.py" + "i" => "ponies.pyi" except IOError: raise # lol # end try # end for classes if functions: txt = bot_template.render(functions=functions) render_file_to_disk(functions[0].filepath, txt) imports = set() imports.add(('enum', 'Enum')) imports.add(('typing', 'Union, List, Optional')) imports.add(('fastapi', 'APIRouter, HTTPException')) imports.add(('telethon', 'TelegramClient')) imports.add(('serializer', 'to_web_api, get_entity')) imports.add(('fastapi.params', 'Query')) imports.add(('telethon.errors', 'BotMethodInvalidError')) imports.add(('telethon.tl.types', 'TypeSendMessageAction')) imports.add(('telethon.client.chats', '_ChatAction')) imports.add(('luckydonaldUtils.logger', 'logging')) imports.add(('telethon.tl.functions.messages', 'SetTypingRequest')) for function in functions: function: Function for the_import in function.imports: the_import: Import imports.add((the_import.path, the_import.name)) # end for # end for # https://stackoverflow.com/a/613218/3423324#how-do-i-sort-a-dictionary-by-value # https://stackoverflow.com/a/4659539/3423324#how-to-sort-by-length-of-string-followed-by-alphabetical-order imports_sorted = [ "from " + path + ' import ' + name for path, name in sorted(imports, key=lambda item: (-len(item[0]), item[0], -len(item[1]), item[1])) ] # imports_sorted.sort(key=lambda item: (-len(item), item)) txt = telegram_bot_api_server_funcs_template.render( functions=functions, imports=imports_sorted) render_file_to_disk( path_join(folder, 'telegram_bot_api_server', 'generated', 'funcs.py'), txt) # end if if message_send_functions: txt = teleflask_messages_template.render( functions=message_send_functions) render_file_to_disk(message_send_functions[0].filepath, txt) # end if if message_send_functions: txt = teleflask_messages_template.render( functions=message_send_functions) render_file_to_disk(message_send_functions[0].filepath, txt)
def main(): bot_template = get_template("bot.template") folder = get_folder_path() filter = get_filter() document = requests.get(BASE_URL) bs = BeautifulSoup(document.content) results = [] for h in bs.select("#dev_page_content > h4"): print("------") anchor = h.find(lol1) if not anchor or not anchor.has_attr("name"): continue link = "{base_url}#{anchor}".format(base_url=BASE_URL, anchor=anchor["name"]) title = h.text descr = [] table_type, param_strings = None, None print("title: " + title) print("link: " + link) if filter and title not in filter: print("Skipping {title}, filtered.".format(title=title)) continue # logger.debug(h) type_strings = [] default_returns = [] for sibling in h.next_siblings: if sibling == "\n": continue if sibling.name in ["p", "blockquote"]: if "return" in sibling.text.lower(): parts_splitted = [] is_first_element = True # truein string, for x in sibling.children: if isinstance(x, NavigableString): if is_first_element: # Start of a new sentence => new list parts_splitted.extend([[foo.lstrip()] for foo in x.split(".") ]) is_first_element = False else: # not = in the middle of a sentence => append parts_splitted[len(parts_splitted) - 1].append( x.split(".", maxsplit=1)[0]) parts_splitted.extend( [[foo] for foo in x.split(".")[1:]]) is_first_element = False is_first_element = x.strip().endswith(".") else: obj = None if x.name in ["a", "em"]: obj = x else: obj = x.text # end if if is_first_element: # if it is at the beginning of the sentence. parts_splitted.append([obj]) is_first_element = False else: parts_splitted[len(parts_splitted) - 1].append(obj) # end if # end for # end for returns__ = [] # array of strings return_text__ = [ ] # array if strings. one item = one sentence. Not ending with a dot. is_array = False for lol_part in parts_splitted: has_return = False returns_ = [] return_text_ = "" for lol_part_part in lol_part: if isinstance(lol_part_part, str): return_text_ += lol_part_part if lol_part_part.strip().lower().endswith( "array of"): is_array = True if "return" in lol_part_part.lower(): has_return = True # end if else: # not str return_text_ += lol_part_part.text if is_array: returns_.append("list of " + lol_part_part.text) is_array = False else: returns_.append(lol_part_part.text) # end for if has_return: # append, so we can have multible sentences. return_text__.append(return_text_.strip()) returns__.extend(returns_) # end if # end for if return_text__ or returns__: # finally set it. default_returns = (". ".join(return_text__).strip(), " or ".join(returns__).strip()) # end if # end if descr.append(sibling.text) elif sibling.name == "table": assert sibling.has_attr( "class") and "table" in sibling["class"] table_type, param_strings = parse_table(sibling) elif sibling.name == "h4": break elif sibling.name == "h3": break elif sibling.name == "hr": # end of page break else: print("unknown: " + sibling.name) # end if # end for if not all([link, title, descr]): print("Skipped: Missing link, title or description") continue if not all([table_type, param_strings]): if title not in WHITELISTED_FUNCS: print( "Skipped. Has no table with Parameters or Fields.\n" "Also isn't a whitelisted function in `code_generator_settings.WHITELISTED_FUNCS`." ) continue # -> else: is in WHITELISTED_FUNCS: table_type = "func" # end if descr = "\n".join(descr) print("descr: " + repr(descr)) params_string = "\n".join( param_strings ) if param_strings else None # WHITELISTED_FUNCS have no params if table_type == "func": seems_valid = False if len(default_returns) != 2: if "return" in descr.lower(): default_returns = ["", "Message"] default_returns[0] = [ x for x in descr.split(".") if "return" in x.lower() ][0].strip() seems_valid = len(default_returns[0].split(".")) == 1 default_returns[1] = " or ".join( type_strings) if type_strings else "Message" default_returns[1] = as_types(default_returns[1], "returns") else: default_returns = ("On success, True is returned", "True") # end if "return" in description else: seems_valid = len(default_returns[0].split(".")) == 1 # end if default set if not seems_valid: returns = answer( "Textual description what the function returns", default_returns[0]) return_type = answer("Return type", default_returns[1]) if isinstance(return_type, str): return_type = as_types(return_type, "return type") # end if else: returns = default_returns[0] return_type = default_returns[1] # end if logger.debug("\n") result = func(title, descr, link, params_string, returns=returns, return_type=return_type) results.append(result) elif table_type == "class": if title in CLASS_TYPE_PATHS: parent_clazz = CLASS_TYPE_PATHS[title][ CLASS_TYPE_PATHS__PARENT] print("superclass: " + parent_clazz) else: parent_clazz = answer("Parent class name", "TgBotApiObject") # end if result = clazz(title, parent_clazz, descr, link, params_string) results.append(result) # end if # end for can_quit = False do_overwrite = confirm( "Can the folder {path} be overwritten?".format(path=folder)) print("vvvvvvvvv") while not can_quit: if do_overwrite: try: import Send2Trash Send2Trash.send2trash(folder) except ImportError: import shutil shutil.rmtree(folder) # end try # end if try: safe_to_file(folder, results) print("Writen to file.") except TemplateError as e: if isinstance(e, TemplateSyntaxError): logger.exception("Template error at {file}:{line}".format( file=e.filename, line=e.lineno)) else: logger.exception("Template error.") # end if # end try can_quit = not confirm("Write again after reloading templates?", default=True) print("#########") print("Exit.")
def main(): bot_template = get_template("bot.template") folder = get_folder_path() filter = get_filter() document = requests.get(BASE_URL) bs = BeautifulSoup(document.content) results = [] for h in bs.select("#dev_page_content > h4"): print("------") anchor = h.find(lol1) if not anchor or not anchor.has_attr("name"): continue link = "{base_url}#{anchor}".format(base_url=BASE_URL, anchor=anchor["name"]) title = h.text descr = [] table_type, param_strings = None, None print("title: " + title) print("link: " + link) if filter and title not in filter: print("Skipping {title}, filtered.".format(title=title)) continue # logger.debug(h) type_strings = [] default_returns = [] for sibling in h.next_siblings: if sibling == "\n": continue if sibling.name in ["p", "blockquote"]: if "return" in sibling.text.lower(): parts_splitted = [] is_first_element = True # truein string, for x in sibling.children: if isinstance(x, NavigableString): if is_first_element: # Start of a new sentence => new list parts_splitted.extend([[foo.lstrip()] for foo in x.split(".")]) is_first_element = False else: # not = in the middle of a sentence => append parts_splitted[len(parts_splitted)-1].append(x.split(".", maxsplit=1)[0]) parts_splitted.extend([[foo] for foo in x.split(".")[1:]]) is_first_element = False is_first_element = x.strip().endswith(".") else: obj = None if x.name in ["a", "em"]: obj = x else: obj = x.text # end if if is_first_element: # if it is at the beginning of the sentence. parts_splitted.append([obj]) is_first_element = False else: parts_splitted[len(parts_splitted)-1].append(obj) # end if # end for # end for returns__ = [] # array of strings return_text__ = [] # array if strings. one item = one sentence. Not ending with a dot. is_array = False for lol_part in parts_splitted: has_return = False returns_ = [] return_text_ = "" for lol_part_part in lol_part: if isinstance(lol_part_part, str): return_text_ += lol_part_part if lol_part_part.strip().lower().endswith("array of"): is_array = True if "return" in lol_part_part.lower(): has_return = True # end if else: # not str return_text_ += lol_part_part.text if is_array: returns_.append("list of " + lol_part_part.text) is_array = False else: returns_.append(lol_part_part.text) # end for if has_return: # append, so we can have multible sentences. return_text__.append(return_text_.strip()) returns__.extend(returns_) # end if # end for if return_text__ or returns__: # finally set it. default_returns = (". ".join(return_text__).strip(), " or ".join(returns__).strip()) # end if # end if descr.append(sibling.text) elif sibling.name == "table": assert sibling.has_attr("class") and "table" in sibling["class"] table_type, param_strings = parse_table(sibling) elif sibling.name == "h4": break elif sibling.name == "h3": break elif sibling.name == "hr": # end of page break else: print("unknown: " + sibling.name) # end if # end for if not all([link, title, descr]): print("Skipped: Missing link, title or description") continue if not all([table_type, param_strings]): if title not in WHITELISTED_FUNCS: print("Skipped. Has no table with Parameters or Fields.\n" "Also isn't a whitelisted function in `code_generator_settings.WHITELISTED_FUNCS`.") continue # -> else: is in WHITELISTED_FUNCS: table_type = "func" # end if descr = "\n".join(descr) print("descr: " + repr(descr)) params_string = "\n".join(param_strings) if param_strings else None # WHITELISTED_FUNCS have no params if table_type == "func": seems_valid = False if len(default_returns) != 2: if "return" in descr.lower(): default_returns = ["", "Message"] default_returns[0] = [x for x in descr.split(".") if "return" in x.lower()][0].strip() seems_valid = len(default_returns[0].split(".")) == 1 default_returns[1] = " or ".join(type_strings) if type_strings else "Message" default_returns[1] = as_types(default_returns[1], "returns") else: default_returns = ("On success, True is returned", "True") # end if "return" in description else: seems_valid = len(default_returns[0].split(".")) == 1 # end if default set if not seems_valid: returns = answer("Textual description what the function returns", default_returns[0]) return_type = answer("Return type", default_returns[1]) if isinstance(return_type, str): return_type = as_types(return_type, "return type") # end if else: returns = default_returns[0] return_type = default_returns[1] # end if logger.debug("\n") result = func(title, descr, link, params_string, returns=returns, return_type=return_type) results.append(result) elif table_type == "class": if title in CLASS_TYPE_PATHS: parent_clazz = CLASS_TYPE_PATHS[title][CLASS_TYPE_PATHS__PARENT] print("superclass: " + parent_clazz) else: parent_clazz = answer("Parent class name", "TgBotApiObject") # end if result = clazz(title, parent_clazz, descr, link, params_string) results.append(result) # end if # end for can_quit = False do_overwrite = confirm("Can the folder {path} be overwritten?".format(path=folder)) print("vvvvvvvvv") while not can_quit: if do_overwrite: try: import Send2Trash Send2Trash.send2trash(folder) except ImportError: import shutil shutil.rmtree(folder) # end try # end if try: safe_to_file(folder, results) with open(path_join(folder, "api.html"), "wb") as f: f.write(document.content) # end if print("Writen to file.") except TemplateError as e: if isinstance(e, TemplateSyntaxError): logger.exception("Template error at {file}:{line}".format(file=e.filename, line=e.lineno)) else: logger.exception("Template error.") # end if # end try can_quit = not confirm("Write again after reloading templates?", default=True) print("#########") print("Exit.")