def safe_to_file(folder, results):
"""
Receives a list of results (type :class:`Clazz` or :class:`Function`), and put them into the right files in :var:`folder`
:param folder: Where the files should be in.
:type folder: str
:param results: A list of :class:`Clazz` or :class:`Function` objects, which will be used to calculate the source code.
:type results: Union(Clazz, Function)
"""
functions = []
clazzes = {} # "filepath": [Class, Class, ...]
# split results into functions and classes
for result in results:
assert isinstance(result, (Clazz, Function))
if isinstance(result, Clazz):
import_path = get_type_path(result.clazz)
import_path = import_path.rstrip(".")
file_path = calc_path_and_create_folders(folder, import_path)
result.filepath = file_path
if file_path not in clazzes:
clazzes[file_path] = []
clazzes[file_path].append(result)
else:
assert isinstance(result, Function)
import_path = "pytgbot.bot."
file_path = calc_path_and_create_folders(folder, import_path)
result.filepath = file_path
functions.append(result)
# end if
# end for
bot_template = get_template("bot.template")
clazzfile_template = get_template("classfile.template")
for path, clazz_list in clazzes.items():
clazz_imports = set()
for clazz_ in clazz_list:
assert isinstance(clazz_, Clazz)
assert isinstance(clazz_.parent_clazz, Type)
clazz_imports.add(clazz_.parent_clazz.as_import)
# end for
clazz_imports = list(clazz_imports)
clazz_imports.sort()
is_sendable = ("sendable" in path)
try:
with open(path, "w") as f:
result = clazzfile_template.render(clazzes=clazz_list, imports=clazz_imports, is_sendable=is_sendable)
result = result.replace("\t", " ")
f.write(result)
# end with
except IOError:
raise # lol
# end try
# end for classes
if functions:
txt = bot_template.render(functions=functions)
with open(functions[0].filepath, "w") as f:
f.write(txt)
def safe_to_file(folder, results):
"""
Receives a list of results (type :class:`Clazz` or :class:`Function`), and put them into the right files in :var:`folder`
:param folder: Where the files should be in.
:type folder: str
:param results: A list of :class:`Clazz` or :class:`Function` objects, which will be used to calculate the source code.
:type results: Union(Clazz, Function)
"""
functions = []
clazzes = {} # "filepath": [Class, Class, ...]
# split results into functions and classes
for result in results:
assert isinstance(result, (Clazz, Function))
if isinstance(result, Clazz):
import_path = get_type_path(result.clazz)
import_path = import_path.rstrip(".")
file_path = calc_path_and_create_folders(folder, import_path)
result.filepath = file_path
if file_path not in clazzes:
clazzes[file_path] = []
clazzes[file_path].append(result)
else:
assert isinstance(result, Function)
import_path = "pytgbot.bot."
file_path = calc_path_and_create_folders(folder, import_path)
result.filepath = file_path
functions.append(result)
# end if
# end for
bot_template = get_template("bot.template")
clazzfile_template = get_template("classfile.template")
for path, clazz_list in clazzes.items():
clazz_imports = set()
for clazz_ in clazz_list:
assert isinstance(clazz_, Clazz)
assert isinstance(clazz_.parent_clazz, Type)
clazz_imports.add(clazz_.parent_clazz.as_import)
# end for
clazz_imports = list(clazz_imports)
clazz_imports.sort()
try:
with open(path, "w") as f:
result = clazzfile_template.render(clazzes=clazz_list, imports=clazz_imports)
result = result.replace("\t", " ")
f.write(result)
# end with
except IOError:
raise # lol
# end try
# end for classes
if functions:
txt = bot_template.render(functions=functions)
with open(functions[0].filepath, "w") as f:
f.write(txt)
def safe_to_file(folder, results):
"""
Receives a list of results (type :class:`Clazz` or :class:`Function`), and put them into the right files in :var:`folder`
:param folder: Where the files should be in.
:type folder: str
:param results: A list of :class:`Clazz` or :class:`Function` objects, which will be used to calculate the source code.
:type results: Union(Clazz, Function)
"""
functions = []
message_send_functions = []
clazzes = {} # "filepath": [Class, Class, ...]
all_the_clazzes = []
# split results into functions and classes
for result in results:
assert isinstance(result, (Clazz, Function))
if isinstance(result, Clazz):
import_path = get_type_path(result.clazz)
import_path = import_path.rstrip(".")
file_path = calc_path_and_create_folders(folder, import_path)
result.filepath = file_path
if file_path not in clazzes:
clazzes[file_path] = []
clazzes[file_path].append(result)
all_the_clazzes.append(result)
else:
assert isinstance(result, Function)
import_path = "pytgbot.bot."
file_path = calc_path_and_create_folders(folder, import_path)
result.filepath = file_path
functions.append(result)
if result.name.startswith('send_'):
import_path = "teleflask_messages."
file_path = calc_path_and_create_folders(folder, import_path)
result2 = safe_eval(
repr(result),
SAVE_VALUES) # serialize + unserialize = deepcopy
result2.filepath = file_path
message_send_functions.append(result2)
# end if
# end if
# end for
bot_template = get_template("bot.template")
clazzfile_template = get_template("classfile.template")
teleflask_messages_template = get_template(
"teleflask_messages_file.template")
typehints_template = get_template("typehintsfile.template")
telegram_bot_api_server_funcs_template = get_template(
"telegram_bot_api_server/funcs.template")
telegram_bot_api_server_class_template = get_template(
"telegram_bot_api_server/classes.template")
mkdir_p(path_join(folder, 'telegram_bot_api_server', 'generated'))
if all_the_clazzes:
txt = telegram_bot_api_server_class_template.render(
clazzes=all_the_clazzes)
render_file_to_disk(
path_join(folder, 'telegram_bot_api_server', 'generated',
'models.py'), txt)
# end if
for path, clazz_list in clazzes.items():
clazz_imports = set()
for clazz_ in clazz_list:
assert isinstance(clazz_, Clazz)
assert isinstance(clazz_.parent_clazz, Type)
clazz_imports.add(clazz_.parent_clazz.as_import)
# end for
clazz_imports = list(clazz_imports)
clazz_imports.sort()
is_sendable = ("sendable" in path)
try:
txt = clazzfile_template.render(clazzes=clazz_list,
imports=clazz_imports,
is_sendable=is_sendable)
txt = txt.replace("\t", " ")
render_file_to_disk(path, txt)
except IOError:
raise # lol
# end try
try:
txt = typehints_template.render(clazzes=clazz_list,
imports=clazz_imports,
is_sendable=is_sendable)
txt = txt.replace("\t", " ")
render_file_to_disk(path + "i",
txt) # "ponies.py" + "i" => "ponies.pyi"
except IOError:
raise # lol
# end try
try:
txt = typehints_template.render(clazzes=clazz_list,
imports=clazz_imports,
is_sendable=is_sendable)
txt = txt.replace("\t", " ")
render_file_to_disk(path + "i",
txt) # "ponies.py" + "i" => "ponies.pyi"
except IOError:
raise # lol
# end try
# end for classes
if functions:
txt = bot_template.render(functions=functions)
render_file_to_disk(functions[0].filepath, txt)
imports = set()
imports.add(('enum', 'Enum'))
imports.add(('typing', 'Union, List, Optional'))
imports.add(('fastapi', 'APIRouter, HTTPException'))
imports.add(('telethon', 'TelegramClient'))
imports.add(('serializer', 'to_web_api, get_entity'))
imports.add(('fastapi.params', 'Query'))
imports.add(('telethon.errors', 'BotMethodInvalidError'))
imports.add(('telethon.tl.types', 'TypeSendMessageAction'))
imports.add(('telethon.client.chats', '_ChatAction'))
imports.add(('luckydonaldUtils.logger', 'logging'))
imports.add(('telethon.tl.functions.messages', 'SetTypingRequest'))
for function in functions:
function: Function
for the_import in function.imports:
the_import: Import
imports.add((the_import.path, the_import.name))
# end for
# end for
# https://stackoverflow.com/a/613218/3423324#how-do-i-sort-a-dictionary-by-value
# https://stackoverflow.com/a/4659539/3423324#how-to-sort-by-length-of-string-followed-by-alphabetical-order
imports_sorted = [
"from " + path + ' import ' + name
for path, name in sorted(imports,
key=lambda item: (-len(item[0]), item[0],
-len(item[1]), item[1]))
]
# imports_sorted.sort(key=lambda item: (-len(item), item))
txt = telegram_bot_api_server_funcs_template.render(
functions=functions, imports=imports_sorted)
render_file_to_disk(
path_join(folder, 'telegram_bot_api_server', 'generated',
'funcs.py'), txt)
# end if
if message_send_functions:
txt = teleflask_messages_template.render(
functions=message_send_functions)
render_file_to_disk(message_send_functions[0].filepath, txt)
# end if
if message_send_functions:
txt = teleflask_messages_template.render(
functions=message_send_functions)
render_file_to_disk(message_send_functions[0].filepath, txt)
def main():
bot_template = get_template("bot.template")
folder = get_folder_path()
filter = get_filter()
document = requests.get(BASE_URL)
bs = BeautifulSoup(document.content)
results = []
for h in bs.select("#dev_page_content > h4"):
print("------")
anchor = h.find(lol1)
if not anchor or not anchor.has_attr("name"):
continue
link = "{base_url}#{anchor}".format(base_url=BASE_URL,
anchor=anchor["name"])
title = h.text
descr = []
table_type, param_strings = None, None
print("title: " + title)
print("link: " + link)
if filter and title not in filter:
print("Skipping {title}, filtered.".format(title=title))
continue
# logger.debug(h)
type_strings = []
default_returns = []
for sibling in h.next_siblings:
if sibling == "\n":
continue
if sibling.name in ["p", "blockquote"]:
if "return" in sibling.text.lower():
parts_splitted = []
is_first_element = True # truein string,
for x in sibling.children:
if isinstance(x, NavigableString):
if is_first_element: # Start of a new sentence => new list
parts_splitted.extend([[foo.lstrip()]
for foo in x.split(".")
])
is_first_element = False
else: # not = in the middle of a sentence => append
parts_splitted[len(parts_splitted) - 1].append(
x.split(".", maxsplit=1)[0])
parts_splitted.extend(
[[foo] for foo in x.split(".")[1:]])
is_first_element = False
is_first_element = x.strip().endswith(".")
else:
obj = None
if x.name in ["a", "em"]:
obj = x
else:
obj = x.text
# end if
if is_first_element: # if it is at the beginning of the sentence.
parts_splitted.append([obj])
is_first_element = False
else:
parts_splitted[len(parts_splitted) -
1].append(obj)
# end if
# end for
# end for
returns__ = [] # array of strings
return_text__ = [
] # array if strings. one item = one sentence. Not ending with a dot.
is_array = False
for lol_part in parts_splitted:
has_return = False
returns_ = []
return_text_ = ""
for lol_part_part in lol_part:
if isinstance(lol_part_part, str):
return_text_ += lol_part_part
if lol_part_part.strip().lower().endswith(
"array of"):
is_array = True
if "return" in lol_part_part.lower():
has_return = True
# end if
else: # not str
return_text_ += lol_part_part.text
if is_array:
returns_.append("list of " +
lol_part_part.text)
is_array = False
else:
returns_.append(lol_part_part.text)
# end for
if has_return: # append, so we can have multible sentences.
return_text__.append(return_text_.strip())
returns__.extend(returns_)
# end if
# end for
if return_text__ or returns__: # finally set it.
default_returns = (". ".join(return_text__).strip(),
" or ".join(returns__).strip())
# end if
# end if
descr.append(sibling.text)
elif sibling.name == "table":
assert sibling.has_attr(
"class") and "table" in sibling["class"]
table_type, param_strings = parse_table(sibling)
elif sibling.name == "h4":
break
elif sibling.name == "h3":
break
elif sibling.name == "hr": # end of page
break
else:
print("unknown: " + sibling.name)
# end if
# end for
if not all([link, title, descr]):
print("Skipped: Missing link, title or description")
continue
if not all([table_type, param_strings]):
if title not in WHITELISTED_FUNCS:
print(
"Skipped. Has no table with Parameters or Fields.\n"
"Also isn't a whitelisted function in `code_generator_settings.WHITELISTED_FUNCS`."
)
continue
# -> else: is in WHITELISTED_FUNCS:
table_type = "func"
# end if
descr = "\n".join(descr)
print("descr: " + repr(descr))
params_string = "\n".join(
param_strings
) if param_strings else None # WHITELISTED_FUNCS have no params
if table_type == "func":
seems_valid = False
if len(default_returns) != 2:
if "return" in descr.lower():
default_returns = ["", "Message"]
default_returns[0] = [
x for x in descr.split(".") if "return" in x.lower()
][0].strip()
seems_valid = len(default_returns[0].split(".")) == 1
default_returns[1] = " or ".join(
type_strings) if type_strings else "Message"
default_returns[1] = as_types(default_returns[1],
"returns")
else:
default_returns = ("On success, True is returned", "True")
# end if "return" in description
else:
seems_valid = len(default_returns[0].split(".")) == 1
# end if default set
if not seems_valid:
returns = answer(
"Textual description what the function returns",
default_returns[0])
return_type = answer("Return type", default_returns[1])
if isinstance(return_type, str):
return_type = as_types(return_type, "return type")
# end if
else:
returns = default_returns[0]
return_type = default_returns[1]
# end if
logger.debug("\n")
result = func(title,
descr,
link,
params_string,
returns=returns,
return_type=return_type)
results.append(result)
elif table_type == "class":
if title in CLASS_TYPE_PATHS:
parent_clazz = CLASS_TYPE_PATHS[title][
CLASS_TYPE_PATHS__PARENT]
print("superclass: " + parent_clazz)
else:
parent_clazz = answer("Parent class name", "TgBotApiObject")
# end if
result = clazz(title, parent_clazz, descr, link, params_string)
results.append(result)
# end if
# end for
can_quit = False
do_overwrite = confirm(
"Can the folder {path} be overwritten?".format(path=folder))
print("vvvvvvvvv")
while not can_quit:
if do_overwrite:
try:
import Send2Trash
Send2Trash.send2trash(folder)
except ImportError:
import shutil
shutil.rmtree(folder)
# end try
# end if
try:
safe_to_file(folder, results)
print("Writen to file.")
except TemplateError as e:
if isinstance(e, TemplateSyntaxError):
logger.exception("Template error at {file}:{line}".format(
file=e.filename, line=e.lineno))
else:
logger.exception("Template error.")
# end if
# end try
can_quit = not confirm("Write again after reloading templates?",
default=True)
print("#########")
print("Exit.")
def main():
bot_template = get_template("bot.template")
folder = get_folder_path()
filter = get_filter()
document = requests.get(BASE_URL)
bs = BeautifulSoup(document.content)
results = []
for h in bs.select("#dev_page_content > h4"):
print("------")
anchor = h.find(lol1)
if not anchor or not anchor.has_attr("name"):
continue
link = "{base_url}#{anchor}".format(base_url=BASE_URL, anchor=anchor["name"])
title = h.text
descr = []
table_type, param_strings = None, None
print("title: " + title)
print("link: " + link)
if filter and title not in filter:
print("Skipping {title}, filtered.".format(title=title))
continue
# logger.debug(h)
type_strings = []
default_returns = []
for sibling in h.next_siblings:
if sibling == "\n":
continue
if sibling.name in ["p", "blockquote"]:
if "return" in sibling.text.lower():
parts_splitted = []
is_first_element = True # truein string,
for x in sibling.children:
if isinstance(x, NavigableString):
if is_first_element: # Start of a new sentence => new list
parts_splitted.extend([[foo.lstrip()] for foo in x.split(".")])
is_first_element = False
else: # not = in the middle of a sentence => append
parts_splitted[len(parts_splitted)-1].append(x.split(".", maxsplit=1)[0])
parts_splitted.extend([[foo] for foo in x.split(".")[1:]])
is_first_element = False
is_first_element = x.strip().endswith(".")
else:
obj = None
if x.name in ["a", "em"]:
obj = x
else:
obj = x.text
# end if
if is_first_element: # if it is at the beginning of the sentence.
parts_splitted.append([obj])
is_first_element = False
else:
parts_splitted[len(parts_splitted)-1].append(obj)
# end if
# end for
# end for
returns__ = [] # array of strings
return_text__ = [] # array if strings. one item = one sentence. Not ending with a dot.
is_array = False
for lol_part in parts_splitted:
has_return = False
returns_ = []
return_text_ = ""
for lol_part_part in lol_part:
if isinstance(lol_part_part, str):
return_text_ += lol_part_part
if lol_part_part.strip().lower().endswith("array of"):
is_array = True
if "return" in lol_part_part.lower():
has_return = True
# end if
else: # not str
return_text_ += lol_part_part.text
if is_array:
returns_.append("list of " + lol_part_part.text)
is_array = False
else:
returns_.append(lol_part_part.text)
# end for
if has_return: # append, so we can have multible sentences.
return_text__.append(return_text_.strip())
returns__.extend(returns_)
# end if
# end for
if return_text__ or returns__: # finally set it.
default_returns = (". ".join(return_text__).strip(), " or ".join(returns__).strip())
# end if
# end if
descr.append(sibling.text)
elif sibling.name == "table":
assert sibling.has_attr("class") and "table" in sibling["class"]
table_type, param_strings = parse_table(sibling)
elif sibling.name == "h4":
break
elif sibling.name == "h3":
break
elif sibling.name == "hr": # end of page
break
else:
print("unknown: " + sibling.name)
# end if
# end for
if not all([link, title, descr]):
print("Skipped: Missing link, title or description")
continue
if not all([table_type, param_strings]):
if title not in WHITELISTED_FUNCS:
print("Skipped. Has no table with Parameters or Fields.\n"
"Also isn't a whitelisted function in `code_generator_settings.WHITELISTED_FUNCS`.")
continue
# -> else: is in WHITELISTED_FUNCS:
table_type = "func"
# end if
descr = "\n".join(descr)
print("descr: " + repr(descr))
params_string = "\n".join(param_strings) if param_strings else None # WHITELISTED_FUNCS have no params
if table_type == "func":
seems_valid = False
if len(default_returns) != 2:
if "return" in descr.lower():
default_returns = ["", "Message"]
default_returns[0] = [x for x in descr.split(".") if "return" in x.lower()][0].strip()
seems_valid = len(default_returns[0].split(".")) == 1
default_returns[1] = " or ".join(type_strings) if type_strings else "Message"
default_returns[1] = as_types(default_returns[1], "returns")
else:
default_returns = ("On success, True is returned", "True")
# end if "return" in description
else:
seems_valid = len(default_returns[0].split(".")) == 1
# end if default set
if not seems_valid:
returns = answer("Textual description what the function returns", default_returns[0])
return_type = answer("Return type", default_returns[1])
if isinstance(return_type, str):
return_type = as_types(return_type, "return type")
# end if
else:
returns = default_returns[0]
return_type = default_returns[1]
# end if
logger.debug("\n")
result = func(title, descr, link, params_string, returns=returns, return_type=return_type)
results.append(result)
elif table_type == "class":
if title in CLASS_TYPE_PATHS:
parent_clazz = CLASS_TYPE_PATHS[title][CLASS_TYPE_PATHS__PARENT]
print("superclass: " + parent_clazz)
else:
parent_clazz = answer("Parent class name", "TgBotApiObject")
# end if
result = clazz(title, parent_clazz, descr, link, params_string)
results.append(result)
# end if
# end for
can_quit = False
do_overwrite = confirm("Can the folder {path} be overwritten?".format(path=folder))
print("vvvvvvvvv")
while not can_quit:
if do_overwrite:
try:
import Send2Trash
Send2Trash.send2trash(folder)
except ImportError:
import shutil
shutil.rmtree(folder)
# end try
# end if
try:
safe_to_file(folder, results)
with open(path_join(folder, "api.html"), "wb") as f:
f.write(document.content)
# end if
print("Writen to file.")
except TemplateError as e:
if isinstance(e, TemplateSyntaxError):
logger.exception("Template error at {file}:{line}".format(file=e.filename, line=e.lineno))
else:
logger.exception("Template error.")
# end if
# end try
can_quit = not confirm("Write again after reloading templates?", default=True)
print("#########")
print("Exit.")
