def calc_path_and_create_folders(folder, import_path):
""" calculate the path and create the needed folders """
file_path = abspath(
path_join(
folder, import_path[:import_path.rfind(".")].replace(
".", folder_seperator) + ".py"))
mkdir_p(dirname(file_path))
return file_path
async def got_some_update(event: events.NewMessage):
"""Process incomming Updates."""
logger.info(f'account {self.user_id!r} got message: {event!s}')
if isinstance(event, (UpdateChannelMessageViews, UpdateChannel)):
logger.info(f'Skipping Update type {type(event)}')
return
# end if
from datetime import datetime
now = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
mkdir_p(f'logs/{self.user_id}/')
with open(f'logs/{self.user_id}/update_{now}.py', 'w') as f:
f.write(
'from telethon.tl.types import *\nfrom telethon.tl.patched import *\nimport datetime\n\n'
)
f.write(str(event))
# end with
assert_type_or_raise(event,
*TypeUpdate.__args__,
parameter_name='event')
try:
update: Update = await to_web_api(
event, client=self
) # provide the client so we can lookup chats and users.
except TypeError as e:
logger.exception('Serializing element failed')
with open(f'logs/{self.user_id}/update_{now}.txt', 'w') as f:
f.write(str(e))
# end with
# await event.respond()
raise events.StopPropagation
# end def
with open(f'logs/{self.user_id}/update_{now}.json', 'w') as f:
json.dump(
update.to_array()
if isinstance(update, TgBotApiObject) else update, f)
# end with
if update is None:
logger.debug(f'Skipping `None` event: {event!r}')
return
# end if
await self.send_event(update)
# await event.respond()
self.update_id += 1
if self.update_id > 2147483647:
self.update_id = self.create_random_update_id()
# end if
raise events.StopPropagation
def get_folder_path():
file = answer("Folder path to store the results.", default="/tmp/pytgbotapi/")
if file:
try:
file = abspath(file)
mkdir_p(file)
with open(path_join(file, "__init__.py"), "w") as f:
f.write(FILE_HEADER)
# end with
except IOError:
pass
# end try
# end if file
return file
def get_folder_path():
file = answer("Folder path to store the results.", default="/tmp/pytgbotapi/")
if file:
try:
file = abspath(file)
mkdir_p(file)
with open(path_join(file, "__init__.py"), "w") as f:
f.write(FILE_HEADER)
# end with
except IOError:
pass
# end try
# end if file
return file
def output(folder, results, html_content=None):
can_quit = False
do_overwrite = confirm(
"Can the folder {path} be overwritten?".format(path=folder))
print("vvvvvvvvv")
while not can_quit:
if do_overwrite:
try:
import Send2Trash
Send2Trash.send2trash(folder)
except ImportError:
import shutil
shutil.rmtree(folder)
# end try
# end if
# write crawled data
mkdir_p(folder)
with open(path_join(folder, "api.py"), "w") as f:
f.write("[\n ")
f.write(",\n ".join([repr(result) for result in results]))
f.write("\n]")
# end for
# end with
if html_content:
with open(path_join(folder, "api.html"), "wb") as f:
f.write(html_content)
# end with
# end if
# write templates
try:
safe_to_file(folder, results)
except TemplateError as e:
if isinstance(e, TemplateSyntaxError):
logger.exception("Template error at {file}:{line}".format(
file=e.filename, line=e.lineno))
else:
logger.exception("Template error.")
# end if
# end try
print("Writen to file.")
can_quit = not confirm("Write again after reloading templates?",
default=True)
print("#########")
print("Exit.")
def calc_path_and_create_folders(folder, import_path, create_folder=True):
"""
calculate the path and create the needed folders
>>> calc_path_and_create_folders(folder='/somewhere/', import_path='foo.bar.BarClass', create_folder=False)
'/somewhere/foo/bar/BarClass'
:param import_path: 'foo.bar.BarClass'
:param folder: base folder where we wanna place 'foo.bar.BarClass' in.
"""
file_path = abspath(
path_join(
folder, import_path[:import_path.rfind(".")].replace(
".", folder_seperator) + ".py"))
if create_folder:
mkdir_p(dirname(file_path))
# end if
return file_path
def plot_clustering(Z, clusters, centroids, it):
cluster_colors = []
plt.figure()
plt.autoscale(True)
for i in range(len(centroids)):
if clusters:
X = np.array([Z[x] for x in range(len(Z)) if clusters[x] == i])
else:
X = np.array(Z)
# end if
if len(X) != 0 and clusters:
p = plt.plot(X[:, 0], X[:, 1], 'x', label='cluster #{}'.format(i))
c = p[0].get_color()
c = colorsys.rgb_to_hls(*colors.to_rgb(
c)) # https://stackoverflow.com/a/49601444/3423324
c = colorsys.hls_to_rgb(c[0], 0.5 * (1 - c[1]), c[2])
cluster_colors.append(c)
else:
if not clusters:
plt.plot(X[:, 0],
X[:, 1],
'x',
color='black',
label='initial cluster'.format(i))
# end if
cluster_colors.append(None)
# end if
# end for
for i in range(len(centroids)):
plt.autoscale(False)
plt.plot(centroids[i][0],
centroids[i][1],
'+',
label='centroid #{}'.format(i),
color=cluster_colors[i],
markersize=20,
scaley=False,
scalex=False)
# end for
# plt.legend(loc="lower right")
mkdir_p('out')
file = 'out/%d.png' % it
plt.savefig(file)
print('saved {!r}'.format(file))
def get_folder_path():
default = "/tmp/pytgbotapi/"
candidate = abspath(path_join(dirname(abspath(__file__)), 'output'))
print(candidate)
if exists(candidate) and isdir(candidate):
default = candidate
# end if
file = answer("Folder path to store the results.", default=default)
if file:
try:
file = abspath(file)
mkdir_p(file)
with open(path_join(file, "__init__.py"), "w") as f:
f.write(FILE_HEADER)
# end with
except IOError:
pass
# end try
# end if file
return file
def safe_to_file(folder, results):
"""
Receives a list of results (type :class:`Clazz` or :class:`Function`), and put them into the right files in :var:`folder`
:param folder: Where the files should be in.
:type folder: str
:param results: A list of :class:`Clazz` or :class:`Function` objects, which will be used to calculate the source code.
:type results: Union(Clazz, Function)
"""
functions = []
message_send_functions = []
clazzes = {} # "filepath": [Class, Class, ...]
all_the_clazzes = []
# split results into functions and classes
for result in results:
assert isinstance(result, (Clazz, Function))
if isinstance(result, Clazz):
import_path = get_type_path(result.clazz)
import_path = import_path.rstrip(".")
file_path = calc_path_and_create_folders(folder, import_path)
result.filepath = file_path
if file_path not in clazzes:
clazzes[file_path] = []
clazzes[file_path].append(result)
all_the_clazzes.append(result)
else:
assert isinstance(result, Function)
import_path = "pytgbot.bot."
file_path = calc_path_and_create_folders(folder, import_path)
result.filepath = file_path
functions.append(result)
if result.name.startswith('send_'):
import_path = "teleflask_messages."
file_path = calc_path_and_create_folders(folder, import_path)
result2 = safe_eval(
repr(result),
SAVE_VALUES) # serialize + unserialize = deepcopy
result2.filepath = file_path
message_send_functions.append(result2)
# end if
# end if
# end for
bot_template = get_template("bot.template")
clazzfile_template = get_template("classfile.template")
teleflask_messages_template = get_template(
"teleflask_messages_file.template")
typehints_template = get_template("typehintsfile.template")
telegram_bot_api_server_funcs_template = get_template(
"telegram_bot_api_server/funcs.template")
telegram_bot_api_server_class_template = get_template(
"telegram_bot_api_server/classes.template")
mkdir_p(path_join(folder, 'telegram_bot_api_server', 'generated'))
if all_the_clazzes:
txt = telegram_bot_api_server_class_template.render(
clazzes=all_the_clazzes)
render_file_to_disk(
path_join(folder, 'telegram_bot_api_server', 'generated',
'models.py'), txt)
# end if
for path, clazz_list in clazzes.items():
clazz_imports = set()
for clazz_ in clazz_list:
assert isinstance(clazz_, Clazz)
assert isinstance(clazz_.parent_clazz, Type)
clazz_imports.add(clazz_.parent_clazz.as_import)
# end for
clazz_imports = list(clazz_imports)
clazz_imports.sort()
is_sendable = ("sendable" in path)
try:
txt = clazzfile_template.render(clazzes=clazz_list,
imports=clazz_imports,
is_sendable=is_sendable)
txt = txt.replace("\t", " ")
render_file_to_disk(path, txt)
except IOError:
raise # lol
# end try
try:
txt = typehints_template.render(clazzes=clazz_list,
imports=clazz_imports,
is_sendable=is_sendable)
txt = txt.replace("\t", " ")
render_file_to_disk(path + "i",
txt) # "ponies.py" + "i" => "ponies.pyi"
except IOError:
raise # lol
# end try
try:
txt = typehints_template.render(clazzes=clazz_list,
imports=clazz_imports,
is_sendable=is_sendable)
txt = txt.replace("\t", " ")
render_file_to_disk(path + "i",
txt) # "ponies.py" + "i" => "ponies.pyi"
except IOError:
raise # lol
# end try
# end for classes
if functions:
txt = bot_template.render(functions=functions)
render_file_to_disk(functions[0].filepath, txt)
imports = set()
imports.add(('enum', 'Enum'))
imports.add(('typing', 'Union, List, Optional'))
imports.add(('fastapi', 'APIRouter, HTTPException'))
imports.add(('telethon', 'TelegramClient'))
imports.add(('serializer', 'to_web_api, get_entity'))
imports.add(('fastapi.params', 'Query'))
imports.add(('telethon.errors', 'BotMethodInvalidError'))
imports.add(('telethon.tl.types', 'TypeSendMessageAction'))
imports.add(('telethon.client.chats', '_ChatAction'))
imports.add(('luckydonaldUtils.logger', 'logging'))
imports.add(('telethon.tl.functions.messages', 'SetTypingRequest'))
for function in functions:
function: Function
for the_import in function.imports:
the_import: Import
imports.add((the_import.path, the_import.name))
# end for
# end for
# https://stackoverflow.com/a/613218/3423324#how-do-i-sort-a-dictionary-by-value
# https://stackoverflow.com/a/4659539/3423324#how-to-sort-by-length-of-string-followed-by-alphabetical-order
imports_sorted = [
"from " + path + ' import ' + name
for path, name in sorted(imports,
key=lambda item: (-len(item[0]), item[0],
-len(item[1]), item[1]))
]
# imports_sorted.sort(key=lambda item: (-len(item), item))
txt = telegram_bot_api_server_funcs_template.render(
functions=functions, imports=imports_sorted)
render_file_to_disk(
path_join(folder, 'telegram_bot_api_server', 'generated',
'funcs.py'), txt)
# end if
if message_send_functions:
txt = teleflask_messages_template.render(
functions=message_send_functions)
render_file_to_disk(message_send_functions[0].filepath, txt)
# end if
if message_send_functions:
txt = teleflask_messages_template.render(
functions=message_send_functions)
render_file_to_disk(message_send_functions[0].filepath, txt)
for game in GAMES
}
GAMES = GAMES_BY_ID.values()
GAMES_COUNT = len(GAMES)
GAMES_COUNT_LEN = len(str(GAMES_COUNT))
DOWNLOAD_TOTAL_SIZE = 0
DOWNLOADS: List[URLData] = [] # file: url
for i, game in enumerate(GAMES):
part = f"[{i + 1:0>{GAMES_COUNT_LEN}}/{GAMES_COUNT}]"
logger.debug(
f'{part}: Found Game {game.title!r} with {len(game.downloads)} downloads.'
)
game_path = path.join(DOWNLOAD_DIR, sanitize_name(game.title))
mkdir_p(game_path)
for platform, download in game.downloads.items():
download_path = path.join(game_path, sanitize_name(platform))
mkdir_p(download_path)
for download_type, download_filename in download.url.items():
download_file_path = path.join(download_path,
sanitize_name(download_filename))
auth_request_data = {
"machine_name": download.machine_name,
"filename": download_filename
}
if download_type == TYPE_WEB:
for meta_file in ('md5', 'sha1'):
value = getattr(download, meta_file)
if value is None:
continue
def calc_path_and_create_folders(folder, import_path):
""" calculate the path and create the needed folders """
file_path = abspath(path_join(folder, import_path[:import_path.rfind(".")].replace(".", folder_seperator) + ".py"))
mkdir_p(dirname(file_path))
return file_path
