def main(argv=None):
import getpass
if argv is None:
argv = sys.argv[1:] # argv[0] is program name
if argv:
url = argv[0]
else:
url = answer("Url to download")
if not url:
print("no url given.")
sys.exit(0)
if "use_login" not in settings or settings.use_login == -1:
use_login = 1 if confirm(
"Use login with pony.fm command?\nYou can automatically fave and comment songs, and/or your download stats will be updated.",
default=False) else 0
settings.use_login = use_login
settings.save_settings()
logger.debug("Saved use_login.")
if settings.use_login == 1:
if "ponyfm_user" not in settings or not settings.ponyfm_user:
default_username = getpass.getuser()
username = answer("Username for pony.fm (poniverse.net)",
default=default_username)
settings.ponyfm_user = username
settings.save_settings()
logger.debug("Saved user.")
else:
username = settings.get("ponyfm_user")
#end if
store = Store(IDENTIFIER)
if "ponyfm_pass" not in settings or not settings.ponyfm_pass:
print(
"Password for pony.fm (If your shell can't hide the input, a warning will be displayed)"
)
__password = store.encrypt(getpass.getpass())
if confirm("Safe password?", default=True):
settings.ponyfm_pass = __password
settings.save_settings()
logger.info("Saved password.")
else:
__password = settings.get("ponyfm_pass")
#end if
ponyfm = PonyFM(username, password=__password, key=store.key)
else:
ponyfm = PonyFM
file = do_complete_load_if_matches(ponyfm, url, cover_as_file=True)
print("File written to {path}".format(path=file))
if confirm("Open folder?", default=True):
open_file_folder(file)
def main():
folder = get_folder_path()
mode = confirm("Load from a dump instead of the API Docs?")
if not mode: # API
load_from_api(folder)
else: # Dump
load_from_dump(folder)
def login(self):
if not self.username or not self.__password or not self.__key:
raise AssertionError("Please set username/password with `.set_credentials(user, pass)` first.")
session = requests.Session()
poniverse_ponyfm_url="https://poniverse.net/oauth/authorize?%2Foauth%2Fauthorize&client_id=3&redirect_uri=https%3A%2F%2Fpony.fm%2Fauth%2Foauth&response_type=code&state=login"
poniverse_start = 'https://poniverse.net/oauth/login'
poniverse_login_url = 'https://poniverse.net/login'
pony_fm_url = 'https://pony.fm'
r = session.get(poniverse_ponyfm_url, headers=HEADERS, verify=False)
assert (r.url == poniverse_start)
token = BeautifulSoup(r.text, "html.parser").findAll(attrs={'name' : '_token'})[0].get('value').encode()
payload = {'_token' : token,
'username' : self.username,
'password' : Store(IDENTIFIER, self.__key).decrypt(self.__password),
'checkbox' : '1',
'submit' : '',
}
r = session.post(poniverse_login_url, data=payload, headers=HEADERS, verify=False, allow_redirects=True,cookies=session.cookies)
if r.url == pony_fm_url: # XXX
logger.info("Logged in")
else:
from luckydonaldUtils.interactions import confirm
if confirm("Login failed. Reset password?", default=True):
from ...utilities.settings import settings
settings.ponyfm_user = ""
settings.ponyfm_pass = ""
settings.save_settings()
import sys
sys.exit(77)
raise AssertionError("login failed")
self.session = session
return session
def command_line_input():
from luckydonaldUtils.interactions import confirm, answer
do_func = confirm("Choose between generating function or class. Do you want a function?", True)
if do_func:
command = answer("Command (the Title)")
description = answer("Description")
link = answer("The link on the api page")
params_string = "---"
params_strings = []
while params_string != "":
params_string = answer("Parameters (sepereated by tabs, and new lines)\nParameters Type Required Description", "")
if params_string and not params_string.strip() == "":
params_strings.append(params_string)
# end if
# end while
returns = answer("Textual description what the function returns", "On success, the sent Message is returned.")
return_type = answer("Return type", "Message")
print("\n")
func(command, description, link, params_string, returns , return_type)
else:
clazze = answer("Class name")
parent_clazz = answer("Parent class name", "object")
description = answer("Description")
link = answer("The link on the api page")
params_string = "--"
params_strings = []
while params_string != "":
params_string = answer("Parameters (sepereated by tabs, and new lines)\nParameters Type Description", "")
if params_string and not params_string.strip() == "":
params_strings.append(params_string)
# end if
# end while
print("\n")
clazz(clazze, parent_clazz, description, link, "\n".join(params_strings))
def output(folder, results, html_content=None):
can_quit = False
do_overwrite = confirm(
"Can the folder {path} be overwritten?".format(path=folder))
print("vvvvvvvvv")
while not can_quit:
if do_overwrite:
try:
import Send2Trash
Send2Trash.send2trash(folder)
except ImportError:
import shutil
shutil.rmtree(folder)
# end try
# end if
# write crawled data
mkdir_p(folder)
with open(path_join(folder, "api.py"), "w") as f:
f.write("[\n ")
f.write(",\n ".join([repr(result) for result in results]))
f.write("\n]")
# end for
# end with
if html_content:
with open(path_join(folder, "api.html"), "wb") as f:
f.write(html_content)
# end with
# end if
# write templates
try:
safe_to_file(folder, results)
except TemplateError as e:
if isinstance(e, TemplateSyntaxError):
logger.exception("Template error at {file}:{line}".format(
file=e.filename, line=e.lineno))
else:
logger.exception("Template error.")
# end if
# end try
print("Writen to file.")
can_quit = not confirm("Write again after reloading templates?",
default=True)
print("#########")
print("Exit.")
def load_api_definitions():
folder = get_folder_path()
mode = confirm("Offline Mode: Load from a dump instead of the API Docs?")
if not mode: # API
results, html_document = load_from_html(folder)
else: # Dump
results, html_document = load_from_dump(folder)
# end def
return folder, html_document, results
def main(argv=None):
import getpass
if argv is None:
argv = sys.argv[1:] # argv[0] is program name
if argv:
url = argv[0]
else:
url = answer("Url to download")
if not url:
print("no url given.")
sys.exit(0)
if "use_login" not in settings or settings.use_login == -1:
use_login = 1 if confirm("Use login with pony.fm command?\nYou can automatically fave and comment songs, and/or your download stats will be updated.", default=False) else 0
settings.use_login = use_login
settings.save_settings()
logger.debug("Saved use_login.")
if settings.use_login == 1:
if "ponyfm_user" not in settings or not settings.ponyfm_user:
default_username = getpass.getuser()
username = answer("Username for pony.fm (poniverse.net)", default=default_username)
settings.ponyfm_user = username
settings.save_settings()
logger.debug("Saved user.")
else:
username = settings.get("ponyfm_user")
#end if
store = Store(IDENTIFIER)
if "ponyfm_pass" not in settings or not settings.ponyfm_pass:
print("Password for pony.fm (If your shell can't hide the input, a warning will be displayed)")
__password = store.encrypt(getpass.getpass())
if confirm("Safe password?", default=True):
settings.ponyfm_pass = __password
settings.save_settings()
logger.info("Saved password.")
else:
__password = settings.get("ponyfm_pass")
#end if
ponyfm = PonyFM(username, password=__password, key=store.key)
else:
ponyfm = PonyFM
file = do_complete_load_if_matches(ponyfm, url, cover_as_file=True)
print("File written to {path}".format(path=file))
if confirm("Open folder?", default=True):
open_file_folder(file)
def login(self):
if not self.username or not self.__password or not self.__key:
raise AssertionError(
"Please set username/password with `.set_credentials(user, pass)` first."
)
session = requests.Session()
poniverse_ponyfm_url = "https://poniverse.net/oauth/authorize?%2Foauth%2Fauthorize&client_id=3&redirect_uri=https%3A%2F%2Fpony.fm%2Fauth%2Foauth&response_type=code&state=login"
poniverse_start = 'https://poniverse.net/oauth/login'
poniverse_login_url = 'https://poniverse.net/login'
pony_fm_url = 'https://pony.fm'
r = session.get(poniverse_ponyfm_url, headers=HEADERS, verify=False)
assert (r.url == poniverse_start)
token = BeautifulSoup(r.text, "html.parser").findAll(
attrs={'name': '_token'})[0].get('value').encode()
payload = {
'_token': token,
'username': self.username,
'password': Store(IDENTIFIER, self.__key).decrypt(self.__password),
'checkbox': '1',
'submit': '',
}
r = session.post(poniverse_login_url,
data=payload,
headers=HEADERS,
verify=False,
allow_redirects=True,
cookies=session.cookies)
if r.url == pony_fm_url: # XXX
logger.info("Logged in")
else:
from luckydonaldUtils.interactions import confirm
if confirm("Login failed. Reset password?", default=True):
from ...utilities.settings import settings
settings.ponyfm_user = ""
settings.ponyfm_pass = ""
settings.save_settings()
import sys
sys.exit(77)
raise AssertionError("login failed")
self.session = session
return session
def main():
bot_template = get_template("bot.template")
folder = get_folder_path()
filter = get_filter()
document = requests.get(BASE_URL)
bs = BeautifulSoup(document.content)
results = []
for h in bs.select("#dev_page_content > h4"):
print("------")
anchor = h.find(lol1)
if not anchor or not anchor.has_attr("name"):
continue
link = "{base_url}#{anchor}".format(base_url=BASE_URL,
anchor=anchor["name"])
title = h.text
descr = []
table_type, param_strings = None, None
print("title: " + title)
print("link: " + link)
if filter and title not in filter:
print("Skipping {title}, filtered.".format(title=title))
continue
# logger.debug(h)
type_strings = []
default_returns = []
for sibling in h.next_siblings:
if sibling == "\n":
continue
if sibling.name in ["p", "blockquote"]:
if "return" in sibling.text.lower():
parts_splitted = []
is_first_element = True # truein string,
for x in sibling.children:
if isinstance(x, NavigableString):
if is_first_element: # Start of a new sentence => new list
parts_splitted.extend([[foo.lstrip()]
for foo in x.split(".")
])
is_first_element = False
else: # not = in the middle of a sentence => append
parts_splitted[len(parts_splitted) - 1].append(
x.split(".", maxsplit=1)[0])
parts_splitted.extend(
[[foo] for foo in x.split(".")[1:]])
is_first_element = False
is_first_element = x.strip().endswith(".")
else:
obj = None
if x.name in ["a", "em"]:
obj = x
else:
obj = x.text
# end if
if is_first_element: # if it is at the beginning of the sentence.
parts_splitted.append([obj])
is_first_element = False
else:
parts_splitted[len(parts_splitted) -
1].append(obj)
# end if
# end for
# end for
returns__ = [] # array of strings
return_text__ = [
] # array if strings. one item = one sentence. Not ending with a dot.
is_array = False
for lol_part in parts_splitted:
has_return = False
returns_ = []
return_text_ = ""
for lol_part_part in lol_part:
if isinstance(lol_part_part, str):
return_text_ += lol_part_part
if lol_part_part.strip().lower().endswith(
"array of"):
is_array = True
if "return" in lol_part_part.lower():
has_return = True
# end if
else: # not str
return_text_ += lol_part_part.text
if is_array:
returns_.append("list of " +
lol_part_part.text)
is_array = False
else:
returns_.append(lol_part_part.text)
# end for
if has_return: # append, so we can have multible sentences.
return_text__.append(return_text_.strip())
returns__.extend(returns_)
# end if
# end for
if return_text__ or returns__: # finally set it.
default_returns = (". ".join(return_text__).strip(),
" or ".join(returns__).strip())
# end if
# end if
descr.append(sibling.text)
elif sibling.name == "table":
assert sibling.has_attr(
"class") and "table" in sibling["class"]
table_type, param_strings = parse_table(sibling)
elif sibling.name == "h4":
break
elif sibling.name == "h3":
break
elif sibling.name == "hr": # end of page
break
else:
print("unknown: " + sibling.name)
# end if
# end for
if not all([link, title, descr]):
print("Skipped: Missing link, title or description")
continue
if not all([table_type, param_strings]):
if title not in WHITELISTED_FUNCS:
print(
"Skipped. Has no table with Parameters or Fields.\n"
"Also isn't a whitelisted function in `code_generator_settings.WHITELISTED_FUNCS`."
)
continue
# -> else: is in WHITELISTED_FUNCS:
table_type = "func"
# end if
descr = "\n".join(descr)
print("descr: " + repr(descr))
params_string = "\n".join(
param_strings
) if param_strings else None # WHITELISTED_FUNCS have no params
if table_type == "func":
seems_valid = False
if len(default_returns) != 2:
if "return" in descr.lower():
default_returns = ["", "Message"]
default_returns[0] = [
x for x in descr.split(".") if "return" in x.lower()
][0].strip()
seems_valid = len(default_returns[0].split(".")) == 1
default_returns[1] = " or ".join(
type_strings) if type_strings else "Message"
default_returns[1] = as_types(default_returns[1],
"returns")
else:
default_returns = ("On success, True is returned", "True")
# end if "return" in description
else:
seems_valid = len(default_returns[0].split(".")) == 1
# end if default set
if not seems_valid:
returns = answer(
"Textual description what the function returns",
default_returns[0])
return_type = answer("Return type", default_returns[1])
if isinstance(return_type, str):
return_type = as_types(return_type, "return type")
# end if
else:
returns = default_returns[0]
return_type = default_returns[1]
# end if
logger.debug("\n")
result = func(title,
descr,
link,
params_string,
returns=returns,
return_type=return_type)
results.append(result)
elif table_type == "class":
if title in CLASS_TYPE_PATHS:
parent_clazz = CLASS_TYPE_PATHS[title][
CLASS_TYPE_PATHS__PARENT]
print("superclass: " + parent_clazz)
else:
parent_clazz = answer("Parent class name", "TgBotApiObject")
# end if
result = clazz(title, parent_clazz, descr, link, params_string)
results.append(result)
# end if
# end for
can_quit = False
do_overwrite = confirm(
"Can the folder {path} be overwritten?".format(path=folder))
print("vvvvvvvvv")
while not can_quit:
if do_overwrite:
try:
import Send2Trash
Send2Trash.send2trash(folder)
except ImportError:
import shutil
shutil.rmtree(folder)
# end try
# end if
try:
safe_to_file(folder, results)
print("Writen to file.")
except TemplateError as e:
if isinstance(e, TemplateSyntaxError):
logger.exception("Template error at {file}:{line}".format(
file=e.filename, line=e.lineno))
else:
logger.exception("Template error.")
# end if
# end try
can_quit = not confirm("Write again after reloading templates?",
default=True)
print("#########")
print("Exit.")
def main():
bot_template = get_template("bot.template")
folder = get_folder_path()
filter = get_filter()
document = requests.get(BASE_URL)
bs = BeautifulSoup(document.content)
results = []
for h in bs.select("#dev_page_content > h4"):
print("------")
anchor = h.find(lol1)
if not anchor or not anchor.has_attr("name"):
continue
link = "{base_url}#{anchor}".format(base_url=BASE_URL, anchor=anchor["name"])
title = h.text
descr = []
table_type, param_strings = None, None
print("title: " + title)
print("link: " + link)
if filter and title not in filter:
print("Skipping {title}, filtered.".format(title=title))
continue
# logger.debug(h)
type_strings = []
default_returns = []
for sibling in h.next_siblings:
if sibling == "\n":
continue
if sibling.name in ["p", "blockquote"]:
if "return" in sibling.text.lower():
parts_splitted = []
is_first_element = True # truein string,
for x in sibling.children:
if isinstance(x, NavigableString):
if is_first_element: # Start of a new sentence => new list
parts_splitted.extend([[foo.lstrip()] for foo in x.split(".")])
is_first_element = False
else: # not = in the middle of a sentence => append
parts_splitted[len(parts_splitted)-1].append(x.split(".", maxsplit=1)[0])
parts_splitted.extend([[foo] for foo in x.split(".")[1:]])
is_first_element = False
is_first_element = x.strip().endswith(".")
else:
obj = None
if x.name in ["a", "em"]:
obj = x
else:
obj = x.text
# end if
if is_first_element: # if it is at the beginning of the sentence.
parts_splitted.append([obj])
is_first_element = False
else:
parts_splitted[len(parts_splitted)-1].append(obj)
# end if
# end for
# end for
returns__ = [] # array of strings
return_text__ = [] # array if strings. one item = one sentence. Not ending with a dot.
is_array = False
for lol_part in parts_splitted:
has_return = False
returns_ = []
return_text_ = ""
for lol_part_part in lol_part:
if isinstance(lol_part_part, str):
return_text_ += lol_part_part
if lol_part_part.strip().lower().endswith("array of"):
is_array = True
if "return" in lol_part_part.lower():
has_return = True
# end if
else: # not str
return_text_ += lol_part_part.text
if is_array:
returns_.append("list of " + lol_part_part.text)
is_array = False
else:
returns_.append(lol_part_part.text)
# end for
if has_return: # append, so we can have multible sentences.
return_text__.append(return_text_.strip())
returns__.extend(returns_)
# end if
# end for
if return_text__ or returns__: # finally set it.
default_returns = (". ".join(return_text__).strip(), " or ".join(returns__).strip())
# end if
# end if
descr.append(sibling.text)
elif sibling.name == "table":
assert sibling.has_attr("class") and "table" in sibling["class"]
table_type, param_strings = parse_table(sibling)
elif sibling.name == "h4":
break
elif sibling.name == "h3":
break
elif sibling.name == "hr": # end of page
break
else:
print("unknown: " + sibling.name)
# end if
# end for
if not all([link, title, descr]):
print("Skipped: Missing link, title or description")
continue
if not all([table_type, param_strings]):
if title not in WHITELISTED_FUNCS:
print("Skipped. Has no table with Parameters or Fields.\n"
"Also isn't a whitelisted function in `code_generator_settings.WHITELISTED_FUNCS`.")
continue
# -> else: is in WHITELISTED_FUNCS:
table_type = "func"
# end if
descr = "\n".join(descr)
print("descr: " + repr(descr))
params_string = "\n".join(param_strings) if param_strings else None # WHITELISTED_FUNCS have no params
if table_type == "func":
seems_valid = False
if len(default_returns) != 2:
if "return" in descr.lower():
default_returns = ["", "Message"]
default_returns[0] = [x for x in descr.split(".") if "return" in x.lower()][0].strip()
seems_valid = len(default_returns[0].split(".")) == 1
default_returns[1] = " or ".join(type_strings) if type_strings else "Message"
default_returns[1] = as_types(default_returns[1], "returns")
else:
default_returns = ("On success, True is returned", "True")
# end if "return" in description
else:
seems_valid = len(default_returns[0].split(".")) == 1
# end if default set
if not seems_valid:
returns = answer("Textual description what the function returns", default_returns[0])
return_type = answer("Return type", default_returns[1])
if isinstance(return_type, str):
return_type = as_types(return_type, "return type")
# end if
else:
returns = default_returns[0]
return_type = default_returns[1]
# end if
logger.debug("\n")
result = func(title, descr, link, params_string, returns=returns, return_type=return_type)
results.append(result)
elif table_type == "class":
if title in CLASS_TYPE_PATHS:
parent_clazz = CLASS_TYPE_PATHS[title][CLASS_TYPE_PATHS__PARENT]
print("superclass: " + parent_clazz)
else:
parent_clazz = answer("Parent class name", "TgBotApiObject")
# end if
result = clazz(title, parent_clazz, descr, link, params_string)
results.append(result)
# end if
# end for
can_quit = False
do_overwrite = confirm("Can the folder {path} be overwritten?".format(path=folder))
print("vvvvvvvvv")
while not can_quit:
if do_overwrite:
try:
import Send2Trash
Send2Trash.send2trash(folder)
except ImportError:
import shutil
shutil.rmtree(folder)
# end try
# end if
try:
safe_to_file(folder, results)
with open(path_join(folder, "api.html"), "wb") as f:
f.write(document.content)
# end if
print("Writen to file.")
except TemplateError as e:
if isinstance(e, TemplateSyntaxError):
logger.exception("Template error at {file}:{line}".format(file=e.filename, line=e.lineno))
else:
logger.exception("Template error.")
# end if
# end try
can_quit = not confirm("Write again after reloading templates?", default=True)
print("#########")
print("Exit.")
row_string += (" " * (length + 2)) + "|"
# end if
# end for
rows.append(row_string)
# end for
table_string = "\n".join(rows)
print("TABLE:\n")
print(table_string)
from luckydonaldUtils.interactions import confirm
from luckydonaldUtils.encoding import to_binary as b
if confirm('save as file & open in browser?'):
import tempfile
import webbrowser
from html import escape
with tempfile.NamedTemporaryFile(suffix='.html', delete=False) as f:
f.write(b("<pre>"))
f.write(b(escape(table_string)))
f.write(b("</pre>"))
f.flush()
print(f.name)
webbrowser.open('file://'+f.name)
# end with
# end if
possibilities = permutations(funcs.keys(), len(funcs))
