def tidy(self):
""" Remove downvoted comments to comply with Rediquette. """
replies = self.bot.user.me().comments.new(limit=100)
for reply in replies:
if reply.score < 0:
with open("deleted.csv", "a", encoding="UTF-8") as removed:
deleted = clevercsv.writer(removed)
if removed.tell() == 0:
deleted.writerow([
"Comment", "Parent", "Thread", "Subreddit", "Time",
"Score"
])
deleted.writerow([
f"{reply.body}", f"{reply.parent().body}",
f"{reply.submission.title}", f"{reply.subreddit}",
f"{pendulum.from_timestamp(reply.created_utc)}",
f"{reply.score}"
])
reply.delete()
def _write_error_test(self, exc, fields, **kwargs):
with tempfile.TemporaryFile("w+", newline="", prefix="ccsv_") as fp:
writer = clevercsv.writer(fp, **kwargs)
with self.assertRaises(exc):
writer.writerow(fields)
fp.seek(0)
self.assertEqual(fp.read(), "")
def _write_test(self, fields, expect, **kwargs):
with tempfile.TemporaryFile("w+", newline="") as fileobj:
writer = clevercsv.writer(fileobj, **kwargs)
writer.writerow(fields)
fileobj.seek(0)
self.assertEqual(fileobj.read(),
expect + writer.dialect.lineterminator)
def _write_tmpfile(self, table, dialect):
""" Write a table to a temporary file using specified dialect """
tmpfd, tmpfname = tempfile.mkstemp(prefix="ccsv_", suffix=".csv")
tmpid = os.fdopen(tmpfd, "w")
w = writer(tmpid, dialect=dialect)
w.writerows(table)
tmpid.close()
return tmpfname
def _read_test(self, table, dialect):
tmpfd, tmpfname = tempfile.mkstemp()
tmpid = os.fdopen(tmpfd, "w")
w = writer(tmpid, dialect=dialect)
w.writerows(table)
tmpid.close()
exp = [list(map(str, r)) for r in table]
try:
self.assertEqual(exp, wrappers.read_csv(tmpfname))
finally:
os.unlink(tmpfname)
def send_message(messages, header="Info"):
hdl, tmpfname = tempfile.mkstemp(prefix="clevercsv_", suffix=".csv")
table = [[header]]
if isinstance(messages, str):
table.append([messages])
else:
for line in messages:
table.append([line])
with os.fdopen(hdl, "w") as fp:
writer = clevercsv.writer(fp)
writer.writerows(table)
print(tmpfname)
sys.stdout.flush()
def _df_test(self, table, dialect, **kwargs):
tmpfd, tmpfname = tempfile.mkstemp(prefix="ccsv_", suffix=".csv")
tmpid = os.fdopen(tmpfd, "w", encoding=kwargs.get("encoding"))
w = writer(tmpid, dialect=dialect)
w.writerows(table)
tmpid.close()
exp_df = pd.DataFrame.from_records(table[1:], columns=table[0])
df = wrappers.read_dataframe(tmpfname)
try:
self.assertTrue(df.equals(exp_df))
finally:
os.unlink(tmpfname)
def _df_test(self, table, dialect):
tmpfd, tmpfname = tempfile.mkstemp()
tmpid = os.fdopen(tmpfd, "w")
w = writer(tmpid, dialect=dialect)
w.writerows(table)
tmpid.close()
exp_df = pd.DataFrame.from_records(table[1:], columns=table[0])
df = wrappers.csv2df(tmpfname)
try:
self.assertTrue(df.equals(exp_df))
finally:
os.unlink(tmpfname)
def catalog():
"""
Scrapes metadata of science fiction and fantasy literary works
from The Internet Speculative Fiction Database and stores them in
a CSV file. If site structure changes significantly, code may stop
functioning properly.
"""
card = 0
for entry in range(199000):
try:
card += 1
page = requests.get(
f"http://www.isfdb.org/cgi-bin/title.cgi?{card}")
parsed = bs4.BeautifulSoup(
page.content,
"html.parser")
content = parsed.find(
id = "content").text.split("\n")
content_string = "##".join(content)
content_title = re.search(
"Title:\\s+[^#]+", content_string).group(0)
title = content_title.split(": ")[1]
content_author = re.search(
"Author:##[^#]+", content_string).group(0)
author = content_author.split("##")[1]
content_date = re.search(
"Date:\\s+\\d+\\-\\d+\\-\\d+", content_string).group(0)
pubdate = content_date.split(" ")[1]
content_type = re.search(
"Type:\\s+[^#]+", content_string).group(0)
booktype = content_type.split(": ")[1]
accepted_booktype = [
"NOVEL",
"SHORTFICTION",
"COLLECTION",
"ANTHOLOGY",
"OMNIBUS",
"POEM",
"NONFICTION",
"ESSAY"]
with open(
"SFF_Dataset.csv", "a", encoding="UTF-8") as sff:
dataset = clevercsv.writer(sff)
if sff.tell() == 0:
dataset.writerow(
["Title",
"Author",
"Publication Date",
"Type"])
if booktype in accepted_booktype:
dataset.writerow(
[title,
author,
pubdate,
booktype])
except:
print(
f"Skipping entry no. {card}: Empty article.", "\n" *4)
continue
def interleaved_neighborhood():
initialize = model_check(param_gen())
global_best = {
"score": initialize["score"],
"parameters": initialize["parameters"]
}
with open("values.csv", "a", encoding="UTF-8") as valuefile:
values = clevercsv.writer(valuefile)
values.writerow([global_best["score"], global_best["parameters"]])
horizon = "local"
while True:
try:
if horizon == "local":
perturb = {
key: abs(gauss(value, uniform(0.1, 2.0)))
if type(value) != str else value
for key, value in global_best["parameters"].items()
}
rounded = {
key: round(value)
if key in ["n_estimators", "num_leaves"] else value
for key, value in perturb.items()
}
local_solution = model_check(rounded)
if local_solution["score"] < global_best["score"]:
with open("values.csv", "a",
encoding="UTF-8") as valuefile:
values = clevercsv.writer(valuefile)
values.writerow([
local_solution["score"],
local_solution["parameters"]
])
global_best["score"] = local_solution["score"]
global_best["parameters"] = local_solution["parameters"]
continue
else:
horizon = "global"
continue
else:
random_solution = model_check(param_gen())
if random_solution["score"] < global_best["score"]:
with open("values.csv", "a",
encoding="UTF-8") as valuefile:
values = clevercsv.writer(valuefile)
values.writerow([
random_solution["score"],
random_solution["parameters"]
])
global_best["score"] = random_solution["score"]
global_best["parameters"] = random_solution["parameters"]
horizon = "local"
continue
else:
horizon = "local"
continue
except Exception as error:
print(error)
def writerAssertEqual(self, input, expected_result):
with tempfile.TemporaryFile("w+", newline="", prefix="ccsv_") as fp:
writer = clevercsv.writer(fp, dialect=self.dialect)
writer.writerows(input)
fp.seek(0)
self.assertEqual(fp.read(), expected_result)
def core(self):
"""
Main function. Watch Reddit comment stream for title + author mentions.
Check YouTube for audiobook and reply with link if suitable one found.
"""
comments = self.bot.subreddit("all").stream.comments(
skip_existing=True)
for comment in comments:
text = comment.body.lower().replace(".", "")
for card in self.catalog:
if (card[1].lower() in text and card[0].lower() in text
and not comment.submission.id in self.responded
and not comment.subreddit.user_is_banned):
self.get_info(card)
if not self.details:
break
audio = ["audiobook", "audio book"]
author_format = [
name.lower() for name in card[1].split(" ")
if len(name) >= 3
]
if (self.details["duration"] > 10800 and card[0].lower()
in self.details["title"].lower()
and any(item in self.details["title"].lower()
for item in audio)
and all(item in self.details["title"].lower()
for item in author_format)):
saw_the_sign = (
"""[^(Source Code)](https://capybasilisk.com/posts/"""
"""2020/04/speculative-fiction-bot/) """
"""^| [^(Feedback)](https://www.reddit.com/message/"""
"""compose?to=Capybasilisk&subject=Robot) """
"""^| [^(Programmer)](https://www.reddit.com/u/"""
"""capybasilisk) """
"""^| ^(Downvote To Remove) """
"""^| ^(Version 1.4.0) """
"""^| ^(Support Robot Rights!)""")
comment.reply(
f"""Hi. You just mentioned *{card[0]}* by """
f"""{card[1]}.\n\nI've found an audiobook of """
"""that novel on YouTube. You can listen to it here"""
f""":\n\n[YouTube | {self.details['title']}]"""
f"""({self.details['webpage_url']})\n\n*I\'m a bot that """
"""searches YouTube for science fiction and fantasy"""
f""" audiobooks.*\n***\n{saw_the_sign}""")
self.responded.append(comment.submission.id)
with open("activity.csv", "a",
encoding="UTF-8") as actlog:
activity = clevercsv.writer(actlog)
if actlog.tell() == 0:
activity.writerow([
"Book", "Comment", "Author", "Thread",
"Subreddit", "Time"
])
activity.writerow([
f"{card[0]} by {card[1]}", f"{comment.body}",
f"{comment.author}",
f"{comment.submission.title}",
f"{comment.subreddit}",
f"{pendulum.now().to_datetime_string()}"
])
self.details = None
break
break
if pendulum.now().to_time_string().endswith("0:00"):
self.tidy()
def square_neighborhood():
global_best = 17000
best_parameters = {}
neighborhood_search = 0
while True:
try:
random_parameters = {
"boosting_type": randrange(2),
"n_estimators": randrange(10000),
"num_leaves": randrange(10000),
"learning_rate": uniform(0.01, 1)}
global_evaluate = model_check(
random_parameters)
if global_evaluate < global_best:
neighborhood_search += round(
abs(
global_evaluate - global_best) ** 2)
global_best = global_evaluate
best_parameters = random_parameters
neighborhood_steps = 0
neighborhood_size = 0.1
with open(
"values.csv", "a", encoding = "UTF-8") as valuefile:
values = clevercsv.writer(valuefile)
values.writerow(
[global_best, best_parameters])
while neighborhood_search > 0:
neighborhood_parameters = {
key:abs(
gauss(
value,
neighborhood_size)) if type(value) == float
else abs(
round(
gauss(
value,
neighborhood_size)))
for key,value in best_parameters.items()}
neighborhood_evaluate = model_check(
neighborhood_parameters)
if neighborhood_evaluate < global_best:
neighborhood_search += abs(
neighborhood_evaluate - global_best) ** 2
global_best = neighborhood_evaluate
best_parameters = neighborhood_parameters
neighborhood_steps = 0
neighborhood_size = 0.1
with open(
"values.csv",
"a", encoding = "UTF-8") as valuefile:
values = clevercsv.writer(valuefile)
values.writerow(
[global_best, best_parameters])
else:
neighborhood_steps += 1
neighborhood_size += 0.0001 * neighborhood_steps
neighborhood_search -= 1
except:
continue
def writerAssertEqual(self, input, expected_result):
with tempfile.TemporaryFile("w+", newline="") as fileobj:
writer = clevercsv.writer(fileobj, dialect=self.dialect)
writer.writerows(input)
fileobj.seek(0)
self.assertEqual(fileobj.read(), expected_result)
def createCSVFile(self, fileName):
self.m_file_path = os.path.join(tempfile.gettempdir(),
fileName + '.csv')
return csv.writer(open(self.m_file_path, "w"),
delimiter=',',
quoting=csv.QUOTE_ALL)
