def tidy(self): """ Remove downvoted comments to comply with Rediquette. """ replies = self.bot.user.me().comments.new(limit=100) for reply in replies: if reply.score < 0: with open("deleted.csv", "a", encoding="UTF-8") as removed: deleted = clevercsv.writer(removed) if removed.tell() == 0: deleted.writerow([ "Comment", "Parent", "Thread", "Subreddit", "Time", "Score" ]) deleted.writerow([ f"{reply.body}", f"{reply.parent().body}", f"{reply.submission.title}", f"{reply.subreddit}", f"{pendulum.from_timestamp(reply.created_utc)}", f"{reply.score}" ]) reply.delete()
def _write_error_test(self, exc, fields, **kwargs): with tempfile.TemporaryFile("w+", newline="", prefix="ccsv_") as fp: writer = clevercsv.writer(fp, **kwargs) with self.assertRaises(exc): writer.writerow(fields) fp.seek(0) self.assertEqual(fp.read(), "")
def _write_test(self, fields, expect, **kwargs): with tempfile.TemporaryFile("w+", newline="") as fileobj: writer = clevercsv.writer(fileobj, **kwargs) writer.writerow(fields) fileobj.seek(0) self.assertEqual(fileobj.read(), expect + writer.dialect.lineterminator)
def _write_tmpfile(self, table, dialect): """ Write a table to a temporary file using specified dialect """ tmpfd, tmpfname = tempfile.mkstemp(prefix="ccsv_", suffix=".csv") tmpid = os.fdopen(tmpfd, "w") w = writer(tmpid, dialect=dialect) w.writerows(table) tmpid.close() return tmpfname
def _read_test(self, table, dialect): tmpfd, tmpfname = tempfile.mkstemp() tmpid = os.fdopen(tmpfd, "w") w = writer(tmpid, dialect=dialect) w.writerows(table) tmpid.close() exp = [list(map(str, r)) for r in table] try: self.assertEqual(exp, wrappers.read_csv(tmpfname)) finally: os.unlink(tmpfname)
def send_message(messages, header="Info"): hdl, tmpfname = tempfile.mkstemp(prefix="clevercsv_", suffix=".csv") table = [[header]] if isinstance(messages, str): table.append([messages]) else: for line in messages: table.append([line]) with os.fdopen(hdl, "w") as fp: writer = clevercsv.writer(fp) writer.writerows(table) print(tmpfname) sys.stdout.flush()
def _df_test(self, table, dialect, **kwargs): tmpfd, tmpfname = tempfile.mkstemp(prefix="ccsv_", suffix=".csv") tmpid = os.fdopen(tmpfd, "w", encoding=kwargs.get("encoding")) w = writer(tmpid, dialect=dialect) w.writerows(table) tmpid.close() exp_df = pd.DataFrame.from_records(table[1:], columns=table[0]) df = wrappers.read_dataframe(tmpfname) try: self.assertTrue(df.equals(exp_df)) finally: os.unlink(tmpfname)
def _df_test(self, table, dialect): tmpfd, tmpfname = tempfile.mkstemp() tmpid = os.fdopen(tmpfd, "w") w = writer(tmpid, dialect=dialect) w.writerows(table) tmpid.close() exp_df = pd.DataFrame.from_records(table[1:], columns=table[0]) df = wrappers.csv2df(tmpfname) try: self.assertTrue(df.equals(exp_df)) finally: os.unlink(tmpfname)
def catalog(): """ Scrapes metadata of science fiction and fantasy literary works from The Internet Speculative Fiction Database and stores them in a CSV file. If site structure changes significantly, code may stop functioning properly. """ card = 0 for entry in range(199000): try: card += 1 page = requests.get( f"http://www.isfdb.org/cgi-bin/title.cgi?{card}") parsed = bs4.BeautifulSoup( page.content, "html.parser") content = parsed.find( id = "content").text.split("\n") content_string = "##".join(content) content_title = re.search( "Title:\\s+[^#]+", content_string).group(0) title = content_title.split(": ")[1] content_author = re.search( "Author:##[^#]+", content_string).group(0) author = content_author.split("##")[1] content_date = re.search( "Date:\\s+\\d+\\-\\d+\\-\\d+", content_string).group(0) pubdate = content_date.split(" ")[1] content_type = re.search( "Type:\\s+[^#]+", content_string).group(0) booktype = content_type.split(": ")[1] accepted_booktype = [ "NOVEL", "SHORTFICTION", "COLLECTION", "ANTHOLOGY", "OMNIBUS", "POEM", "NONFICTION", "ESSAY"] with open( "SFF_Dataset.csv", "a", encoding="UTF-8") as sff: dataset = clevercsv.writer(sff) if sff.tell() == 0: dataset.writerow( ["Title", "Author", "Publication Date", "Type"]) if booktype in accepted_booktype: dataset.writerow( [title, author, pubdate, booktype]) except: print( f"Skipping entry no. {card}: Empty article.", "\n" *4) continue
def interleaved_neighborhood(): initialize = model_check(param_gen()) global_best = { "score": initialize["score"], "parameters": initialize["parameters"] } with open("values.csv", "a", encoding="UTF-8") as valuefile: values = clevercsv.writer(valuefile) values.writerow([global_best["score"], global_best["parameters"]]) horizon = "local" while True: try: if horizon == "local": perturb = { key: abs(gauss(value, uniform(0.1, 2.0))) if type(value) != str else value for key, value in global_best["parameters"].items() } rounded = { key: round(value) if key in ["n_estimators", "num_leaves"] else value for key, value in perturb.items() } local_solution = model_check(rounded) if local_solution["score"] < global_best["score"]: with open("values.csv", "a", encoding="UTF-8") as valuefile: values = clevercsv.writer(valuefile) values.writerow([ local_solution["score"], local_solution["parameters"] ]) global_best["score"] = local_solution["score"] global_best["parameters"] = local_solution["parameters"] continue else: horizon = "global" continue else: random_solution = model_check(param_gen()) if random_solution["score"] < global_best["score"]: with open("values.csv", "a", encoding="UTF-8") as valuefile: values = clevercsv.writer(valuefile) values.writerow([ random_solution["score"], random_solution["parameters"] ]) global_best["score"] = random_solution["score"] global_best["parameters"] = random_solution["parameters"] horizon = "local" continue else: horizon = "local" continue except Exception as error: print(error)
def writerAssertEqual(self, input, expected_result): with tempfile.TemporaryFile("w+", newline="", prefix="ccsv_") as fp: writer = clevercsv.writer(fp, dialect=self.dialect) writer.writerows(input) fp.seek(0) self.assertEqual(fp.read(), expected_result)
def core(self): """ Main function. Watch Reddit comment stream for title + author mentions. Check YouTube for audiobook and reply with link if suitable one found. """ comments = self.bot.subreddit("all").stream.comments( skip_existing=True) for comment in comments: text = comment.body.lower().replace(".", "") for card in self.catalog: if (card[1].lower() in text and card[0].lower() in text and not comment.submission.id in self.responded and not comment.subreddit.user_is_banned): self.get_info(card) if not self.details: break audio = ["audiobook", "audio book"] author_format = [ name.lower() for name in card[1].split(" ") if len(name) >= 3 ] if (self.details["duration"] > 10800 and card[0].lower() in self.details["title"].lower() and any(item in self.details["title"].lower() for item in audio) and all(item in self.details["title"].lower() for item in author_format)): saw_the_sign = ( """[^(Source Code)](https://capybasilisk.com/posts/""" """2020/04/speculative-fiction-bot/) """ """^| [^(Feedback)](https://www.reddit.com/message/""" """compose?to=Capybasilisk&subject=Robot) """ """^| [^(Programmer)](https://www.reddit.com/u/""" """capybasilisk) """ """^| ^(Downvote To Remove) """ """^| ^(Version 1.4.0) """ """^| ^(Support Robot Rights!)""") comment.reply( f"""Hi. You just mentioned *{card[0]}* by """ f"""{card[1]}.\n\nI've found an audiobook of """ """that novel on YouTube. You can listen to it here""" f""":\n\n[YouTube | {self.details['title']}]""" f"""({self.details['webpage_url']})\n\n*I\'m a bot that """ """searches YouTube for science fiction and fantasy""" f""" audiobooks.*\n***\n{saw_the_sign}""") self.responded.append(comment.submission.id) with open("activity.csv", "a", encoding="UTF-8") as actlog: activity = clevercsv.writer(actlog) if actlog.tell() == 0: activity.writerow([ "Book", "Comment", "Author", "Thread", "Subreddit", "Time" ]) activity.writerow([ f"{card[0]} by {card[1]}", f"{comment.body}", f"{comment.author}", f"{comment.submission.title}", f"{comment.subreddit}", f"{pendulum.now().to_datetime_string()}" ]) self.details = None break break if pendulum.now().to_time_string().endswith("0:00"): self.tidy()
def square_neighborhood(): global_best = 17000 best_parameters = {} neighborhood_search = 0 while True: try: random_parameters = { "boosting_type": randrange(2), "n_estimators": randrange(10000), "num_leaves": randrange(10000), "learning_rate": uniform(0.01, 1)} global_evaluate = model_check( random_parameters) if global_evaluate < global_best: neighborhood_search += round( abs( global_evaluate - global_best) ** 2) global_best = global_evaluate best_parameters = random_parameters neighborhood_steps = 0 neighborhood_size = 0.1 with open( "values.csv", "a", encoding = "UTF-8") as valuefile: values = clevercsv.writer(valuefile) values.writerow( [global_best, best_parameters]) while neighborhood_search > 0: neighborhood_parameters = { key:abs( gauss( value, neighborhood_size)) if type(value) == float else abs( round( gauss( value, neighborhood_size))) for key,value in best_parameters.items()} neighborhood_evaluate = model_check( neighborhood_parameters) if neighborhood_evaluate < global_best: neighborhood_search += abs( neighborhood_evaluate - global_best) ** 2 global_best = neighborhood_evaluate best_parameters = neighborhood_parameters neighborhood_steps = 0 neighborhood_size = 0.1 with open( "values.csv", "a", encoding = "UTF-8") as valuefile: values = clevercsv.writer(valuefile) values.writerow( [global_best, best_parameters]) else: neighborhood_steps += 1 neighborhood_size += 0.0001 * neighborhood_steps neighborhood_search -= 1 except: continue
def writerAssertEqual(self, input, expected_result): with tempfile.TemporaryFile("w+", newline="") as fileobj: writer = clevercsv.writer(fileobj, dialect=self.dialect) writer.writerows(input) fileobj.seek(0) self.assertEqual(fileobj.read(), expected_result)
def createCSVFile(self, fileName): self.m_file_path = os.path.join(tempfile.gettempdir(), fileName + '.csv') return csv.writer(open(self.m_file_path, "w"), delimiter=',', quoting=csv.QUOTE_ALL)