def get_aritle(number): article = get_aritle_by_number(number) if article is None or not article.is_public: return render_template('404.html'), 404 form = CommentForm() if request.method == 'POST': if form.validate_on_submit(): try: month_number = int(form.checker.data) except ValueError: month_number = 0 if month_number != datetime.date.today().month: form.checker.errors = ["Sorry, but please prove you are a human."] form.checker.data = "" else: comment = Comment( article_number=number, author=form.author.data, comment=form.comment.data, ) comment.save() return redirect(article.get_absolute_url()) comments = get_comments(number) return render_template( 'article.html', article=article, form=form, comments=comments, )
def blog(postid): if postid <= 0: return "<h1> 404 Error </h1>" elif request.method == "GET": return render_template("blog.html", post=utils.get_post(postid), comments=utils.get_comments(postid)) elif request.form["Submit"] == "Comment": postid = utils.new_post(session["username"], comment) post = utils.get_post(postid) return render_template("/blog/", post)
def get(self, *args, **kwargs): username=self.get_cookie("username") id=self.get_argument('id') id=int(id) sale_detail=utils.get_detail(id) if sale_detail=='fail': self.redirect('/') else: sale_comments=utils.get_comments(id) self.render('detail.html',sale_detail=sale_detail,sale_comments=sale_comments,id=id,username=username)
def get(self, thread_id): # as comment ids are positive, comments since id>-1 will return all since_comment_id = self.get_argument("since", default=-1) sql_cursor = SQL_CON.cursor() stat_code, ret_json = utils.get_comments(sql_cursor, thread_id, since_comment_id) sql_cursor.close() self.set_status(stat_code) self.write(ret_json)
def main(): parser = argparse.ArgumentParser( description="""Count comments in a given number of posts""") parser.add_argument( '-c', '--conf', type=str, metavar='', required=True, help='Specify the path of the configuration file') args = parser.parse_args() config_path = args.conf start = time.time() logger = get_logger(__name__) logger.setLevel(logging.DEBUG) conf = load_config(config_path) supported_languages = ["it", "en"] lang = input("Insert language (it, en): ") if lang not in supported_languages: logger.error("Please provide a valid language. Supported: 'en', 'it'") sys.exit(1) else: try: model = conf.get(lang) nlp = spacy.load(model) except OSError: logger.error("Could not find model in conf file. Please double check") sys.exit(0) n_posts = check_n_posts() if not n_posts.isdigit() and n_posts != "-1": logger.error("Please give a number. Exiting") sys.exit(0) try: access_token = conf["access_token"] page_id = conf["page_id"] n_top_entities = conf["n_top_entities"] data_dir_path = os.path.join(page_id, conf["data_dir_name"]) data_filename = "{}_{}.tsv".format(conf["data_entities_prefix"], str(n_posts)) plots_dir_path = os.path.join(page_id, conf["plots_dir_name"]) barplot_filename = "{}_{}posts_ner.png".format(conf["barplot_filename"], str(n_posts)) barplot_filepath = os.path.join(plots_dir_path, barplot_filename) except KeyError: logger.error( "Invalid configuration file. Please check template and retry") sys.exit(0) try: graph = facebook.GraphAPI(access_token) logger.info("Graph API connected") profile = graph.get_object(page_id) except facebook.GraphAPIError as e: logger.error("Could not log in. {}".format(e)) sys.exit(0) if n_posts != "": logger.info("Getting the last {} posts".format(n_posts)) else: logger.warning( "Requesting posts with no limits. " "This could be susceptible of limitations" " in the near future due to high rate" ) local_start = time.time() posts = graph.get_connections(profile["id"], "posts", limit=n_posts) comments = [] for post in posts["data"]: url_post = "https://www.facebook.com/posts/{}".format(post["id"]) logger.info("Getting data for post {}".format(url_post)) post_data = get_post_data(access_token, post["id"]) post_comments = get_comments(post_data) if len(post_comments) == 0: logger.warning( """Apparently, there are no comments at the selected post Check the actual post on its Facebook page https://www.facebook.com/posts/{}""".format(post["id"]) ) comments.extend(post_comments) if len(comments) == 0: logger.error("Could not get any comments. Exiting gracefully") sys.exit(0) elif len(comments) < 100: logger.warning( "Found {} comment(s). Not enough data " "to make much sense. Plots will be made regardless".format( len(comments) ) ) else: logger.info("Got {} comments from {} post(s) in {} seconds".format( len(comments), len(posts["data"]), round((time.time() - local_start), 1))) local_start = time.time() entities = [] for comment in comments: ents = get_entities(nlp, comment) entities.extend(ents) logger.info("Extracted {} entities out of {} comments in {} seconds".format( len(entities), len(comments), round((time.time() - local_start), 2))) entities_data = count_entities(entities) create_nonexistent_dir(data_dir_path) data_filepath = os.path.join(data_dir_path, data_filename) columns = ["entities", "count"] data_to_tsv(entities_data, columns, data_filepath) logger.info("Saved {} unique entities and their counts in {} ".format( len(entities_data), data_filepath)) create_nonexistent_dir(plots_dir_path) plot_labels = ["Entities", "Counts"] save_barplot(entities_data, plot_labels, n_top_entities, barplot_filepath, type_="entities") logger.info("Bar plot saved at {}".format(barplot_filepath)) logger.info("\a\a\aDIN DONE! in {} seconds".format( round((time.time() - start), 1)))
import utils from PIL import Image, ImageDraw def level_9(text): """Find the coordinates in the comments and draw the lines.""" mo = re.search(r'first:((\d*,?)*)', text.replace('\n', '')) if mo: first_pat = mo.group(1) first = [int(num) for num in first_pat.split(',')] im = Image.new('RGBA', (max(first) + 10, max(first) + 10)) draw = ImageDraw.Draw(im) draw.line(first) mo = re.search(r'second:((\d*,?)*)', text.replace('\n', '')) if mo: second_pat = mo.group(1) second = [int(num) for num in second_pat.split(',')] draw.line(second) im.show() if __name__ == '__main__': comments = utils.get_comments( 'http://www.pythonchallenge.com/pc/return/good.html', user='******') if comments: for comment in comments: level_9(comment) else: print(" [*] No comments found.")
def main(): parser = argparse.ArgumentParser( description="""Count comments in a given number of posts""") parser.add_argument('-c', '--conf', type=str, metavar='', required=True, help='Specify the path of the configuration file') args = parser.parse_args() config_path = args.conf start = time.time() logger = get_logger(__name__) logger.setLevel(logging.DEBUG) conf = load_config(config_path) n_posts = check_n_posts() if not n_posts.isdigit() and n_posts != "-1": logger.error("Please give a number. Exiting") sys.exit(0) try: access_token = conf["access_token"] page_id = conf["page_id"] n_top_words = conf["n_top_words"] data_dir_path = os.path.join(page_id, conf["data_dir_name"]) data_filename = "{}_{}.tsv".format(conf["data_wc_prefix"], str(n_posts)) plots_dir_path = os.path.join(page_id, conf["plots_dir_name"]) wc_plot_filename = "{}_{}posts.png".format(conf["wc_plot_filename"], str(n_posts)) wc_plot_filepath = os.path.join(plots_dir_path, wc_plot_filename) barplot_filename = "{}_{}posts.png".format(conf["barplot_filename"], str(n_posts)) barplot_filepath = os.path.join(plots_dir_path, barplot_filename) except KeyError: logger.error( "Invalid configuration file. Please check template and retry") sys.exit(0) try: graph = facebook.GraphAPI(access_token) logger.info("Graph API connected") profile = graph.get_object(page_id) except facebook.GraphAPIError as e: logger.error("Could not log in. {}".format(e)) sys.exit(0) local_start = time.time() posts = graph.get_connections(profile["id"], "posts", limit=n_posts) comments = [] for post in posts["data"]: url_post = "https://www.facebook.com/posts/{}".format(post["id"]) logger.info("Getting data for post {}".format(url_post)) post_data = get_post_data(access_token, post["id"]) post_comments = get_comments(post_data) if len(post_comments) == 0: logger.warning( """Apparently, there are no comments at the selected post Check the actual post on its Facebook page https://www.facebook.com/posts/{}""".format(post["id"])) comments.extend(post_comments) if len(comments) == 0: logger.error("Could not get any comments. Exiting gracefully") sys.exit(0) elif len(comments) < 100: logger.warning( "Found {} comment(s). Not enough data " "to make much sense. Plots will be made regardless".format( len(comments))) else: logger.info("Got {} comments from {} post(s) in {} seconds".format( len(comments), len(posts["data"]), round((time.time() - local_start), 1))) local_start = time.time() preprocessed_comments = [ TextPreprocessor(comm).preprocess() for comm in comments ] logger.info("Preprocessed {} comments out of {} in {} seconds".format( len(preprocessed_comments), len(comments), round((time.time() - local_start), 2))) wordcount_data = do_wordcount(preprocessed_comments) create_nonexistent_dir(data_dir_path) data_filepath = os.path.join(data_dir_path, data_filename) columns = ["word", "count"] data_to_tsv(wordcount_data, columns, data_filepath) logger.info("Saved {} words and their counts in {} ".format( len(wordcount_data), data_filepath)) create_nonexistent_dir(plots_dir_path) plot_labels = ["Words", "Counts"] save_barplot(wordcount_data, plot_labels, n_top_words, barplot_filepath) logger.info("Bar plot saved at {}".format(barplot_filepath)) unstemmed_comments = [ TextPreprocessor(comm).base_preprocess() for comm in comments ] long_string = " ".join(uc for uc in unstemmed_comments) p = Plotter(long_string) p.save_wordcloud_plot(wc_plot_filepath) logger.info("Wordcloud plot saved at {}".format(wc_plot_filepath)) logger.info("\a\a\aDIN DONE! in {} seconds".format( round((time.time() - start), 1)))
def print_anaysis(tbl, id, scoring, type=1): display_html(analyse_review(utils.get_comments(tbl, 'id', id , 'comments'), list(tbl[tbl['id'] == id ][scoring].values[0]), type))
import re import utils def level_3(text): """Find the lowercase letters in text that are surrounded by exactly 3 uppercase letters.""" pattern = re.compile(r'[a-z][A-Z]{3}([a-z])[A-Z]{3}[a-z]') return ''.join(pattern.findall(text)) if __name__ == '__main__': comments = utils.get_comments( 'http://www.pythonchallenge.com/pc/def/equality.html') if comments: for comment in comments: print(level_3(comment)) else: print(" [*] No comments found.")
def main(): parser = argparse.ArgumentParser( description="""Count comments in a given number of posts""") parser.add_argument('-c', '--conf', type=str, metavar='', required=True, help='Specify the path of the configuration file') args = parser.parse_args() config_path = args.conf start = time.time() logger = get_logger(__name__) logger.setLevel(logging.DEBUG) conf = load_config(config_path) supported_languages = ["it", "en"] lang = input("Insert language (it, en): ") if lang not in supported_languages: logger.error("Please provide a valid language. Supported: 'en', 'it'") sys.exit(1) else: try: model = conf.get(lang) nlp = spacy.load(model) except OSError: logger.error( "Could not find model in conf file. Please double check") sys.exit(0) post_id = "" while post_id == "": post_id = input("Provide post ID: ") try: access_token = conf["access_token"] page_id = conf["page_id"] n_top_entities = conf["n_top_entities"] data_dir_path = os.path.join(page_id, conf["data_dir_name"]) data_filename = "{}_{}{}".format(conf["data_entities_prefix"], post_id, ".csv") plots_dir_path = os.path.join(page_id, conf["plots_dir_name"]) barplot_filename = "{}_{}{}".format(conf["barplot_filename"], post_id, "_ner.png") barplot_filepath = os.path.join(plots_dir_path, barplot_filename) except KeyError: logger.error( "Invalid configuration file. Please check template and retry") sys.exit(0) actual_post_id = page_id + "_" + post_id url_post = "https://www.facebook.com/posts/{}".format(actual_post_id) logger.info("Getting data for post {}".format(url_post)) local_start = time.time() data = get_post_data(access_token, actual_post_id) comments = get_comments(data) if len(comments) == 0: logger.error("""Apparently, there are no comments at the selected post Check the actual post on its Facebook page https://www.facebook.com/{}/posts/{}""".format(page_id, post_id)) sys.exit(0) elif len(comments) < 100: logger.warning( "Got {} comments. Not enough data " "to make much sense. Plots will be made regardless".format( len(comments))) else: logger.info("Got {} comments in {} seconds".format( len(comments), round((time.time() - local_start), 2))) local_start = time.time() entities = [] for comment in comments: ents = get_entities(nlp, comment) entities.extend(ents) logger.info( "Extracted {} entities out of {} comments in {} seconds".format( len(entities), len(comments), round((time.time() - local_start), 2))) entities_data = count_entities(entities) create_nonexistent_dir(data_dir_path) data_filepath = os.path.join(data_dir_path, data_filename) columns = ["entities", "count"] data_to_tsv(entities_data, columns, data_filepath) logger.info("Saved {} unique entities and their counts in {} ".format( len(entities_data), data_filepath)) create_nonexistent_dir(plots_dir_path) plot_labels = ["Entities", "Counts"] save_barplot(entities_data, plot_labels, n_top_entities, barplot_filepath) logger.info("Bar plot saved at {}".format(barplot_filepath)) logger.info("\a\a\aDIN DONE! in {} seconds".format( round((time.time() - start), 1)))
def main(): parser = argparse.ArgumentParser( description="""Count comments in a given number of posts""") parser.add_argument('-c', '--conf', type=str, metavar='', required=True, help='Specify the path of the configuration file') args = parser.parse_args() config_path = args.conf start = time.time() logger = get_logger(__name__) logger.setLevel(logging.DEBUG) conf = load_config(config_path) n_posts = check_n_posts() if not n_posts.isdigit(): logger.error("Please give a number. Exiting") sys.exit(0) try: access_token = conf["access_token"] page_id = conf["page_id"] except KeyError: logger.error( "Invalid configuration file. Please check template and retry") sys.exit(0) try: graph = facebook.GraphAPI(access_token) logger.info("Graph API connected") profile = graph.get_object(page_id) except facebook.GraphAPIError as e: logger.error("Could not log in. {}".format(e)) sys.exit(0) local_start = time.time() posts = graph.get_connections(profile["id"], "posts", limit=n_posts) comments = [] for post in posts["data"]: url_post = "https://www.facebook.com/posts/{}".format(post["id"]) logger.info("Getting data for post {}".format(url_post)) post_data = get_post_data(access_token, post["id"]) post_comments = get_comments(post_data) if len(post_comments) == 0: logger.warning( """Apparently, there are no comments at the selected post Check the actual post on its Facebook page https://www.facebook.com/posts/{}""".format(post["id"])) comments.extend(post_comments) if len(comments) == 0: logger.error("Could not get any comments. Exiting gracefully") sys.exit(0) elif len(comments) < 100: logger.warning( "Found {} comment(s). Not enough data " "to make much sense. Plots will be made regardless".format( len(comments))) else: logger.info("Got {} comments from {} post(s) in {} seconds".format( len(comments), len(posts["data"]), round((time.time() - local_start), 1))) data_dir_name = os.path.join(page_id, conf["data_dir_name"]) create_nonexistent_dir(data_dir_name) data_filename = "{}_comments.tsv".format(len(comments)) data_filepath = os.path.join(data_dir_name, data_filename) data = zip(comments, [0] * len(comments)) columns = ["comment", "sentiment"] data_to_tsv(data, columns, data_filepath) logger.info("Saved {} comments in {} ".format(len(comments), data_filepath)) logger.info("\a\a\aDIN DONE! in {} seconds".format( round((time.time() - start), 1)))
def main(): parser = argparse.ArgumentParser( description="""Count comments in a given post""") parser.add_argument('-c', '--conf', type=str, metavar='', required=True, help='Specify the path of the configuration file') args = parser.parse_args() config_path = args.conf start = time.time() logger = get_logger(__name__) logger.setLevel(logging.DEBUG) conf = load_config(config_path) post_id = "" while post_id == "": post_id = input("Provide post ID: ") try: access_token = conf["access_token"] page_id = conf["page_id"] n_top_words = conf["n_top_words"] data_dir_path = os.path.join(page_id, conf["data_dir_name"]) data_filename = "{}_{}{}".format(conf["data_wc_prefix"], post_id, ".csv") plots_dir_path = os.path.join(page_id, conf["plots_dir_name"], "single_posts", post_id) wc_plot_filename = "{}_{}{}".format(conf["wc_plot_filename"], post_id, ".png") wc_plot_filepath = os.path.join(plots_dir_path, wc_plot_filename) barplot_filename = "{}_{}{}".format(conf["barplot_filename"], post_id, ".png") barplot_filepath = os.path.join(plots_dir_path, barplot_filename) except KeyError: logger.error( "Invalid configuration file. Please check template and retry") sys.exit(0) url_post = "https://www.facebook.com/posts/{}".format(post_id) logger.info("Getting data for post {}".format(url_post)) actual_post_id = page_id + "_" + post_id local_start = time.time() data = get_post_data(access_token, actual_post_id) comments = get_comments(data) if len(comments) == 0: logger.error("""Apparently, there are no comments at the selected post Check the actual post on its Facebook page https://www.facebook.com/{}/posts/{}""".format(page_id, post_id)) sys.exit(0) elif len(comments) < 100: logger.warning( "Got {} comments. Not enough data " "to make much sense. Plots will be made regardless".format( len(comments))) else: logger.info("Got {} comments in {} seconds".format( len(comments), round((time.time() - local_start), 2))) local_start = time.time() preprocessed_comments = [ TextPreprocessor(comm).preprocess() for comm in comments ] logger.info("Preprocessed {} comments out of {} in {} seconds".format( len(preprocessed_comments), len(comments), round((time.time() - local_start), 1))) logger.info("Performing word count") wordcount_data = do_wordcount(preprocessed_comments) create_nonexistent_dir(data_dir_path) data_filepath = os.path.join(data_dir_path, data_filename) columns = ["word", "count"] data_to_tsv(wordcount_data, columns, data_filepath) logger.info("Saved {} words and their counts in {} ".format( len(wordcount_data), data_filepath)) create_nonexistent_dir(plots_dir_path) plot_labels = ["Words", "Counts"] save_barplot(wordcount_data, plot_labels, n_top_words, barplot_filepath) logger.info("Bar plot saved at {}".format(barplot_filepath)) unstemmed_comments = [ TextPreprocessor(comm).base_preprocess() for comm in comments ] long_string = " ".join(uc for uc in unstemmed_comments) p = Plotter(long_string) p.save_wordcloud_plot(wc_plot_filepath) logger.info("Word Cloud plot saved at {}".format(wc_plot_filepath)) logger.info("\a\a\aDIN DONE!") logger.info("Total time of execution: {} seconds".format( round((time.time() - start), 1)))
import utils comments = utils.get_comments(5692814887223296) comments.reverse() for comment in comments: print comment["old_task_instance_status"] print comment["new_task_instance_status"] # print comment["text"] print