def get_aritle(number):
article = get_aritle_by_number(number)
if article is None or not article.is_public:
return render_template('404.html'), 404
form = CommentForm()
if request.method == 'POST':
if form.validate_on_submit():
try:
month_number = int(form.checker.data)
except ValueError:
month_number = 0
if month_number != datetime.date.today().month:
form.checker.errors = ["Sorry, but please prove you are a human."]
form.checker.data = ""
else:
comment = Comment(
article_number=number,
author=form.author.data,
comment=form.comment.data,
)
comment.save()
return redirect(article.get_absolute_url())
comments = get_comments(number)
return render_template(
'article.html',
article=article,
form=form,
comments=comments,
)
def blog(postid):
if postid <= 0:
return "<h1> 404 Error </h1>"
elif request.method == "GET":
return render_template("blog.html", post=utils.get_post(postid), comments=utils.get_comments(postid))
elif request.form["Submit"] == "Comment":
postid = utils.new_post(session["username"], comment)
post = utils.get_post(postid)
return render_template("/blog/", post)
def get(self, *args, **kwargs):
username=self.get_cookie("username")
id=self.get_argument('id')
id=int(id)
sale_detail=utils.get_detail(id)
if sale_detail=='fail':
self.redirect('/')
else:
sale_comments=utils.get_comments(id)
self.render('detail.html',sale_detail=sale_detail,sale_comments=sale_comments,id=id,username=username)
def get(self, thread_id):
# as comment ids are positive, comments since id>-1 will return all
since_comment_id = self.get_argument("since", default=-1)
sql_cursor = SQL_CON.cursor()
stat_code, ret_json = utils.get_comments(sql_cursor, thread_id,
since_comment_id)
sql_cursor.close()
self.set_status(stat_code)
self.write(ret_json)
def blog(postid):
if postid <= 0:
return "<h1> 404 Error </h1>"
elif request.method == "GET":
return render_template("blog.html",
post=utils.get_post(postid),
comments=utils.get_comments(postid))
elif request.form["Submit"] == "Comment":
postid = utils.new_post(session["username"], comment)
post = utils.get_post(postid)
return render_template("/blog/", post)
def main():
parser = argparse.ArgumentParser(
description="""Count comments in a given number of posts""")
parser.add_argument(
'-c', '--conf', type=str, metavar='', required=True,
help='Specify the path of the configuration file')
args = parser.parse_args()
config_path = args.conf
start = time.time()
logger = get_logger(__name__)
logger.setLevel(logging.DEBUG)
conf = load_config(config_path)
supported_languages = ["it", "en"]
lang = input("Insert language (it, en): ")
if lang not in supported_languages:
logger.error("Please provide a valid language. Supported: 'en', 'it'")
sys.exit(1)
else:
try:
model = conf.get(lang)
nlp = spacy.load(model)
except OSError:
logger.error("Could not find model in conf file. Please double check")
sys.exit(0)
n_posts = check_n_posts()
if not n_posts.isdigit() and n_posts != "-1":
logger.error("Please give a number. Exiting")
sys.exit(0)
try:
access_token = conf["access_token"]
page_id = conf["page_id"]
n_top_entities = conf["n_top_entities"]
data_dir_path = os.path.join(page_id, conf["data_dir_name"])
data_filename = "{}_{}.tsv".format(conf["data_entities_prefix"], str(n_posts))
plots_dir_path = os.path.join(page_id, conf["plots_dir_name"])
barplot_filename = "{}_{}posts_ner.png".format(conf["barplot_filename"], str(n_posts))
barplot_filepath = os.path.join(plots_dir_path, barplot_filename)
except KeyError:
logger.error(
"Invalid configuration file. Please check template and retry")
sys.exit(0)
try:
graph = facebook.GraphAPI(access_token)
logger.info("Graph API connected")
profile = graph.get_object(page_id)
except facebook.GraphAPIError as e:
logger.error("Could not log in. {}".format(e))
sys.exit(0)
if n_posts != "":
logger.info("Getting the last {} posts".format(n_posts))
else:
logger.warning(
"Requesting posts with no limits. "
"This could be susceptible of limitations"
" in the near future due to high rate"
)
local_start = time.time()
posts = graph.get_connections(profile["id"], "posts", limit=n_posts)
comments = []
for post in posts["data"]:
url_post = "https://www.facebook.com/posts/{}".format(post["id"])
logger.info("Getting data for post {}".format(url_post))
post_data = get_post_data(access_token, post["id"])
post_comments = get_comments(post_data)
if len(post_comments) == 0:
logger.warning(
"""Apparently, there are no comments at the selected post
Check the actual post on its Facebook page
https://www.facebook.com/posts/{}""".format(post["id"])
)
comments.extend(post_comments)
if len(comments) == 0:
logger.error("Could not get any comments. Exiting gracefully")
sys.exit(0)
elif len(comments) < 100:
logger.warning(
"Found {} comment(s). Not enough data "
"to make much sense. Plots will be made regardless".format(
len(comments)
)
)
else:
logger.info("Got {} comments from {} post(s) in {} seconds".format(
len(comments), len(posts["data"]), round((time.time() - local_start), 1)))
local_start = time.time()
entities = []
for comment in comments:
ents = get_entities(nlp, comment)
entities.extend(ents)
logger.info("Extracted {} entities out of {} comments in {} seconds".format(
len(entities), len(comments), round((time.time() - local_start), 2)))
entities_data = count_entities(entities)
create_nonexistent_dir(data_dir_path)
data_filepath = os.path.join(data_dir_path, data_filename)
columns = ["entities", "count"]
data_to_tsv(entities_data, columns, data_filepath)
logger.info("Saved {} unique entities and their counts in {} ".format(
len(entities_data), data_filepath))
create_nonexistent_dir(plots_dir_path)
plot_labels = ["Entities", "Counts"]
save_barplot(entities_data, plot_labels, n_top_entities, barplot_filepath, type_="entities")
logger.info("Bar plot saved at {}".format(barplot_filepath))
logger.info("\a\a\aDIN DONE! in {} seconds".format(
round((time.time() - start), 1)))
import utils
from PIL import Image, ImageDraw
def level_9(text):
"""Find the coordinates in the comments and draw the lines."""
mo = re.search(r'first:((\d*,?)*)', text.replace('\n', ''))
if mo:
first_pat = mo.group(1)
first = [int(num) for num in first_pat.split(',')]
im = Image.new('RGBA', (max(first) + 10, max(first) + 10))
draw = ImageDraw.Draw(im)
draw.line(first)
mo = re.search(r'second:((\d*,?)*)', text.replace('\n', ''))
if mo:
second_pat = mo.group(1)
second = [int(num) for num in second_pat.split(',')]
draw.line(second)
im.show()
if __name__ == '__main__':
comments = utils.get_comments(
'http://www.pythonchallenge.com/pc/return/good.html', user='******')
if comments:
for comment in comments:
level_9(comment)
else:
print(" [*] No comments found.")
def main():
parser = argparse.ArgumentParser(
description="""Count comments in a given number of posts""")
parser.add_argument('-c',
'--conf',
type=str,
metavar='',
required=True,
help='Specify the path of the configuration file')
args = parser.parse_args()
config_path = args.conf
start = time.time()
logger = get_logger(__name__)
logger.setLevel(logging.DEBUG)
conf = load_config(config_path)
n_posts = check_n_posts()
if not n_posts.isdigit() and n_posts != "-1":
logger.error("Please give a number. Exiting")
sys.exit(0)
try:
access_token = conf["access_token"]
page_id = conf["page_id"]
n_top_words = conf["n_top_words"]
data_dir_path = os.path.join(page_id, conf["data_dir_name"])
data_filename = "{}_{}.tsv".format(conf["data_wc_prefix"],
str(n_posts))
plots_dir_path = os.path.join(page_id, conf["plots_dir_name"])
wc_plot_filename = "{}_{}posts.png".format(conf["wc_plot_filename"],
str(n_posts))
wc_plot_filepath = os.path.join(plots_dir_path, wc_plot_filename)
barplot_filename = "{}_{}posts.png".format(conf["barplot_filename"],
str(n_posts))
barplot_filepath = os.path.join(plots_dir_path, barplot_filename)
except KeyError:
logger.error(
"Invalid configuration file. Please check template and retry")
sys.exit(0)
try:
graph = facebook.GraphAPI(access_token)
logger.info("Graph API connected")
profile = graph.get_object(page_id)
except facebook.GraphAPIError as e:
logger.error("Could not log in. {}".format(e))
sys.exit(0)
local_start = time.time()
posts = graph.get_connections(profile["id"], "posts", limit=n_posts)
comments = []
for post in posts["data"]:
url_post = "https://www.facebook.com/posts/{}".format(post["id"])
logger.info("Getting data for post {}".format(url_post))
post_data = get_post_data(access_token, post["id"])
post_comments = get_comments(post_data)
if len(post_comments) == 0:
logger.warning(
"""Apparently, there are no comments at the selected post
Check the actual post on its Facebook page
https://www.facebook.com/posts/{}""".format(post["id"]))
comments.extend(post_comments)
if len(comments) == 0:
logger.error("Could not get any comments. Exiting gracefully")
sys.exit(0)
elif len(comments) < 100:
logger.warning(
"Found {} comment(s). Not enough data "
"to make much sense. Plots will be made regardless".format(
len(comments)))
else:
logger.info("Got {} comments from {} post(s) in {} seconds".format(
len(comments), len(posts["data"]),
round((time.time() - local_start), 1)))
local_start = time.time()
preprocessed_comments = [
TextPreprocessor(comm).preprocess() for comm in comments
]
logger.info("Preprocessed {} comments out of {} in {} seconds".format(
len(preprocessed_comments), len(comments),
round((time.time() - local_start), 2)))
wordcount_data = do_wordcount(preprocessed_comments)
create_nonexistent_dir(data_dir_path)
data_filepath = os.path.join(data_dir_path, data_filename)
columns = ["word", "count"]
data_to_tsv(wordcount_data, columns, data_filepath)
logger.info("Saved {} words and their counts in {} ".format(
len(wordcount_data), data_filepath))
create_nonexistent_dir(plots_dir_path)
plot_labels = ["Words", "Counts"]
save_barplot(wordcount_data, plot_labels, n_top_words, barplot_filepath)
logger.info("Bar plot saved at {}".format(barplot_filepath))
unstemmed_comments = [
TextPreprocessor(comm).base_preprocess() for comm in comments
]
long_string = " ".join(uc for uc in unstemmed_comments)
p = Plotter(long_string)
p.save_wordcloud_plot(wc_plot_filepath)
logger.info("Wordcloud plot saved at {}".format(wc_plot_filepath))
logger.info("\a\a\aDIN DONE! in {} seconds".format(
round((time.time() - start), 1)))
def print_anaysis(tbl, id, scoring, type=1):
display_html(analyse_review(utils.get_comments(tbl, 'id', id , 'comments'), list(tbl[tbl['id'] == id ][scoring].values[0]), type))
import re
import utils
def level_3(text):
"""Find the lowercase letters in text that are surrounded by exactly 3 uppercase letters."""
pattern = re.compile(r'[a-z][A-Z]{3}([a-z])[A-Z]{3}[a-z]')
return ''.join(pattern.findall(text))
if __name__ == '__main__':
comments = utils.get_comments(
'http://www.pythonchallenge.com/pc/def/equality.html')
if comments:
for comment in comments:
print(level_3(comment))
else:
print(" [*] No comments found.")
def main():
parser = argparse.ArgumentParser(
description="""Count comments in a given number of posts""")
parser.add_argument('-c',
'--conf',
type=str,
metavar='',
required=True,
help='Specify the path of the configuration file')
args = parser.parse_args()
config_path = args.conf
start = time.time()
logger = get_logger(__name__)
logger.setLevel(logging.DEBUG)
conf = load_config(config_path)
supported_languages = ["it", "en"]
lang = input("Insert language (it, en): ")
if lang not in supported_languages:
logger.error("Please provide a valid language. Supported: 'en', 'it'")
sys.exit(1)
else:
try:
model = conf.get(lang)
nlp = spacy.load(model)
except OSError:
logger.error(
"Could not find model in conf file. Please double check")
sys.exit(0)
post_id = ""
while post_id == "":
post_id = input("Provide post ID: ")
try:
access_token = conf["access_token"]
page_id = conf["page_id"]
n_top_entities = conf["n_top_entities"]
data_dir_path = os.path.join(page_id, conf["data_dir_name"])
data_filename = "{}_{}{}".format(conf["data_entities_prefix"], post_id,
".csv")
plots_dir_path = os.path.join(page_id, conf["plots_dir_name"])
barplot_filename = "{}_{}{}".format(conf["barplot_filename"], post_id,
"_ner.png")
barplot_filepath = os.path.join(plots_dir_path, barplot_filename)
except KeyError:
logger.error(
"Invalid configuration file. Please check template and retry")
sys.exit(0)
actual_post_id = page_id + "_" + post_id
url_post = "https://www.facebook.com/posts/{}".format(actual_post_id)
logger.info("Getting data for post {}".format(url_post))
local_start = time.time()
data = get_post_data(access_token, actual_post_id)
comments = get_comments(data)
if len(comments) == 0:
logger.error("""Apparently, there are no comments at the selected post
Check the actual post on its Facebook page
https://www.facebook.com/{}/posts/{}""".format(page_id, post_id))
sys.exit(0)
elif len(comments) < 100:
logger.warning(
"Got {} comments. Not enough data "
"to make much sense. Plots will be made regardless".format(
len(comments)))
else:
logger.info("Got {} comments in {} seconds".format(
len(comments), round((time.time() - local_start), 2)))
local_start = time.time()
entities = []
for comment in comments:
ents = get_entities(nlp, comment)
entities.extend(ents)
logger.info(
"Extracted {} entities out of {} comments in {} seconds".format(
len(entities), len(comments), round((time.time() - local_start),
2)))
entities_data = count_entities(entities)
create_nonexistent_dir(data_dir_path)
data_filepath = os.path.join(data_dir_path, data_filename)
columns = ["entities", "count"]
data_to_tsv(entities_data, columns, data_filepath)
logger.info("Saved {} unique entities and their counts in {} ".format(
len(entities_data), data_filepath))
create_nonexistent_dir(plots_dir_path)
plot_labels = ["Entities", "Counts"]
save_barplot(entities_data, plot_labels, n_top_entities, barplot_filepath)
logger.info("Bar plot saved at {}".format(barplot_filepath))
logger.info("\a\a\aDIN DONE! in {} seconds".format(
round((time.time() - start), 1)))
def main():
parser = argparse.ArgumentParser(
description="""Count comments in a given number of posts""")
parser.add_argument('-c',
'--conf',
type=str,
metavar='',
required=True,
help='Specify the path of the configuration file')
args = parser.parse_args()
config_path = args.conf
start = time.time()
logger = get_logger(__name__)
logger.setLevel(logging.DEBUG)
conf = load_config(config_path)
n_posts = check_n_posts()
if not n_posts.isdigit():
logger.error("Please give a number. Exiting")
sys.exit(0)
try:
access_token = conf["access_token"]
page_id = conf["page_id"]
except KeyError:
logger.error(
"Invalid configuration file. Please check template and retry")
sys.exit(0)
try:
graph = facebook.GraphAPI(access_token)
logger.info("Graph API connected")
profile = graph.get_object(page_id)
except facebook.GraphAPIError as e:
logger.error("Could not log in. {}".format(e))
sys.exit(0)
local_start = time.time()
posts = graph.get_connections(profile["id"], "posts", limit=n_posts)
comments = []
for post in posts["data"]:
url_post = "https://www.facebook.com/posts/{}".format(post["id"])
logger.info("Getting data for post {}".format(url_post))
post_data = get_post_data(access_token, post["id"])
post_comments = get_comments(post_data)
if len(post_comments) == 0:
logger.warning(
"""Apparently, there are no comments at the selected post
Check the actual post on its Facebook page
https://www.facebook.com/posts/{}""".format(post["id"]))
comments.extend(post_comments)
if len(comments) == 0:
logger.error("Could not get any comments. Exiting gracefully")
sys.exit(0)
elif len(comments) < 100:
logger.warning(
"Found {} comment(s). Not enough data "
"to make much sense. Plots will be made regardless".format(
len(comments)))
else:
logger.info("Got {} comments from {} post(s) in {} seconds".format(
len(comments), len(posts["data"]),
round((time.time() - local_start), 1)))
data_dir_name = os.path.join(page_id, conf["data_dir_name"])
create_nonexistent_dir(data_dir_name)
data_filename = "{}_comments.tsv".format(len(comments))
data_filepath = os.path.join(data_dir_name, data_filename)
data = zip(comments, [0] * len(comments))
columns = ["comment", "sentiment"]
data_to_tsv(data, columns, data_filepath)
logger.info("Saved {} comments in {} ".format(len(comments),
data_filepath))
logger.info("\a\a\aDIN DONE! in {} seconds".format(
round((time.time() - start), 1)))
def main():
parser = argparse.ArgumentParser(
description="""Count comments in a given post""")
parser.add_argument('-c',
'--conf',
type=str,
metavar='',
required=True,
help='Specify the path of the configuration file')
args = parser.parse_args()
config_path = args.conf
start = time.time()
logger = get_logger(__name__)
logger.setLevel(logging.DEBUG)
conf = load_config(config_path)
post_id = ""
while post_id == "":
post_id = input("Provide post ID: ")
try:
access_token = conf["access_token"]
page_id = conf["page_id"]
n_top_words = conf["n_top_words"]
data_dir_path = os.path.join(page_id, conf["data_dir_name"])
data_filename = "{}_{}{}".format(conf["data_wc_prefix"], post_id,
".csv")
plots_dir_path = os.path.join(page_id, conf["plots_dir_name"],
"single_posts", post_id)
wc_plot_filename = "{}_{}{}".format(conf["wc_plot_filename"], post_id,
".png")
wc_plot_filepath = os.path.join(plots_dir_path, wc_plot_filename)
barplot_filename = "{}_{}{}".format(conf["barplot_filename"], post_id,
".png")
barplot_filepath = os.path.join(plots_dir_path, barplot_filename)
except KeyError:
logger.error(
"Invalid configuration file. Please check template and retry")
sys.exit(0)
url_post = "https://www.facebook.com/posts/{}".format(post_id)
logger.info("Getting data for post {}".format(url_post))
actual_post_id = page_id + "_" + post_id
local_start = time.time()
data = get_post_data(access_token, actual_post_id)
comments = get_comments(data)
if len(comments) == 0:
logger.error("""Apparently, there are no comments at the selected post
Check the actual post on its Facebook page
https://www.facebook.com/{}/posts/{}""".format(page_id, post_id))
sys.exit(0)
elif len(comments) < 100:
logger.warning(
"Got {} comments. Not enough data "
"to make much sense. Plots will be made regardless".format(
len(comments)))
else:
logger.info("Got {} comments in {} seconds".format(
len(comments), round((time.time() - local_start), 2)))
local_start = time.time()
preprocessed_comments = [
TextPreprocessor(comm).preprocess() for comm in comments
]
logger.info("Preprocessed {} comments out of {} in {} seconds".format(
len(preprocessed_comments), len(comments),
round((time.time() - local_start), 1)))
logger.info("Performing word count")
wordcount_data = do_wordcount(preprocessed_comments)
create_nonexistent_dir(data_dir_path)
data_filepath = os.path.join(data_dir_path, data_filename)
columns = ["word", "count"]
data_to_tsv(wordcount_data, columns, data_filepath)
logger.info("Saved {} words and their counts in {} ".format(
len(wordcount_data), data_filepath))
create_nonexistent_dir(plots_dir_path)
plot_labels = ["Words", "Counts"]
save_barplot(wordcount_data, plot_labels, n_top_words, barplot_filepath)
logger.info("Bar plot saved at {}".format(barplot_filepath))
unstemmed_comments = [
TextPreprocessor(comm).base_preprocess() for comm in comments
]
long_string = " ".join(uc for uc in unstemmed_comments)
p = Plotter(long_string)
p.save_wordcloud_plot(wc_plot_filepath)
logger.info("Word Cloud plot saved at {}".format(wc_plot_filepath))
logger.info("\a\a\aDIN DONE!")
logger.info("Total time of execution: {} seconds".format(
round((time.time() - start), 1)))
import utils
comments = utils.get_comments(5692814887223296)
comments.reverse()
for comment in comments:
print comment["old_task_instance_status"]
print comment["new_task_instance_status"]
# print comment["text"]
print
