def update_category(category_id=0):
if category_id == 0:
category_id = int(input("category ID: "))
category = db.cat_from_snippet(category_id, numeric_snippet=True)
try:
print('Current category name: {0}'.format(category.category_name))
except AttributeError:
print('Category not found. Return to main menu')
return
new_category_name = btc.read_text(
"Enter new category name or '.' to cancel: ")
if new_category_name != '.':
update_choice = btc.read_bool(
decision='Update category name from {0} to {1}?'.format(
category.category_name, new_category_name),
yes='1',
no='2',
yes_option='update',
no_option='cancel')
if update_choice == True:
db.update_category2(category_id=category_id,
new_value=new_category_name,
update_type='name')
print('Category update complete\n')
elif update_choice == False:
print('Update cancelled.\n')
def scrape_article_name(article_id):
article = db.get_article(article_id)
if article == None:
print('There is no article with that ID. article NOT found.\n')
else:
print()
display_single_article(article, str(article.ArticleID))
article_choice = btc.read_int_ranged(
'1 to rescrape title, 2 to leave as is: ',
min_value=1,
max_value=2)
if article_choice == 1:
try:
new_article_news_item = na.get_article_from_url(article.link)
new_title = new_article_news_item.title
print('''
New title: {0}
Old title: {1}'''.format(new_title, article.name))
except:
new_title = 'Title scrape failed'
title_choice = btc.read_int_ranged(
'1 to replace title, 2 to keep original title: ',
min_value=1,
max_value=2)
if title_choice == 1:
db.update_article_name(article_id, new_title)
elif title_choice == 2:
print('article update cancelled')
elif article_choice == 2:
print('article update cancelled')
def update_category_section(category_id=0):
if category_id == 0:
category_id = int(input("category ID: "))
category = db.cat_from_snippet(category_id, numeric_snippet=True)
try:
print('Current category name: {0}'.format(category.category_name))
print('Current section name: {0}'.format(category.section_name))
except Exception as e:
print(e)
category.sectionID = 1
#setattr(category.section_id, 'section_id', None)
#except AttributeError:
# print('Category not found. Return to main menu')
# return
display_sections()
new_category_section = btc.read_int(
"Enter new section id or '.' to cancel: ")
if new_category_section != '.':
update_choice = btc.read_bool(
decision='Update category section from {0} to {1}?'.format(
category.sectionID, new_category_section),
yes='1',
no='2',
yes_option='update',
no_option='cancel')
if update_choice == True:
db.update_category2(category_id=category_id,
new_value=new_category_section,
update_type='section')
print('Category update complete\n')
elif update_choice == False:
print('Update cancelled.\n')
def manual_add(link=None):
'''
Add a read_bool choice at the end to confirm that the user wants to add
the article.
'''
display_categories()
try:
category = btc.read_int('Enter category for article: ')
except Exception as e:
print(e)
return
category = db.cat_from_snippet(category, numeric_snippet=True)
if category == None:
print('Article creation cancelled')
return
new_article = Article.manual_add(link=link, category=category)
#display the new article before the user makes the decision to confirm
confirm_article = btc.read_bool(decision="Finalize the article?",
yes='y',
no='n',
yes_option='Confirm',
no_option='Cancel')
if confirm_article == True:
db.add_article(new_article)
print(new_article.title + " was added to database.\n")
else:
print('Add article cancelled. Return to main menu.')
return
def update_article_description(article_id):
article = db.get_article(article_id)
if article == None:
print("There is no article with that ID. article NOT found.\n")
else:
print()
display_single_article(article, str(article.ArticleID))
article_choice = btc.read_int_ranged(
'1 to edit article description, 2 to leave as is: ',
min_value=1,
max_value=2)
if article_choice == 1:
description_choice = btc.read_text(
'View article description? y/n: ')
if description_choice == 'y':
article_summary = na.get_article_summary(article.link)
print(article_summary)
new_description = btc.read_text(
'Enter new description or "." to cancel: ')
if new_description != '.':
db.update_article_description(article_id, new_description)
print('Article description updated.\n')
else:
print('Edit cancelled, article description unchanged')
else:
print('Edit cancelled, article description unchanged')
def get_stats(start_date=None, end_date=None):
#del command
print('Get stats between two dates')
if not start_date:
start_date = btc.read_text('Enter start date: ')
if not end_date:
end_date = btc.read_text('Enter end date: ')
try:
get_date_range_category_stats(start_date, end_date)
except Exception as e:
print(e)
return
def get_csv_in_directory():
file_list = glob.glob('*.csv')
print('Files available: ')
for item in file_list:
print(item, end='\n')
#try:
importing_file_name = btc.read_text(
'Enter filename from the list or ". " to cancel: ')
if importing_file_name != '.':
filename = '{0}.csv'.format(importing_file_name)
csv_articles = create_csv_list(filename)
print(csv_articles)
print('Articles to import:')
try:
for article in csv_articles:
try:
csv_article = csv_item_to_article(article)
db.add_article_from_csv(csv_article)
print(csv_article.name + " was added to database.\n")
except IndexError:
print('Add article failed')
print('Import complete, return to main menu')
except TypeError:
print('File not found')
return
def update_article_author(article_id):
article = db.get_article(article_id)
if article == None:
print("There is no article with that ID. article NOT found.\n")
else:
print()
display_single_article(article, str(article.ArticleID))
article_choice = btc.read_int_ranged(
'1 to edit article author, 2 to leave as is: ',
min_value=1,
max_value=2)
if article_choice == 1:
new_author = btc.read_text(
'Enter new author name or . to cancel: ')
if new_author != '.':
db.update_article_author(article_id, new_author)
else:
print('Edit cancelled, article title unchanged')
def display_articles_by_author(author_snippet=None):
if not author_snippet:
author_snippet = btc.read_text("Author Name: ")
articles = db.display_articles_by_author(author_snippet)
if articles == None:
print("There are no articles by that author. article NOT found.\n")
else:
print()
display_articles(articles, "AUTHOR: " + str(articles[0].author))
def update_category(category_id=0):
if category_id == 0:
category_id = int(input("category ID: "))
category = db.get_category(category_id)
#articles = db.get_articles_by_category_id(category_id)
#display_articles(articles, category.category_name.upper())
print('Current category name: {0}'.format(category.category_name))
new_category_name = btc.read_text(
"Enter new category name or '.' to cancel: ")
if new_category_name != '.':
update_choice = btc.read_int_ranged(
"1 to change article name to {0}, 2 to cancel: ".format(
new_category_name), 1, 2)
if update_choice == 1:
db.update_category(category_id, new_category_name)
print('Category update complete\n')
elif update_choice == 2:
print('Update cancelled.\n')
def do_qa(self, args):
'''Adds an article with a single type of the command line.
quick_add url cat_id date'''
if args.category_id:
if args.description:
app.qa(session=a.d.session,
url=args.url,
category_id=args.category_id,
date=args.date,
description=' '.join(args.description))
if not args.description:
app.qa(session=a.d.session,
url=args.url,
category_id=args.category_id,
date=args.date)
else:
print('category does not exist')
category_name = btc.read_text(
'Enter new category name or "." to cancel: ')
if category_name != ".":
app.display_sections()
section_choice = btc.read_int('Enter section ID: ')
app.add_pub_or_cat(session=a.d.session,
add_type='category',
new_name=category_name,
second_item=section_choice)
new_cat = a.d.session.query(Category).filter(
Category.name.like(f'%{category_name}%')).one()
print('the new category is: ', new_cat)
#print(type(new_category))
if not args.description:
#we convert the category id to a string so it passes the assertion test in the app file
app.qa(session=a.d.session,
url=args.url,
category_id=str(new_cat.category_id),
date=args.date)
else:
app.qa(session=a.d.session,
url=args.url,
category_id=str(new_cat.category_id),
date=args.date,
description=' '.join(args.description))
def update_article(article_id, update_type):
update_types = {'author', 'publication', 'date', 'category_id'}
if update_type not in update_types:
print('Invalid update type')
return
else:
article = db.get_article(article_id)
if article == None:
print("There is no article with that ID. article NOT found.\n")
else:
print()
display_single_article(article, str(article.articleID))
article_choice = btc.read_int_ranged(
'1 to edit article {0}, 2 to leave as is: '.format(
update_type),
min_value=1,
max_value=2)
if article_choice == 1:
if update_type == 'date':
new_value = btc.read_date(
'Enter new date or . to cancel: ')
elif update_type == 'category_id':
display_categories()
new_value = btc.read_int('Enter new category_id: ')
new_category = db.cat_from_snippet(new_value)
if new_category == None: #Make sure the new category exists
print(
'There is no category with that ID, article category NOT updated.\n'
)
return
#If the new category exists, then we go on to update the article
else:
new_value = btc.read_text(
'Enter new {0} or . to cancel: '.format(update_type))
if new_value != '.':
db.update_article(article_id=article_id,
new_value=new_value,
update_type=update_type)
else:
print('Edit cancelled, return to main menu')
return
else:
print('Edit cancelled, article title unchanged')
def get_articles_by_category(category=None, start_date=None, end_date=None):
if not category:
category = btc.read_text("Enter category name or number here: ")
if not start_date:
start_date = btc.read_date("Enter starting date: ")
#start_date = parse(start_date)
if not end_date:
end_date = btc.read_date("Enter ending date: ")
if category.isalpha() == True:
from_snippet(snippet=category,
start_date=start_date,
end_date=end_date,
snippet_type='category')
elif category.isnumeric() == True:
from_snippet(snippet=category,
start_date=start_date,
end_date=end_date,
snippet_type='category_id')
def get_articles_by_category(category=None, start_date=None, end_date=None):
#This is a function slated for replacement as I streamline the codebase
#of this app.
if not category:
category = btc.read_text("Enter category name or number here: ")
if not start_date:
start_date = btc.read_date("Enter starting date: ")
if not end_date:
end_date = btc.read_date("Enter ending date: ")
if category.isalpha() == True:
from_snippet(snippet=category,
start_date=start_date,
end_date=end_date,
snippet_type='category')
elif category.isnumeric() == True:
from_snippet(snippet=category,
start_date=start_date,
end_date=end_date,
snippet_type='category_id')
def from_input(cls):
try:
print('Manual section creation')
print('Press "." to cancel')
name = btc.read_text('Section name: ')
return cls(section_name=name)
except Exception as e:
print(e)
return
except KeyboardInterrupt:
print('Ctrl+C pressed, add category cancelled')
def from_input(cls):
try:
print('Manual category creation')
print('Press "." to cancel')
name = btc.read_text('Category name: ')
#description = read_text('Description: ')
return cls(category_name=name)
except Exception as e:
print(e)
return
except KeyboardInterrupt:
print('Ctrl+C pressed, add category cancelled')
def export_roundup_by_category():
display_categories()
roundup_categories = db.get_categories()
categories_remaining = len(roundup_categories)
categories_for_roundup = []
for category in roundup_categories:
print('Categories remaining: {0}'.format(categories_remaining))
print('Include {0}'.format(category.category_name))
category_choice = btc.read_int_ranged('1 to include, 2 to exclude: ',
1, 2)
if category_choice != 1:
categories_for_roundup.append(category)
roundup_title = btc.read_text('Enter the roundup title: ')
roundup_month = btc.read_int_ranged('Enter roundup month: ', 1, 12)
roundup_year = btc.read_int_ranged('Enter roundup year: ', 1, 2100)
filename = btc.read_text('Enter roundup filename: ')
roundup_choice = btc.read_int_ranged(
'Enter 1 to export roundup, 2 to cancel: ', 1, 2)
if roundup_choice == 1:
for category in categories_for_roundup:
# for category in roundup_categories:
category.articles = db.get_articles_for_roundup(
roundup_month, roundup_year, category.id)
roundup_docx.create_complete_roundup(filename=filename,
roundup_title=roundup_title,
categories=categories_for_roundup)
#display_title()
elif roundup_choice == 2:
print('Roundup export cancelled. Return to main menu.\n')
def export_roundup_by_date():
roundup_title = btc.read_text('Enter the roundup title: ')
start_date = btc.read_date('Enter the starting date: ')
end_date = btc.read_date('Enter the ending date: ')
print('start date: ', start_date, 'end date: ', end_date)
print('start date type:', type(start_date), 'end date type:',
type(end_date))
filename = btc.read_text('Enter roundup filename: ')
roundup_choice = btc.read_int_ranged(
'Enter 1 to export roundup, 2 to cancel: ', 1, 2)
if roundup_choice == 1:
roundup_categories = db.get_categories() #We get the articles by
#category to sort them by category
for category in roundup_categories:
category.articles = db.get_articles_for_roundup(
start_date, end_date, category.categoryID)
print(len(category.articles))
roundup_docx.create_complete_roundup(filename=filename,
roundup_title=roundup_title,
categories=roundup_categories)
elif roundup_choice == 2:
print('Roundup export cancelled. Return to main menu.\n')
def update_article_name(article_id):
article = db.get_article(article_id)
if article == None:
print("There is no article with that ID. article NOT found.\n")
else:
print()
display_single_article(article, str(article.ArticleID))
article_choice = btc.read_int_ranged(
'1 to edit article title, 2 to leave as is: ',
min_value=1,
max_value=2)
if article_choice == 1:
try:
newsItem1 = na.get_article_from_url(article.link)
updated_title = newsItem1.title
except Exception as e:
print('Scrape failed because of {0}'.format(e))
updated_title = 'Invalid'
print('Rescraped title: {0}'.format(updated_title))
title_choice = btc.read_int_ranged(
'1 - existing title, 2 - scraped title, 3 - manual input: ', 1,
3)
if title_choice == 1:
print('Title update cancelled, article title unchanged.')
return
elif title_choice == 2:
db.update_article_name(article_id, updated_title)
print('Title update complete. Return to main menu.')
elif title_choice == 3:
new_title = btc.read_text('Enter new title or . to cancel: ')
if new_title != '.':
db.update_article_name(article_id, new_title)
else:
print('Edit cancelled, return to main menu')
return
else:
print('Edit cancelled, article title unchanged')
def from_input(cls, link, category):
try:
print('Manual article creation')
print('Press Ctrl+C to cancel')
if not link:
print('No link supplied, manual_add must be followed by link')
return
name = btc.read_text('Article title: ')
new_date = btc.read_date('Article date: ')
author = btc.read_text('Author: ')
publication = btc.read_text('Publication: ')
category = category
description = btc.read_text('Description: ')
return cls(link=link, name=name, date=new_date,
author=author, publication=publication,
category=category, description=description)
except Exception as e:
print(e)
return
pass
except KeyboardInterrupt:
print('Ctrl+C pressed, add article cancelled')
def finalize_desc_month(command):
if not command or command == '':
new_month = btc.read_int_ranged('Enter new month: ',
min_value=1,
max_value=12)
new_year = btc.read_int_ranged('Enter new year: ',
min_value=1,
max_value=2100)
articles_to_finalize = db.get_articles_by_month(month=new_month,
year=new_year)
articles_remaining = len(articles_to_finalize)
for article in articles_to_finalize:
print('{0} unreviewed articles'.format(articles_remaining))
update_article_description(article.ArticleID)
description_choice = btc.read_int_ranged(
'{0} descriptions remaining. Press 1 to continue, 2 to cancel: '
.format(articles_remaining), 1, 2)
articles_remaining -= 1
if description_choice == 2:
print('Update descriptions cancelled')
break
def update_article_category(article_id):
article = db.get_article(article_id)
if article == None:
print("There is no article with that ID. article NOT found.\n")
else:
print()
display_single_article(article, str(article.ArticleID))
article_choice = btc.read_int_ranged(
'1 to edit article category, 2 to leave as is: ',
min_value=1,
max_value=2)
if article_choice == 1:
new_category_id = btc.read_int('Enter new category_id: ')
result = db.get_category(new_category_id)
#Add in some text that is displayed to make it clear that the category is being updated
if result == None:
print(
'There is no category with that ID, article category NOT updated.\n'
)
else:
db.update_article_category(article_id, new_category_id)
else:
print('Edit cancelled, article title unchanged')
def update_article_date(article_id):
article = db.get_article(article_id)
if article == None:
print("There is no article with that ID. article NOT found.\n")
else:
print()
display_single_article(article, str(article.ArticleID))
article_choice = btc.read_int_ranged(
'1 to edit article date, 2 to leave as is: ',
min_value=1,
max_value=2)
if article_choice == 1:
new_date = btc.read_date('Enter new date: ')
date_choice = btc.read_int_ranged(
'1 to change date to: {0}, 2 to cancel: '.format(new_date),
min_value=1,
max_value=2)
if date_choice == 1:
db.update_article_date(article_id, new_date)
print('Update complete.\n')
elif date_choice == 2:
print('Edit cancelled, article date unchanged')
else:
print('Edit cancelled, article date unchanged')
def delete_section(command):
del command
section_id = int(input("section ID: "))
categories_in_section = db.get_section_categories(section_id=section_id)
if categories_in_section > 0:
print('Category contains articles, cannot be deleted')
elif categories_in_section == 0:
delete_choice = btc.read_bool(decision='Are you sure?',
yes='1',
no='2',
yes_option='delete',
no_option='cancel')
if delete_choice == True:
db.delete_item(item_id=section_id, item_type='section')
else:
print('Delete cancelled, returning to category menu')
def delete_category():
category_id = int(input("category ID: "))
articles_in_category = db.get_article_count(category_id=category_id,
start_date=None,
end_date=None)
if articles_in_category > 0:
print('Category contains articles, cannot be deleted')
elif articles_in_category == 0:
delete_choice = btc.read_bool(decision='Are you sure?',
yes='1',
no='2',
yes_option='delete',
no_option='cancel')
if delete_choice == True:
db.delete_item(item_id=category_id, item_type='category')
else:
print('Delete cancelled, returning to category menu')
def display_articles_by_publication(publication_snippet=None):
if not publication_snippet:
publication_snippet = btc.read_text("Publication: ")
publications = db.display_articles_by_publication(publication_snippet)
if publications == None:
print(
"There are no articles by that publication. article NOT found.\n")
return
else:
print()
try:
display_articles(
publications,
"PUBLICATION: " + str(publications[0].publication))
except IndexError as e:
display_articles(publications,
"PUBLICATION: " + str('Error: {0}'.format(e)))
def finalize_title_updates(month, year):
articles = db.get_articles_by_month(month=month, year=year)
articles_remaining = len(articles)
for article in articles:
print('{0} articles remaining'.format(articles_remaining))
display_single_article(article, title_term=article.ArticleID)
strip_choice = btc.read_int_ranged(
'1 to update title, 2 to skip, 3 to return to main menu: ', 1, 3)
if strip_choice == 1:
update_article_name(article.ArticleID)
articles_remaining -= 1
if strip_choice == 2:
articles_remaining -= 1
print('Article title unchanged.')
if strip_choice == 3:
print('strip titles cancelled')
#display_title()
break
def finalize_article_descriptions(start_date, end_date):
#undescribed = db.get_date_range_undescribed_articles(description_snippet='Not specified',
# start_date=start_date,
# end_date=end_date)
undescribed = db.get_snippet(snippet='Not specified',
start_date=start_date,
end_date=end_date,
snippet_type='description')
#We call the get_snippet function with "Not specified" as the description
#because that's the description for all undescribed articles imported
#from csv files
undescribed_articles = len(undescribed)
print('Total undescribed articles: {0}'.format(undescribed_articles))
for article in undescribed:
print(
'Undescribed articles remaining: {0}'.format(undescribed_articles))
update_article_description(article.ArticleID)
undescribed_articles -= 1
description_choice = btc.read_int_ranged(
'{0} descriptions remaining. Press 1 to continue, 2 to cancel: '.
format(undescribed_articles), 1, 2)
if description_choice == 2:
print('Update descriptions cancelled')
break
def manual_add(link=None):
category = btc.read_int('Enter category for article: ')
category = db.get_category(category)
new_article = Article.manual_add(link=link, category=category)
db.add_article(new_article)
print(new_article.title + " was added to database.\n")
def from_newspaper(link):
'''
Adds an article from the newspaper module after downloading it
'''
for i in tqdm.tqdm(range(1)):
try:
newNewsItem = na.get_article_from_url(link)
except:
print('Article download failed, invalid URL')
print('Returning to main menu')
return
print(newNewsItem)
try:
name = newNewsItem.title #get the title for the article
except Exception as e:
print(e)
name = btc.read_text('Please enter title: ')
#get article author
try:
author = ' '.join(newNewsItem.authors)
#get article publication
except Exception as e:
print(e)
author = btc.read_text('Please enter author: ')
try:
#works for most websites, but not Sudan Tribune
publication = newNewsItem.meta_data['og']['site_name']
except Exception as e:
print(e)
publication = btc.read_text('Please enter publication: ')
try:
year = newNewsItem.publish_date.year
month = newNewsItem.publish_date.month
day = newNewsItem.publish_date.day
new_date = datetime.date(day=day, month=month, year=year)
except Exception as e:
print(e)
#use the new btc.read_date() function to simplify this
try:
new_date = btc.read_date('Enter article date MM/DD/YYYY: ')
except Exception as e:
print('invalid date', e)
try:
summary = newNewsItem.summary
except Exception as e:
print(e)
print('Summary download failed')
summary = 'Summary not found'
try:
keywords = ', '.join(newNewsItem.keywords)
except Exception as e:
print(e)
print('Keyword download failed')
keywords = 'keywords not found'
print('TITLE - {0} - AUTHOR {1}'.format(name, author))
print('DATE - {0} - PUBLICATION {1}'.format(new_date.strftime("%m/%d/%Y"),
publication))
print('KEYWORDS: ', keywords)
display_categories()
category_id = btc.read_text("Category ID: ")
category = db.get_category(category_id)
if category == None:
print('There is no category with that ID. article NOT added.\n')
return
description_choice = btc.read_text('View article description? y/n: ')
if description_choice == 'y':
print('Title: {0}'.format(name))
print('Summary: {0}'.format(summary))
print('Keywords: {0}'.format(keywords))
description = btc.read_text("Description or '.' to cancel: ")
if description == ".":
return
else:
new_article = Article(name=name,
date=new_date,
category=category,
link=link,
description=description,
author=author,
publication=publication)
display_single_article(article=new_article,
title_term=new_article.name)
confirm_article = btc.read_bool(decision="Finalize the article?",
yes='y',
no='n',
yes_option='Confirm',
no_option='Cancel')
#This is the user's last chance to decide if they want to add the article
if confirm_article == True:
db.add_article(new_article)
print(new_article.name + " was added to database.\n")
elif confirm_article == False:
print('Article add cancelled. Return to main menu.')