def start(self):
if self.email_textfield.get() == '' and \
self.radio_var.get() == self.NEW_EMAIL:
self.warn_var.set('Enter an email')
return
else:
self.receiver = self.email_textfield.get()
debug.d_print('Receiver: ', self.receiver)
if self.mode == self.DOI_SEARCH_MODE:
self.start_button.config(state="disabled")
# starts a thread
self.doi_search_thread = threading.Thread(
name='doi-search-worker', target=self.doi_search_worker)
# self.doi_search_thread.setDaemon(True)
self.doi_search_thread.start()
self.disable_all_buttons()
self.warn_var.set('Started')
elif self.mode == self.REALITY_CHECK_MODE:
self.start_button.config(state="disabled")
# starts a thread
self.reality_check_thread = threading.Thread(
name='reality-check-worker', target=self.reality_check_worker)
# self.reality_check_thread.setDaemon(True)
self.reality_check_thread.start()
self.disable_all_buttons()
self.warn_var.set('Started')
def doi_search_worker(self):
self.top_message_var.set('DOI-SEARCH')
self.file_var.set(self.input_file_path.split('/')[-1])
self.search_article()
# after doi search is done
self.warn_var.set('DOI Search FINISHED')
self.output_file_path = self.main_system.continue_output_file_path
debug.d_print('Result: ', self.output_file_path)
self.top_message_var.set('REALITY CHECK READY')
self.continue_button.config(state="normal")
self.continue_button_var.set(self.continue_msg +
self.output_file_path.split('/')[-1] +
'.csv')
self.enable_initial_buttons()
if self.no_doi_file_warning:
self.warn_var.set(self.NO_DOI_MSG + '\n' +
'DOI-Search finished.\nEmail has been sent.')
messagebox.showwarning(
'NO-DOI File', 'NO-DOI file has entries. '
'Check No-DOI file.')
self.no_doi_file_warning = False
else:
self.warn_var.set('DOI-Search finished.\nEmail has been sent.')
def send(self, file_array):
if self.receiver == "":
debug.d_print("There is no specified receiver")
return
msg = MIMEMultipart()
msg['Subject'] = self.subject
msg['From'] = self.from_name + " <" + self.sender + ">"
msg['To'] = self.receiver
body = self.body
msg.attach(MIMEText(body, "plain"))
# Read the supplied file
for file_path in file_array:
fp = open(file_path, "rb")
attachment = MIMEBase("text", "csv")
attachment.set_payload(fp.read())
fp.close()
encoders.encode_base64(attachment)
f_name = file_path.split('/')[-1] # get only the name.csv
attachment.add_header("Content-Disposition",
"attachment",
filename=f_name)
msg.attach(attachment)
with SMTP(self.smtp_server, self.port) as server:
server.ehlo()
debug.d_print("Sending message...")
server.sendmail(self.sender, self.receiver, msg.as_string())
def reality_check_worker(self):
self.top_message_var.set('REALITY CHECK')
self.file_var.set(self.input_file_path.split('/')[-1])
self.check_reality()
# after reality check is done
debug.d_print('Result:', 'Reality check has been finished.')
self.warn_var.set('Reality Check FINISHED\nEmail has been sent.')
self.enable_initial_buttons()
self.disable_email_widgets()
def upload_file(self):
"""
Allows a user to browse a csv file and upload it.
:return:
"""
self.input_file_path = filedialog.askopenfilename(
initialdir="currdir",
title="Select File",
filetypes=(("csv files", "*.csv"), ("all files", "*.*")))
if self.input_file_path == '':
debug.d_print('File select canceled.')
self.disable_start_button()
self.disable_email_widgets()
self.file_var.set('no file')
return
f_name = self.input_file_path.split('/')[-1] # get only the name.csv
debug.d_print('Uploaded: ', self.input_file_path)
self.file_var.set(f_name)
self.file_name = f_name
self.continue_button.configure(
state='disabled') # continue button disabled
# checks if the uploaded file is valid
with open(self.input_file_path, 'r', encoding='utf8') as csv_file:
reader = csv.reader(csv_file)
header = next(reader) # only for python 3
debug.d_print('Columns: ', header)
if header == self.KBART_HEADER:
self.mode = self.MODE_NOT_SET
debug.d_print('*This is the standard format (not accepted)')
elif header == self.JOURNAL_CSV_HEADER or header == self.JOURNAL_RESULT_CSV_HEADER \
or header == self.OXFORD_HEADER:
self.mode = self.DOI_SEARCH_MODE
self.is_ready = True
self.start_button.config(state="normal")
self.top_message_var.set('DOI-SEARCH')
self.warn_var.set('')
self.enable_email_widgets()
elif header == self.TEMP_CSV_HEADER:
self.mode = self.REALITY_CHECK_MODE
self.is_ready = True
self.start_button.config(state="normal")
self.top_message_var.set('REALITY CHECK')
self.warn_var.set('')
self.enable_email_widgets()
else:
self.mode = self.MODE_NOT_SET
self.warn_var.set('Wrong file (wrong columns)')
self.start_button.config(state="disabled")
self.main_system.update(MainUI.FILE_UPLOADED)
def restore_progress(self):
debug.d_print('|progress restored|')
self.ui.restore_ui(self.status) # ui is recovered.
if self.status == 'doi-search':
self.create_journal_list()
# self.iterate_journal_list(self.DOI_SEARCH_MODE)
doi_search_thread = threading.Thread(
name='doi-search-worker', target=self.ui.doi_search_worker)
# doi_search_thread.setDaemon(True) #
doi_search_thread.start()
elif self.status == 'reality-check':
self.recreate_journal_list()
# self.iterate_journal_list(self.REALITY_CHECK_MODE)
reality_check_thread = threading.Thread(
name='doi-search-worker', target=self.ui.reality_check_worker)
# reality_check_thread.setDaemon(True) #
reality_check_thread.start()
def send_email(self, mode):
"""
Send the result file to a specified email address.
:return:
"""
use_server = False
self.config = configparser.ConfigParser()
self.config.read(config_utils.config.PATH_TO_SMTP_SERVER_INI)
if use_server:
# Sender is a server
emailer = email_server.EmailHandler(
smtp_server=self.config['email-info']['smtp-server'],
port=int(self.config['email-info']['port']),
from_name=self.config['email-info']['from'],
sender=self.config['email-info']['sender'],
domain=self.config['email-info']['domain'],
subject=self.config['email-info']['subject'],
body=self.config['email-info']
['body']) # using a server name to send
else:
# Sender is personal address
emailer = email_handler.EmailHandler()
emailer.set_sender(sender=self.sender, password=self.password)
emailer.set_receiver(receiver=self.receiver)
emailer.set_subject(subject=mode + ' finished')
emailer.set_body(body='The ' + mode +
' has been finished. Two are files attached.\n\n')
f1 = csv_reader.path + self.output_file_path + '.csv'
f2 = csv_reader.path + self.wrong_file_path + '.csv'
files = [f1, f2]
try:
emailer.send(files)
except smtplib.SMTPRecipientsRefused:
print('Email was incorrect')
debug.d_print('Email has been sent.')
def send(self, file_array):
if not self.valid_sender:
debug.d_print("Can not send an email without a valid sender")
return
if self.receiver == "":
debug.d_print("There is no specified receiver")
return
msg = MIMEMultipart()
msg['Subject'] = self.subject
msg['From'] = 'UPEI REALITY CHECK SYSTEM'
msg['To'] = self.receiver
body = self.body
msg.attach(MIMEText(body, "plain"))
# Read the supplied file
for file_path in file_array:
fp = open(file_path, "rb")
attachment = MIMEBase("text", "csv")
attachment.set_payload(fp.read())
fp.close()
encoders.encode_base64(attachment)
f_name = file_path.split('/')[-1] # get only the name.csv
attachment.add_header("Content-Disposition",
"attachment",
filename=f_name)
msg.attach(attachment)
with smtplib.SMTP_SSL(self.smtp_server, self.port) as server:
server.ehlo()
server.login(self.sender, self.password)
debug.d_print("Sending message...")
server.sendmail(self.sender, self.receiver, msg.as_string())
def check_reality(self, journal):
"""
Screen scrape and determine the journal reality.
:param journal: a journal object
:return:
"""
for year in journal.year_dict:
article = journal.year_dict[year][self.ARTICLE]
doi = article.doi
if doi is None:
debug.d_print_detail(str(year), ':', 'no-doi')
article.accessible = False
article.result = 'No-DOI'
else:
try:
result = screenscraper.check_journal(
doi, journal.package) # reality check
exception_details = ['', '', '']
except Exception as ex:
template = "An exception of type {0} occurred. Arguments:\n{1!r}"
message = template.format(type(ex).__name__, ex.args)
exception_details = [
type(ex).__name__, ex.args,
traceback.format_exc()
]
debug.d_print(message)
debug.d_print(year)
debug.d_print('|exception happened|')
result = result_enum.Result.OtherException
article.result = result # result is stored in article
if result is result_enum.Result.Access:
article.accessible = True
if result is result_enum.Result.OpenAccess:
article.accessible = True
article.open = True
if result is result_enum.Result.FreeAccess:
article.accessible = True
article.free = True
if result is result_enum.Result.OtherException:
article.exception = True
article.exception_details = exception_details
article.result = self.convert_result(
result) # result is checked
debug.d_print_detail(str(year), ':', str(result), '=',
'https://doi.org/' + str(doi))
journal.record_wrong_years() # wrong years are updated
journal.record_free_years() # free years are updated
if status == self.main_system.DOI_SEARCH_MODE:
self.mode = self.DOI_SEARCH_MODE
if status == self.main_system.REALITY_CHECK_MODE:
self.mode = self.REALITY_CHECK_MODE
self.is_ready = False
self.receiver = self.main_system.receiver
self.temp_receiver = ''
def reset_member_variables(self):
# threads
self.doi_search_thread = None
self.reality_check_thread = None
# member variables
self.input_file_path = None
self.output_file_path = None
self.wrong_file_path = None
self.file_name = None
self.mode = self.MODE_NOT_SET
self.is_ready = False
# self.receiver = self.main_system.receiver
self.temp_receiver = ''
self.no_doi_file_warning = False
def set_no_doi_file_warning(self):
self.no_doi_file_warning = True
if __name__ == '__main__':
debug.d_print('ui')
def main():
main_system = MainSystem()
debug.d_print('"PROGRAM TERMINATED"')
def __init__(self):
debug.d_print("system turned on\n")
self.journal_list = None
# Config for the progress
self.config = configparser.ConfigParser()
self.config.read(config_utils.config.PATH_TO_PROGRESS_INI)
self.complete = None
# handles no progress.ini exists or file path is incorrect
try:
self.complete = self.config['progress']['complete'] # PROBLEM
except KeyError as e: # if
progress_ini_path = config_utils.config.clear_progress()
self.config.read(progress_ini_path)
self.complete = self.config['progress']['complete']
debug.d_print('new progress.ini created')
# Config for progress
self.status = self.config['progress']['status']
self.current_index = int(self.config['progress']['current-index'])
self.input_file_path = self.config['progress']['input-file-path']
self.output_file_path = self.config['progress']['output-file-path']
self.wrong_file_path = self.config['progress']['wrong-file-path']
self.exception_file_path = self.config['progress'][
'exception-file-path']
self.file_name = None
self.continue_output_file_path = None
# Config for email
self.email_config = configparser.ConfigParser()
self.email_config.read(config_utils.config.PATH_TO_EMAIL_INI)
self.sender = self.email_config['email']['sender']
self.receiver = self.email_config['email']['receiver']
self.password = self.email_config['email']['password']
# Prepare UI
self.ui = None
self.root = tk.Tk()
self.root.title("Journal Reality Checking System")
self.root.geometry("600x400")
if self.complete == 'False':
title = self.status
msg = 'Do you want to recover the progress last time?'
yes = popup.resume_yesno(title, msg)
if yes:
self.ui = main_ui.MainUI(master=self.root, main_system=self)
self.restore_progress()
self.ui.mainloop() # starts UI
else:
config_utils.config.clear_progress()
self.reset_member_variables()
# Instantiate MainUI class object
self.ui = main_ui.MainUI(master=self.root, main_system=self)
self.ui.mainloop() # starts UI
else:
# Instantiate MainUI class object
self.ui = main_ui.MainUI(master=self.root, main_system=self)
self.ui.mainloop() # starts UI
def iterate_journal_list(self, mode):
"""
Iterates a journal list
:param mode: 'doi-search' or 'reality-check'
:return:
"""
# prints start messages
if mode == self.DOI_SEARCH_MODE:
debug.d_print('\n===============DOI-SEARCH=================')
if mode == self.REALITY_CHECK_MODE:
debug.d_print('\n==============REALITY-CHECK===============')
# updates email address
config_utils.config.update_email(self.receiver)
# retrieve index
index = self.current_index
# if there is no progress remaining
if index == self.LAST_FINISHED: # -1
d = str(datetime.datetime.today())
date = d[5:7] + d[8:10]
# date = d[0:4] + d[5:7] + d[8:10] + '-' + d[11:13] + d[14:16]
input_file_name = self.input_file_path.split('/')[-1][0:-4]
# decides names of the result files
if mode == self.DOI_SEARCH_MODE:
self.output_file_path = 'TEMP-DOI-' + date + '_from_' + input_file_name # file name
self.wrong_file_path = 'NO-DOI-' + date + '_from_' + input_file_name
# self.output_file_path = date + '-TEMP-DOI' # file name
# self.wrong_file_path = date + '-NO-DOI'
elif mode == self.REALITY_CHECK_MODE:
self.output_file_path = 'RESULT-JOURNALS-' + date + '_from_' + input_file_name # file name
self.wrong_file_path = 'PROBLEM-JOURNALS-' + date + '_from_' + input_file_name
self.exception_file_path = 'EXCEPTION-JOURNALS-' + date + '_from_' + input_file_name
# self.output_file_path = date + '-RESULT-JOURNALS' # file name
# self.wrong_file_path = date + '-PROBLEM-JOURNALS'
# creates csv files for appending
if mode == self.DOI_SEARCH_MODE:
csv_reader.prepare_temp_csv(
self.output_file_path) # creates a csv temp file
elif mode == self.REALITY_CHECK_MODE:
csv_reader.prepare_result_csv(
self.output_file_path) # creates a csv temp file
csv_reader.prepare_exception_csv(
self.exception_file_path) # creates a csv temp file
csv_reader.prepare_wrong_csv(self.wrong_file_path)
index = 0
config_utils.config.update_progress(
self.input_file_path,
self.output_file_path,
self.wrong_file_path,
self.exception_file_path,
status=mode,
index=index,
title=self.journal_list[index].title)
# Iterates a list of journals using index
list_size = len(self.journal_list)
while index < list_size:
debug.d_print(index + 1, ":", self.journal_list[index])
title = self.journal_list[index].title
config_utils.config.update_progress(self.input_file_path,
self.output_file_path,
self.wrong_file_path,
self.exception_file_path,
status=mode,
index=index,
title=title)
if self.ui is not None:
self.ui.notify_progress(index + 1, list_size)
# doi-search or reality-check will be called
if not self.journal_list[index].has_problem:
if mode == self.DOI_SEARCH_MODE:
try:
self.search_article(
self.journal_list[index]) # DOI Search
except KeyError:
self.journal_list[index].has_problem = True
self.journal_list[
index].problem_detail = 'Unknown-DOI-Search-Error'
except json.decoder.JSONDecodeError:
self.journal_list[index].has_problem = True
self.journal_list[
index].problem_detail = 'JSON-Decode-Error-DOI-Search'
elif mode == self.REALITY_CHECK_MODE:
self.check_reality(
self.journal_list[index]) # Reality Check
# recording results into csv
if not self.journal_list[index].has_problem:
if mode == self.DOI_SEARCH_MODE:
csv_reader.append_doi_row(self.journal_list[index],
self.output_file_path)
elif mode == self.REALITY_CHECK_MODE:
csv_reader.append_journal_row(self.journal_list[index],
self.output_file_path)
else:
if mode == self.DOI_SEARCH_MODE:
csv_reader.append_problem_doi_row(self.journal_list[index],
self.output_file_path)
elif mode == self.REALITY_CHECK_MODE:
csv_reader.append_problem_journal_row(
self.journal_list[index], self.output_file_path)
# recording problems into csv
csv_reader.append_wrong_row(mode=mode,
journal=self.journal_list[index],
file_name=self.wrong_file_path)
# recording exceptions into csv
if mode == self.REALITY_CHECK_MODE and not self.journal_list[
index].has_problem:
csv_reader.append_exception_row(self.journal_list[index],
self.exception_file_path)
index = index + 1
# prints progresses
if not self.journal_list[index - 1].has_problem:
debug.d_print(index, '/', list_size,
'finished\n') # prints progress
else:
debug.d_print(index, '/', list_size,
'skipped\n') # prints progress
# the temp doi file is ready to be continued
if mode == self.DOI_SEARCH_MODE:
self.continue_output_file_path = 'Data-Files/Output-Files/' + self.output_file_path
if csv_reader.has_entry(self.wrong_file_path):
self.ui.wrong_file_path = self.wrong_file_path
self.ui.set_no_doi_file_warning()
# work is done. clear data
config_utils.config.clear_progress()
self.send_email(mode)
self.reset_member_variables()
# date2 = re.fullmatch('[0-9]{2}/[0-9]{2}/[0-9]{4}', date) # dd/mm/yyyy
# if date2 is not None:
# return date[6:10] + '-' + date[3:5] + '-' + date[0:2]
# date format is not expected
raise ValueError
if __name__ == '__main__':
# m = re.match('[0-9]{4}-[0-9]{2}-[0-9]{2}', '2000-3-21')
# n = re.match('[0-9]{2}/[0-9]{2}/[0-9]{4}', '33/43/3333')
# debug.d_print(m)
# debug.d_print(n)
b = 0
e = 1
debug.d_print(Journal.format_date('1991-12-12', b))
debug.d_print(Journal.format_date('1991-1-12', e))
debug.d_print(Journal.format_date('1991-12-2', b))
debug.d_print(Journal.format_date('1991-1-3', e))
debug.d_print(Journal.format_date('11/31/1993', b))
debug.d_print(Journal.format_date('1/31/1993', b))
debug.d_print(Journal.format_date('11/1/1993', b))
debug.d_print(Journal.format_date('1/1/1993', b))
debug.d_print(Journal.format_date('1991-12', b))
debug.d_print(Journal.format_date('1933-03', e))
debug.d_print(Journal.format_date('2091', b))
debug.d_print(Journal.format_date('2001', e))
debug.d_print(Journal.format_date('', e))
