def start(self): if self.email_textfield.get() == '' and \ self.radio_var.get() == self.NEW_EMAIL: self.warn_var.set('Enter an email') return else: self.receiver = self.email_textfield.get() debug.d_print('Receiver: ', self.receiver) if self.mode == self.DOI_SEARCH_MODE: self.start_button.config(state="disabled") # starts a thread self.doi_search_thread = threading.Thread( name='doi-search-worker', target=self.doi_search_worker) # self.doi_search_thread.setDaemon(True) self.doi_search_thread.start() self.disable_all_buttons() self.warn_var.set('Started') elif self.mode == self.REALITY_CHECK_MODE: self.start_button.config(state="disabled") # starts a thread self.reality_check_thread = threading.Thread( name='reality-check-worker', target=self.reality_check_worker) # self.reality_check_thread.setDaemon(True) self.reality_check_thread.start() self.disable_all_buttons() self.warn_var.set('Started')
def doi_search_worker(self): self.top_message_var.set('DOI-SEARCH') self.file_var.set(self.input_file_path.split('/')[-1]) self.search_article() # after doi search is done self.warn_var.set('DOI Search FINISHED') self.output_file_path = self.main_system.continue_output_file_path debug.d_print('Result: ', self.output_file_path) self.top_message_var.set('REALITY CHECK READY') self.continue_button.config(state="normal") self.continue_button_var.set(self.continue_msg + self.output_file_path.split('/')[-1] + '.csv') self.enable_initial_buttons() if self.no_doi_file_warning: self.warn_var.set(self.NO_DOI_MSG + '\n' + 'DOI-Search finished.\nEmail has been sent.') messagebox.showwarning( 'NO-DOI File', 'NO-DOI file has entries. ' 'Check No-DOI file.') self.no_doi_file_warning = False else: self.warn_var.set('DOI-Search finished.\nEmail has been sent.')
def send(self, file_array): if self.receiver == "": debug.d_print("There is no specified receiver") return msg = MIMEMultipart() msg['Subject'] = self.subject msg['From'] = self.from_name + " <" + self.sender + ">" msg['To'] = self.receiver body = self.body msg.attach(MIMEText(body, "plain")) # Read the supplied file for file_path in file_array: fp = open(file_path, "rb") attachment = MIMEBase("text", "csv") attachment.set_payload(fp.read()) fp.close() encoders.encode_base64(attachment) f_name = file_path.split('/')[-1] # get only the name.csv attachment.add_header("Content-Disposition", "attachment", filename=f_name) msg.attach(attachment) with SMTP(self.smtp_server, self.port) as server: server.ehlo() debug.d_print("Sending message...") server.sendmail(self.sender, self.receiver, msg.as_string())
def reality_check_worker(self): self.top_message_var.set('REALITY CHECK') self.file_var.set(self.input_file_path.split('/')[-1]) self.check_reality() # after reality check is done debug.d_print('Result:', 'Reality check has been finished.') self.warn_var.set('Reality Check FINISHED\nEmail has been sent.') self.enable_initial_buttons() self.disable_email_widgets()
def upload_file(self): """ Allows a user to browse a csv file and upload it. :return: """ self.input_file_path = filedialog.askopenfilename( initialdir="currdir", title="Select File", filetypes=(("csv files", "*.csv"), ("all files", "*.*"))) if self.input_file_path == '': debug.d_print('File select canceled.') self.disable_start_button() self.disable_email_widgets() self.file_var.set('no file') return f_name = self.input_file_path.split('/')[-1] # get only the name.csv debug.d_print('Uploaded: ', self.input_file_path) self.file_var.set(f_name) self.file_name = f_name self.continue_button.configure( state='disabled') # continue button disabled # checks if the uploaded file is valid with open(self.input_file_path, 'r', encoding='utf8') as csv_file: reader = csv.reader(csv_file) header = next(reader) # only for python 3 debug.d_print('Columns: ', header) if header == self.KBART_HEADER: self.mode = self.MODE_NOT_SET debug.d_print('*This is the standard format (not accepted)') elif header == self.JOURNAL_CSV_HEADER or header == self.JOURNAL_RESULT_CSV_HEADER \ or header == self.OXFORD_HEADER: self.mode = self.DOI_SEARCH_MODE self.is_ready = True self.start_button.config(state="normal") self.top_message_var.set('DOI-SEARCH') self.warn_var.set('') self.enable_email_widgets() elif header == self.TEMP_CSV_HEADER: self.mode = self.REALITY_CHECK_MODE self.is_ready = True self.start_button.config(state="normal") self.top_message_var.set('REALITY CHECK') self.warn_var.set('') self.enable_email_widgets() else: self.mode = self.MODE_NOT_SET self.warn_var.set('Wrong file (wrong columns)') self.start_button.config(state="disabled") self.main_system.update(MainUI.FILE_UPLOADED)
def restore_progress(self): debug.d_print('|progress restored|') self.ui.restore_ui(self.status) # ui is recovered. if self.status == 'doi-search': self.create_journal_list() # self.iterate_journal_list(self.DOI_SEARCH_MODE) doi_search_thread = threading.Thread( name='doi-search-worker', target=self.ui.doi_search_worker) # doi_search_thread.setDaemon(True) # doi_search_thread.start() elif self.status == 'reality-check': self.recreate_journal_list() # self.iterate_journal_list(self.REALITY_CHECK_MODE) reality_check_thread = threading.Thread( name='doi-search-worker', target=self.ui.reality_check_worker) # reality_check_thread.setDaemon(True) # reality_check_thread.start()
def send_email(self, mode): """ Send the result file to a specified email address. :return: """ use_server = False self.config = configparser.ConfigParser() self.config.read(config_utils.config.PATH_TO_SMTP_SERVER_INI) if use_server: # Sender is a server emailer = email_server.EmailHandler( smtp_server=self.config['email-info']['smtp-server'], port=int(self.config['email-info']['port']), from_name=self.config['email-info']['from'], sender=self.config['email-info']['sender'], domain=self.config['email-info']['domain'], subject=self.config['email-info']['subject'], body=self.config['email-info'] ['body']) # using a server name to send else: # Sender is personal address emailer = email_handler.EmailHandler() emailer.set_sender(sender=self.sender, password=self.password) emailer.set_receiver(receiver=self.receiver) emailer.set_subject(subject=mode + ' finished') emailer.set_body(body='The ' + mode + ' has been finished. Two are files attached.\n\n') f1 = csv_reader.path + self.output_file_path + '.csv' f2 = csv_reader.path + self.wrong_file_path + '.csv' files = [f1, f2] try: emailer.send(files) except smtplib.SMTPRecipientsRefused: print('Email was incorrect') debug.d_print('Email has been sent.')
def send(self, file_array): if not self.valid_sender: debug.d_print("Can not send an email without a valid sender") return if self.receiver == "": debug.d_print("There is no specified receiver") return msg = MIMEMultipart() msg['Subject'] = self.subject msg['From'] = 'UPEI REALITY CHECK SYSTEM' msg['To'] = self.receiver body = self.body msg.attach(MIMEText(body, "plain")) # Read the supplied file for file_path in file_array: fp = open(file_path, "rb") attachment = MIMEBase("text", "csv") attachment.set_payload(fp.read()) fp.close() encoders.encode_base64(attachment) f_name = file_path.split('/')[-1] # get only the name.csv attachment.add_header("Content-Disposition", "attachment", filename=f_name) msg.attach(attachment) with smtplib.SMTP_SSL(self.smtp_server, self.port) as server: server.ehlo() server.login(self.sender, self.password) debug.d_print("Sending message...") server.sendmail(self.sender, self.receiver, msg.as_string())
def check_reality(self, journal): """ Screen scrape and determine the journal reality. :param journal: a journal object :return: """ for year in journal.year_dict: article = journal.year_dict[year][self.ARTICLE] doi = article.doi if doi is None: debug.d_print_detail(str(year), ':', 'no-doi') article.accessible = False article.result = 'No-DOI' else: try: result = screenscraper.check_journal( doi, journal.package) # reality check exception_details = ['', '', ''] except Exception as ex: template = "An exception of type {0} occurred. Arguments:\n{1!r}" message = template.format(type(ex).__name__, ex.args) exception_details = [ type(ex).__name__, ex.args, traceback.format_exc() ] debug.d_print(message) debug.d_print(year) debug.d_print('|exception happened|') result = result_enum.Result.OtherException article.result = result # result is stored in article if result is result_enum.Result.Access: article.accessible = True if result is result_enum.Result.OpenAccess: article.accessible = True article.open = True if result is result_enum.Result.FreeAccess: article.accessible = True article.free = True if result is result_enum.Result.OtherException: article.exception = True article.exception_details = exception_details article.result = self.convert_result( result) # result is checked debug.d_print_detail(str(year), ':', str(result), '=', 'https://doi.org/' + str(doi)) journal.record_wrong_years() # wrong years are updated journal.record_free_years() # free years are updated
if status == self.main_system.DOI_SEARCH_MODE: self.mode = self.DOI_SEARCH_MODE if status == self.main_system.REALITY_CHECK_MODE: self.mode = self.REALITY_CHECK_MODE self.is_ready = False self.receiver = self.main_system.receiver self.temp_receiver = '' def reset_member_variables(self): # threads self.doi_search_thread = None self.reality_check_thread = None # member variables self.input_file_path = None self.output_file_path = None self.wrong_file_path = None self.file_name = None self.mode = self.MODE_NOT_SET self.is_ready = False # self.receiver = self.main_system.receiver self.temp_receiver = '' self.no_doi_file_warning = False def set_no_doi_file_warning(self): self.no_doi_file_warning = True if __name__ == '__main__': debug.d_print('ui')
def main(): main_system = MainSystem() debug.d_print('"PROGRAM TERMINATED"')
def __init__(self): debug.d_print("system turned on\n") self.journal_list = None # Config for the progress self.config = configparser.ConfigParser() self.config.read(config_utils.config.PATH_TO_PROGRESS_INI) self.complete = None # handles no progress.ini exists or file path is incorrect try: self.complete = self.config['progress']['complete'] # PROBLEM except KeyError as e: # if progress_ini_path = config_utils.config.clear_progress() self.config.read(progress_ini_path) self.complete = self.config['progress']['complete'] debug.d_print('new progress.ini created') # Config for progress self.status = self.config['progress']['status'] self.current_index = int(self.config['progress']['current-index']) self.input_file_path = self.config['progress']['input-file-path'] self.output_file_path = self.config['progress']['output-file-path'] self.wrong_file_path = self.config['progress']['wrong-file-path'] self.exception_file_path = self.config['progress'][ 'exception-file-path'] self.file_name = None self.continue_output_file_path = None # Config for email self.email_config = configparser.ConfigParser() self.email_config.read(config_utils.config.PATH_TO_EMAIL_INI) self.sender = self.email_config['email']['sender'] self.receiver = self.email_config['email']['receiver'] self.password = self.email_config['email']['password'] # Prepare UI self.ui = None self.root = tk.Tk() self.root.title("Journal Reality Checking System") self.root.geometry("600x400") if self.complete == 'False': title = self.status msg = 'Do you want to recover the progress last time?' yes = popup.resume_yesno(title, msg) if yes: self.ui = main_ui.MainUI(master=self.root, main_system=self) self.restore_progress() self.ui.mainloop() # starts UI else: config_utils.config.clear_progress() self.reset_member_variables() # Instantiate MainUI class object self.ui = main_ui.MainUI(master=self.root, main_system=self) self.ui.mainloop() # starts UI else: # Instantiate MainUI class object self.ui = main_ui.MainUI(master=self.root, main_system=self) self.ui.mainloop() # starts UI
def iterate_journal_list(self, mode): """ Iterates a journal list :param mode: 'doi-search' or 'reality-check' :return: """ # prints start messages if mode == self.DOI_SEARCH_MODE: debug.d_print('\n===============DOI-SEARCH=================') if mode == self.REALITY_CHECK_MODE: debug.d_print('\n==============REALITY-CHECK===============') # updates email address config_utils.config.update_email(self.receiver) # retrieve index index = self.current_index # if there is no progress remaining if index == self.LAST_FINISHED: # -1 d = str(datetime.datetime.today()) date = d[5:7] + d[8:10] # date = d[0:4] + d[5:7] + d[8:10] + '-' + d[11:13] + d[14:16] input_file_name = self.input_file_path.split('/')[-1][0:-4] # decides names of the result files if mode == self.DOI_SEARCH_MODE: self.output_file_path = 'TEMP-DOI-' + date + '_from_' + input_file_name # file name self.wrong_file_path = 'NO-DOI-' + date + '_from_' + input_file_name # self.output_file_path = date + '-TEMP-DOI' # file name # self.wrong_file_path = date + '-NO-DOI' elif mode == self.REALITY_CHECK_MODE: self.output_file_path = 'RESULT-JOURNALS-' + date + '_from_' + input_file_name # file name self.wrong_file_path = 'PROBLEM-JOURNALS-' + date + '_from_' + input_file_name self.exception_file_path = 'EXCEPTION-JOURNALS-' + date + '_from_' + input_file_name # self.output_file_path = date + '-RESULT-JOURNALS' # file name # self.wrong_file_path = date + '-PROBLEM-JOURNALS' # creates csv files for appending if mode == self.DOI_SEARCH_MODE: csv_reader.prepare_temp_csv( self.output_file_path) # creates a csv temp file elif mode == self.REALITY_CHECK_MODE: csv_reader.prepare_result_csv( self.output_file_path) # creates a csv temp file csv_reader.prepare_exception_csv( self.exception_file_path) # creates a csv temp file csv_reader.prepare_wrong_csv(self.wrong_file_path) index = 0 config_utils.config.update_progress( self.input_file_path, self.output_file_path, self.wrong_file_path, self.exception_file_path, status=mode, index=index, title=self.journal_list[index].title) # Iterates a list of journals using index list_size = len(self.journal_list) while index < list_size: debug.d_print(index + 1, ":", self.journal_list[index]) title = self.journal_list[index].title config_utils.config.update_progress(self.input_file_path, self.output_file_path, self.wrong_file_path, self.exception_file_path, status=mode, index=index, title=title) if self.ui is not None: self.ui.notify_progress(index + 1, list_size) # doi-search or reality-check will be called if not self.journal_list[index].has_problem: if mode == self.DOI_SEARCH_MODE: try: self.search_article( self.journal_list[index]) # DOI Search except KeyError: self.journal_list[index].has_problem = True self.journal_list[ index].problem_detail = 'Unknown-DOI-Search-Error' except json.decoder.JSONDecodeError: self.journal_list[index].has_problem = True self.journal_list[ index].problem_detail = 'JSON-Decode-Error-DOI-Search' elif mode == self.REALITY_CHECK_MODE: self.check_reality( self.journal_list[index]) # Reality Check # recording results into csv if not self.journal_list[index].has_problem: if mode == self.DOI_SEARCH_MODE: csv_reader.append_doi_row(self.journal_list[index], self.output_file_path) elif mode == self.REALITY_CHECK_MODE: csv_reader.append_journal_row(self.journal_list[index], self.output_file_path) else: if mode == self.DOI_SEARCH_MODE: csv_reader.append_problem_doi_row(self.journal_list[index], self.output_file_path) elif mode == self.REALITY_CHECK_MODE: csv_reader.append_problem_journal_row( self.journal_list[index], self.output_file_path) # recording problems into csv csv_reader.append_wrong_row(mode=mode, journal=self.journal_list[index], file_name=self.wrong_file_path) # recording exceptions into csv if mode == self.REALITY_CHECK_MODE and not self.journal_list[ index].has_problem: csv_reader.append_exception_row(self.journal_list[index], self.exception_file_path) index = index + 1 # prints progresses if not self.journal_list[index - 1].has_problem: debug.d_print(index, '/', list_size, 'finished\n') # prints progress else: debug.d_print(index, '/', list_size, 'skipped\n') # prints progress # the temp doi file is ready to be continued if mode == self.DOI_SEARCH_MODE: self.continue_output_file_path = 'Data-Files/Output-Files/' + self.output_file_path if csv_reader.has_entry(self.wrong_file_path): self.ui.wrong_file_path = self.wrong_file_path self.ui.set_no_doi_file_warning() # work is done. clear data config_utils.config.clear_progress() self.send_email(mode) self.reset_member_variables()
# date2 = re.fullmatch('[0-9]{2}/[0-9]{2}/[0-9]{4}', date) # dd/mm/yyyy # if date2 is not None: # return date[6:10] + '-' + date[3:5] + '-' + date[0:2] # date format is not expected raise ValueError if __name__ == '__main__': # m = re.match('[0-9]{4}-[0-9]{2}-[0-9]{2}', '2000-3-21') # n = re.match('[0-9]{2}/[0-9]{2}/[0-9]{4}', '33/43/3333') # debug.d_print(m) # debug.d_print(n) b = 0 e = 1 debug.d_print(Journal.format_date('1991-12-12', b)) debug.d_print(Journal.format_date('1991-1-12', e)) debug.d_print(Journal.format_date('1991-12-2', b)) debug.d_print(Journal.format_date('1991-1-3', e)) debug.d_print(Journal.format_date('11/31/1993', b)) debug.d_print(Journal.format_date('1/31/1993', b)) debug.d_print(Journal.format_date('11/1/1993', b)) debug.d_print(Journal.format_date('1/1/1993', b)) debug.d_print(Journal.format_date('1991-12', b)) debug.d_print(Journal.format_date('1933-03', e)) debug.d_print(Journal.format_date('2091', b)) debug.d_print(Journal.format_date('2001', e)) debug.d_print(Journal.format_date('', e))