def main(file = None, log_level = 0): log.silent('Initing parser on file: %s', file) if file is None: log.warning('Please run using the following: python parser.py path/to/file.name') exit() if not os.path.isfile(file): log.warning('File does not exist: %s', (file,)) exit() if os.path.getsize(file) == 0: log.warning('Can\'t parse an empty file - %s',(file,)) exit() QRTEParserLogger.LISTENING_THRESHOLD = 2 QRTEParser.parse(file, outfile=file.split('.')[0]+'_out.csv',exit_q_unique = False)
def ParseUploadedFile(datafile_pk): """ Parses the file indicated by the primary key. Emails an error reporting as well. :param datafile_pk: :return: """ datafile = DataFile.objects.get(pk=datafile_pk) QRTEParserLogger.LISTENING_THRESHOLD = -3 input_file = "%s%s.%s" % (settings.FILES_IN, datafile.name_system, datafile.file_type) output_file = settings.FILES_OUT + datafile.name_out print("Running parser on file: %s" % input_file) # Set stdout & err to log file result = None try: # Try parsing the file input_file = "%s%s.%s" % (settings.FILES_IN, datafile.name_system, datafile.file_type) output_file = settings.FILES_OUT + datafile.name_out result = QRTEParser.parse( file="%s%s.%s" % (settings.FILES_IN, datafile.name_system, datafile.file_type), outfile=settings.FILES_OUT + datafile.name_out) # Send out email notifying researcher that Parsing was completed. # Include link to generated filename # Update database with success/fail report. except QRTEParserException as e: # Handle caught QRTEParserException # Send email that QRTEParser ran into known exception. # Update database with fail report print(e) raise e pass except Exception as e: # Handle unknown exception # Send email that QRTEParser ran into unknown exception. Also send to [email protected] # Update database with fail report print(e) raise e pass print("Parsing finished with output file: %s" % output_file) pass
def ParseUploadedFile(datafile_pk): """ Parses the file indicated by the primary key. Emails an error reporting as well. :param datafile_pk: :return: """ datafile = DataFile.objects.get(pk=datafile_pk) QRTEParserLogger.LISTENING_THRESHOLD = -3 input_file = "%s%s.%s" % (settings.FILES_IN,datafile.name_system,datafile.file_type) output_file = settings.FILES_OUT + datafile.name_out print("Running parser on file: %s" % input_file) # Set stdout & err to log file result = None try: # Try parsing the file input_file = "%s%s.%s" % (settings.FILES_IN,datafile.name_system,datafile.file_type) output_file = settings.FILES_OUT + datafile.name_out result = QRTEParser.parse(file="%s%s.%s" % (settings.FILES_IN,datafile.name_system,datafile.file_type), outfile = settings.FILES_OUT + datafile.name_out) # Send out email notifying researcher that Parsing was completed. # Include link to generated filename # Update database with success/fail report. except QRTEParserException as e: # Handle caught QRTEParserException # Send email that QRTEParser ran into known exception. # Update database with fail report print(e) raise e pass except Exception as e: # Handle unknown exception # Send email that QRTEParser ran into unknown exception. Also send to [email protected] # Update database with fail report print(e) raise e pass print("Parsing finished with output file: %s" % output_file) pass
def ParseUploadedFile(datafile_pk): """ Parses the file indicated by the primary key. Emails an error reporting as well. :param datafile_pk: :return: """ datafile = DataFile.objects.get(pk=datafile_pk) QRTEParserLogger.LISTENING_THRESHOLD = 3 input_file = "%s%s.%s" % (settings.FILES_IN, datafile.name_system, datafile.file_type) log_file = settings.FILES_LOG + datafile.name_system + '.log' print("Running parser on file: %s" % input_file) # Set stdout & err to log file f = open(log_file, 'w') QRTEParserLogger.OUT = f result = None try: # Try parsing the file input_file = "%s%s.%s" % (settings.FILES_IN, datafile.name_system, datafile.file_type) output_file = settings.FILES_OUT + datafile.name_system + '.csv' result = QRTEParser.parse(file=input_file, outfile=output_file, entrance='WEB') # Send out email notifying researcher that Parsing was completed. tpl = get_template('email/success.html') send_mail(datafile.email, '[QRTEParser] Successfully parsed %s' % datafile.name_in, tpl.render({ 'download_url': datafile.host_name + reverse(download) + '?uuid=' + str(datafile.download_uuid), 'log_url': datafile.host_name + reverse(download_log) + '?uuid=' + str(datafile.download_uuid), 'report_mail': settings.EMAIL_SENDER, 'filename': datafile.name_in } )) # Update database with success/fail report. f.close() except QRTEParserException as e: # Handle caught QRTEParserException # Send email that QRTEParser ran into known exception. tpl = get_template('email/fail.html') send_mail(datafile.email, '[QRTEParser] Failed to parse %s' % datafile.name_in, tpl.render({ 'error_string': str(e), 'log_url': datafile.host_name + reverse(download_log) + '?uuid=' + str(datafile.download_uuid), 'report_mail': settings.EMAIL_SENDER, 'filename': datafile.name_in } )) # Update database with fail report print(e) f.close() raise e pass except Exception as e: # Handle unknown exception # Send email that QRTEParser ran into unknown exception. Also send to [email protected] tpl = get_template('email/fail.html') send_mail(datafile.email, '[QRTEParser] Failed to parse %s' % datafile.name_in, tpl.render({ 'error_string': str(e), 'log_url': datafile.host_name + reverse(download_log) + '?uuid=' + str(datafile.download_uuid), 'report_mail': settings.EMAIL_SENDER, 'filename': datafile.name_in } )) # Update database with fail report print(e) f.close() raise e pass print("Parsing finished with output file: %s" % output_file) f.close() pass