def main(file = None, log_level = 0):
log.silent('Initing parser on file: %s', file)
if file is None:
log.warning('Please run using the following: python parser.py path/to/file.name')
exit()
if not os.path.isfile(file):
log.warning('File does not exist: %s', (file,))
exit()
if os.path.getsize(file) == 0:
log.warning('Can\'t parse an empty file - %s',(file,))
exit()
QRTEParserLogger.LISTENING_THRESHOLD = 2
QRTEParser.parse(file, outfile=file.split('.')[0]+'_out.csv',exit_q_unique = False)
def ParseUploadedFile(datafile_pk):
"""
Parses the file indicated by the primary key. Emails an error reporting as well.
:param datafile_pk:
:return:
"""
datafile = DataFile.objects.get(pk=datafile_pk)
QRTEParserLogger.LISTENING_THRESHOLD = -3
input_file = "%s%s.%s" % (settings.FILES_IN, datafile.name_system,
datafile.file_type)
output_file = settings.FILES_OUT + datafile.name_out
print("Running parser on file: %s" % input_file)
# Set stdout & err to log file
result = None
try:
# Try parsing the file
input_file = "%s%s.%s" % (settings.FILES_IN, datafile.name_system,
datafile.file_type)
output_file = settings.FILES_OUT + datafile.name_out
result = QRTEParser.parse(
file="%s%s.%s" %
(settings.FILES_IN, datafile.name_system, datafile.file_type),
outfile=settings.FILES_OUT + datafile.name_out)
# Send out email notifying researcher that Parsing was completed.
# Include link to generated filename
# Update database with success/fail report.
except QRTEParserException as e:
# Handle caught QRTEParserException
# Send email that QRTEParser ran into known exception.
# Update database with fail report
print(e)
raise e
pass
except Exception as e:
# Handle unknown exception
# Send email that QRTEParser ran into unknown exception. Also send to [email protected]
# Update database with fail report
print(e)
raise e
pass
print("Parsing finished with output file: %s" % output_file)
pass
def ParseUploadedFile(datafile_pk):
"""
Parses the file indicated by the primary key. Emails an error reporting as well.
:param datafile_pk:
:return:
"""
datafile = DataFile.objects.get(pk=datafile_pk)
QRTEParserLogger.LISTENING_THRESHOLD = -3
input_file = "%s%s.%s" % (settings.FILES_IN,datafile.name_system,datafile.file_type)
output_file = settings.FILES_OUT + datafile.name_out
print("Running parser on file: %s" % input_file)
# Set stdout & err to log file
result = None
try:
# Try parsing the file
input_file = "%s%s.%s" % (settings.FILES_IN,datafile.name_system,datafile.file_type)
output_file = settings.FILES_OUT + datafile.name_out
result = QRTEParser.parse(file="%s%s.%s" % (settings.FILES_IN,datafile.name_system,datafile.file_type), outfile = settings.FILES_OUT + datafile.name_out)
# Send out email notifying researcher that Parsing was completed.
# Include link to generated filename
# Update database with success/fail report.
except QRTEParserException as e:
# Handle caught QRTEParserException
# Send email that QRTEParser ran into known exception.
# Update database with fail report
print(e)
raise e
pass
except Exception as e:
# Handle unknown exception
# Send email that QRTEParser ran into unknown exception. Also send to [email protected]
# Update database with fail report
print(e)
raise e
pass
print("Parsing finished with output file: %s" % output_file)
pass
def ParseUploadedFile(datafile_pk):
"""
Parses the file indicated by the primary key. Emails an error reporting as well.
:param datafile_pk:
:return:
"""
datafile = DataFile.objects.get(pk=datafile_pk)
QRTEParserLogger.LISTENING_THRESHOLD = 3
input_file = "%s%s.%s" % (settings.FILES_IN, datafile.name_system, datafile.file_type)
log_file = settings.FILES_LOG + datafile.name_system + '.log'
print("Running parser on file: %s" % input_file)
# Set stdout & err to log file
f = open(log_file, 'w')
QRTEParserLogger.OUT = f
result = None
try:
# Try parsing the file
input_file = "%s%s.%s" % (settings.FILES_IN, datafile.name_system, datafile.file_type)
output_file = settings.FILES_OUT + datafile.name_system + '.csv'
result = QRTEParser.parse(file=input_file, outfile=output_file, entrance='WEB')
# Send out email notifying researcher that Parsing was completed.
tpl = get_template('email/success.html')
send_mail(datafile.email, '[QRTEParser] Successfully parsed %s' % datafile.name_in, tpl.render({
'download_url': datafile.host_name + reverse(download) + '?uuid=' + str(datafile.download_uuid),
'log_url': datafile.host_name + reverse(download_log) + '?uuid=' + str(datafile.download_uuid),
'report_mail': settings.EMAIL_SENDER,
'filename': datafile.name_in
}
))
# Update database with success/fail report.
f.close()
except QRTEParserException as e:
# Handle caught QRTEParserException
# Send email that QRTEParser ran into known exception.
tpl = get_template('email/fail.html')
send_mail(datafile.email, '[QRTEParser] Failed to parse %s' % datafile.name_in, tpl.render({
'error_string': str(e),
'log_url': datafile.host_name + reverse(download_log) + '?uuid=' + str(datafile.download_uuid),
'report_mail': settings.EMAIL_SENDER,
'filename': datafile.name_in
}
))
# Update database with fail report
print(e)
f.close()
raise e
pass
except Exception as e:
# Handle unknown exception
# Send email that QRTEParser ran into unknown exception. Also send to [email protected]
tpl = get_template('email/fail.html')
send_mail(datafile.email, '[QRTEParser] Failed to parse %s' % datafile.name_in, tpl.render({
'error_string': str(e),
'log_url': datafile.host_name + reverse(download_log) + '?uuid=' + str(datafile.download_uuid),
'report_mail': settings.EMAIL_SENDER,
'filename': datafile.name_in
}
))
# Update database with fail report
print(e)
f.close()
raise e
pass
print("Parsing finished with output file: %s" % output_file)
f.close()
pass