Exemplo n.º 1
0
def main(file = None, log_level = 0):
    log.silent('Initing parser on file: %s', file)
    if file is None:
        log.warning('Please run using the following: python parser.py path/to/file.name')
        exit()
        
    if not os.path.isfile(file):
        log.warning('File does not exist: %s', (file,))
        exit()
        
    if os.path.getsize(file) == 0:
        log.warning('Can\'t parse an empty file - %s',(file,))
        exit()
    QRTEParserLogger.LISTENING_THRESHOLD = 2
    QRTEParser.parse(file, outfile=file.split('.')[0]+'_out.csv',exit_q_unique = False)
Exemplo n.º 2
0
def ParseUploadedFile(datafile_pk):
    """
    Parses the file indicated by the primary key. Emails an error reporting as well.
    :param datafile_pk:
    :return:
    """
    datafile = DataFile.objects.get(pk=datafile_pk)
    QRTEParserLogger.LISTENING_THRESHOLD = -3

    input_file = "%s%s.%s" % (settings.FILES_IN, datafile.name_system,
                              datafile.file_type)
    output_file = settings.FILES_OUT + datafile.name_out

    print("Running parser on file: %s" % input_file)

    # Set stdout & err to log file

    result = None
    try:
        # Try parsing the file
        input_file = "%s%s.%s" % (settings.FILES_IN, datafile.name_system,
                                  datafile.file_type)
        output_file = settings.FILES_OUT + datafile.name_out
        result = QRTEParser.parse(
            file="%s%s.%s" %
            (settings.FILES_IN, datafile.name_system, datafile.file_type),
            outfile=settings.FILES_OUT + datafile.name_out)
        # Send out email notifying researcher that Parsing was completed.
        # Include link to generated filename

        # Update database with success/fail report.
    except QRTEParserException as e:
        # Handle caught QRTEParserException

        # Send email that QRTEParser ran into known exception.

        # Update database with fail report
        print(e)
        raise e
        pass
    except Exception as e:
        # Handle unknown exception

        # Send email that QRTEParser ran into unknown exception. Also send to [email protected]

        # Update database with fail report
        print(e)
        raise e
        pass

    print("Parsing finished with output file: %s" % output_file)

    pass
Exemplo n.º 3
0
def ParseUploadedFile(datafile_pk):
    """
    Parses the file indicated by the primary key. Emails an error reporting as well.
    :param datafile_pk:
    :return:
    """
    datafile = DataFile.objects.get(pk=datafile_pk)
    QRTEParserLogger.LISTENING_THRESHOLD = -3

    input_file = "%s%s.%s" % (settings.FILES_IN,datafile.name_system,datafile.file_type)
    output_file = settings.FILES_OUT + datafile.name_out

    print("Running parser on file: %s" % input_file)

    # Set stdout & err to log file

    result = None
    try:
        # Try parsing the file
        input_file = "%s%s.%s" % (settings.FILES_IN,datafile.name_system,datafile.file_type)
        output_file = settings.FILES_OUT + datafile.name_out
        result = QRTEParser.parse(file="%s%s.%s" % (settings.FILES_IN,datafile.name_system,datafile.file_type), outfile = settings.FILES_OUT + datafile.name_out)
        # Send out email notifying researcher that Parsing was completed.
        # Include link to generated filename


        # Update database with success/fail report.
    except QRTEParserException as e:
        # Handle caught QRTEParserException

        # Send email that QRTEParser ran into known exception.

        # Update database with fail report
        print(e)
        raise e
        pass
    except Exception as e:
        # Handle unknown exception

        # Send email that QRTEParser ran into unknown exception. Also send to [email protected]

        # Update database with fail report
        print(e)
        raise e
        pass

    print("Parsing finished with output file: %s" % output_file)

    pass
Exemplo n.º 4
0
def ParseUploadedFile(datafile_pk):
    """
    Parses the file indicated by the primary key. Emails an error reporting as well.
    :param datafile_pk:
    :return:
    """
    datafile = DataFile.objects.get(pk=datafile_pk)
    QRTEParserLogger.LISTENING_THRESHOLD = 3

    input_file = "%s%s.%s" % (settings.FILES_IN, datafile.name_system, datafile.file_type)
    log_file = settings.FILES_LOG + datafile.name_system + '.log'

    print("Running parser on file: %s" % input_file)

    # Set stdout & err to log file
    f = open(log_file, 'w')
    QRTEParserLogger.OUT = f
    result = None
    try:
        # Try parsing the file
        input_file = "%s%s.%s" % (settings.FILES_IN, datafile.name_system, datafile.file_type)
        output_file = settings.FILES_OUT + datafile.name_system + '.csv'
        result = QRTEParser.parse(file=input_file, outfile=output_file, entrance='WEB')

        # Send out email notifying researcher that Parsing was completed.
        tpl = get_template('email/success.html')
        send_mail(datafile.email, '[QRTEParser] Successfully parsed %s' % datafile.name_in, tpl.render({
            'download_url': datafile.host_name + reverse(download) + '?uuid=' + str(datafile.download_uuid),
            'log_url': datafile.host_name + reverse(download_log) + '?uuid=' + str(datafile.download_uuid),
            'report_mail': settings.EMAIL_SENDER,
            'filename': datafile.name_in
        }
        ))

        # Update database with success/fail report.
        f.close()
    except QRTEParserException as e:
        # Handle caught QRTEParserException

        # Send email that QRTEParser ran into known exception.

        tpl = get_template('email/fail.html')
        send_mail(datafile.email, '[QRTEParser] Failed to parse %s' % datafile.name_in, tpl.render({
            'error_string': str(e),
            'log_url': datafile.host_name + reverse(download_log) + '?uuid=' + str(datafile.download_uuid),
            'report_mail': settings.EMAIL_SENDER,
            'filename': datafile.name_in
        }
        ))

        # Update database with fail report
        print(e)
        f.close()
        raise e
        pass
    except Exception as e:
        # Handle unknown exception

        # Send email that QRTEParser ran into unknown exception. Also send to [email protected]

        tpl = get_template('email/fail.html')
        send_mail(datafile.email, '[QRTEParser] Failed to parse %s' % datafile.name_in, tpl.render({
            'error_string': str(e),
            'log_url': datafile.host_name + reverse(download_log) + '?uuid=' + str(datafile.download_uuid),
            'report_mail': settings.EMAIL_SENDER,
            'filename': datafile.name_in
        }
        ))

        # Update database with fail report
        print(e)
        f.close()
        raise e
        pass

    print("Parsing finished with output file: %s" % output_file)
    f.close()

    pass