Ejemplo n.º 1
0
    def handle(self, *args, **options):
        new_filings = new_filing.objects.filter(filing_is_downloaded=False, header_is_processed=False).order_by('filing_number')
        for filing in new_filings:
            if filing.filing_number < 1015680:
                continue
            print "need to handle %s" % (filing.filing_number)
            location = FEC_DOWNLOAD % (filing.filing_number)
            local_location = FILECACHE_DIRECTORY + "/" + str(filing.filing_number) + ".fec"
            cmd = "curl \"%s\" -o %s" % (location, local_location)
            print "Running command %s" % (cmd)
            # run it from a curl shell
            proc = subprocess.Popen(cmd,shell=True)
            # if it's there when we're done, mark it as downloaded


            print "Now sleeping for 1 second"
            sleep(1)
            
            # Putting the file check after the 1 second sleep gives better results; sometimes it appears absent if it's not before
            if path.isfile(local_location):
                print "File %s is there" % (filing.filing_number)
                filing.filing_is_downloaded=True
                filing.save()
            else:
                print "MISSING: %s" % (filing.filing_number)
Ejemplo n.º 2
0
 def handle(self, *args, **options):
     downloaded_filings = new_filing.objects.filter(filing_is_downloaded=True, header_is_processed=False).order_by('filing_number')
     for filing in downloaded_filings:
         print "Entering filing %s, entry_time %s" % (filing.filing_number, filing.process_time)
         result_header = None
         try: 
             result_header = process_new_filing(filing, fp=fp, filing_time=filing.process_time, filing_time_is_exact=True)
         ## It seems like the FEC's response is now to give a page not found response instead of a 500 error or something. The result is that the except no longer seems to apply. 
         except IOError:
             # if the file's missing, keep running. 
             print "MISSING FILING: %s" % (filing.filing_number)  
             continue
         if result_header:
             filing.header_is_processed=True
             filing.save()
         else:
             print "Header not created right... %s" % (filing)
 def handle(self, *args, **options):
     logger.info('ENTER_HEADERS_FROM_NEW_FILINGS - starting regular run')
     
     downloaded_filings = Filing.objects.filter(filing_is_downloaded="1", header_is_processed="0").order_by('filing_number')
     for filing in downloaded_filings:
         result_header = None
         logger.info('ENTER_HEADERS_FROM_NEW_FILINGS - processing %s' % (filing.filing_number))
         
         try: 
             result_header = process_new_filing(filing, fp=fp, filing_time=filing.process_time, filing_time_is_exact=True)
         ## It seems like the FEC's response is now to give a page not found response instead of a 500 error or something. The result is that the except no longer seems to apply. 
         except IOError:
             # if the file's missing, keep running. 
             logger.error("Filing marked as downloaded is unavailable: %s" % (filing.filing_number))
             # Mark that an error has occurred.
             filing.header_is_processed = 'E'
             filing.save()
             
             continue
         if result_header:
             filing.header_is_processed="1"
             filing.save()
         else:
             
             logger.error("Header not created right: %s" % (filing.filing_number))
             filing.header_is_processed = 'E'
             filing.save()
             
             
Ejemplo n.º 4
0
    def handle(self, *args, **options):
        logger.info('ENTER_HEADERS_FROM_NEW_FILINGS - starting regular run')

        downloaded_filings = Filing.objects.filter(
            filing_is_downloaded="1",
            header_is_processed="0").order_by('filing_number')
        for filing in downloaded_filings:
            result_header = None
            logger.info('ENTER_HEADERS_FROM_NEW_FILINGS - processing %s' %
                        (filing.filing_number))

            try:
                result_header = process_new_filing(
                    filing,
                    fp=fp,
                    filing_time=filing.process_time,
                    filing_time_is_exact=True)
            ## It seems like the FEC's response is now to give a page not found response instead of a 500 error or something. The result is that the except no longer seems to apply.
            except IOError:
                # if the file's missing, keep running.
                logger.error("Filing marked as downloaded is unavailable: %s" %
                             (filing.filing_number))
                # Mark that an error has occurred.
                filing.header_is_processed = 'E'
                filing.save()

                continue
            if result_header:
                filing.header_is_processed = "1"
                filing.save()
            else:

                logger.error("Header not created right: %s" %
                             (filing.filing_number))
                filing.header_is_processed = 'E'
                filing.save()