def handle(self, *args, **options):
new_filings = new_filing.objects.filter(filing_is_downloaded=False, header_is_processed=False).order_by('filing_number')
for filing in new_filings:
if filing.filing_number < 1015680:
continue
print "need to handle %s" % (filing.filing_number)
location = FEC_DOWNLOAD % (filing.filing_number)
local_location = FILECACHE_DIRECTORY + "/" + str(filing.filing_number) + ".fec"
cmd = "curl \"%s\" -o %s" % (location, local_location)
print "Running command %s" % (cmd)
# run it from a curl shell
proc = subprocess.Popen(cmd,shell=True)
# if it's there when we're done, mark it as downloaded
print "Now sleeping for 1 second"
sleep(1)
# Putting the file check after the 1 second sleep gives better results; sometimes it appears absent if it's not before
if path.isfile(local_location):
print "File %s is there" % (filing.filing_number)
filing.filing_is_downloaded=True
filing.save()
else:
print "MISSING: %s" % (filing.filing_number)
def handle(self, *args, **options):
downloaded_filings = new_filing.objects.filter(filing_is_downloaded=True, header_is_processed=False).order_by('filing_number')
for filing in downloaded_filings:
print "Entering filing %s, entry_time %s" % (filing.filing_number, filing.process_time)
result_header = None
try:
result_header = process_new_filing(filing, fp=fp, filing_time=filing.process_time, filing_time_is_exact=True)
## It seems like the FEC's response is now to give a page not found response instead of a 500 error or something. The result is that the except no longer seems to apply.
except IOError:
# if the file's missing, keep running.
print "MISSING FILING: %s" % (filing.filing_number)
continue
if result_header:
filing.header_is_processed=True
filing.save()
else:
print "Header not created right... %s" % (filing)
def handle(self, *args, **options):
logger.info('ENTER_HEADERS_FROM_NEW_FILINGS - starting regular run')
downloaded_filings = Filing.objects.filter(filing_is_downloaded="1", header_is_processed="0").order_by('filing_number')
for filing in downloaded_filings:
result_header = None
logger.info('ENTER_HEADERS_FROM_NEW_FILINGS - processing %s' % (filing.filing_number))
try:
result_header = process_new_filing(filing, fp=fp, filing_time=filing.process_time, filing_time_is_exact=True)
## It seems like the FEC's response is now to give a page not found response instead of a 500 error or something. The result is that the except no longer seems to apply.
except IOError:
# if the file's missing, keep running.
logger.error("Filing marked as downloaded is unavailable: %s" % (filing.filing_number))
# Mark that an error has occurred.
filing.header_is_processed = 'E'
filing.save()
continue
if result_header:
filing.header_is_processed="1"
filing.save()
else:
logger.error("Header not created right: %s" % (filing.filing_number))
filing.header_is_processed = 'E'
filing.save()
def handle(self, *args, **options):
logger.info('ENTER_HEADERS_FROM_NEW_FILINGS - starting regular run')
downloaded_filings = Filing.objects.filter(
filing_is_downloaded="1",
header_is_processed="0").order_by('filing_number')
for filing in downloaded_filings:
result_header = None
logger.info('ENTER_HEADERS_FROM_NEW_FILINGS - processing %s' %
(filing.filing_number))
try:
result_header = process_new_filing(
filing,
fp=fp,
filing_time=filing.process_time,
filing_time_is_exact=True)
## It seems like the FEC's response is now to give a page not found response instead of a 500 error or something. The result is that the except no longer seems to apply.
except IOError:
# if the file's missing, keep running.
logger.error("Filing marked as downloaded is unavailable: %s" %
(filing.filing_number))
# Mark that an error has occurred.
filing.header_is_processed = 'E'
filing.save()
continue
if result_header:
filing.header_is_processed = "1"
filing.save()
else:
logger.error("Header not created right: %s" %
(filing.filing_number))
filing.header_is_processed = 'E'
filing.save()