def _create_source_and_load_test_json_to_entries(self):
source = Source.objects.create(
name='Test source',
url='http://example.com/foo/bar/testing',
date=datetime.date(2011, 9, 1))
data = json.loads(open(self.expected_data_json, 'r').read())
KenyaParser.create_entries_from_data_and_source(data, source)
return source
def _create_source_and_load_test_json_to_entries(self):
source = Source.objects.create(
name = 'Test source',
url = 'http://example.com/foo/bar/testing',
date = datetime.date( 2011, 9, 1 )
)
data = json.loads( open( self.expected_data_json, 'r' ).read() )
KenyaParser.create_entries_from_data_and_source( data, source )
return source
def handle_noargs(self, **options):
for source in Source.objects.all().requires_processing():
if int(options.get('verbosity')) >= 2:
print "Looking at %s" % source
source.last_processing_attempt = datetime.datetime.now()
source.save()
pdf = source.file()
html = KenyaParser.convert_pdf_to_html(pdf)
data = KenyaParser.convert_html_to_data(html)
KenyaParser.create_entries_from_data_and_source(data, source)
def handle_noargs(self, **options):
for source in Source.objects.all().requires_processing():
if int(options.get('verbosity')) >= 2:
print "Looking at %s" % source
source.last_processing_attempt = datetime.datetime.now()
source.save()
pdf = source.file()
html = KenyaParser.convert_pdf_to_html( pdf )
data = KenyaParser.convert_html_to_data( html )
KenyaParser.create_entries_from_data_and_source( data, source )
def handle_noargs(self, **options):
verbose = int(options.get('verbosity')) >= 2
for source in Source.objects.all().requires_processing():
if verbose:
message = "{0}: Looking at {1}"
print message.format(source.list_page, source)
source.last_processing_attempt = datetime.datetime.now()
source.save()
pdf = source.file()
try:
html = KenyaParser.convert_pdf_to_html( pdf )
data = KenyaParser.convert_html_to_data( html )
KenyaParser.create_entries_from_data_and_source( data, source )
except Exception as e:
print "There was an exception when parsing {0}".format(pdf)
raise
