Python preprocess Examples

Example #1

def process_pdf(parser, pdfurl, errors):
    postlistelib.exit_if_no_cpu_left(0, out_of_cpu, errors)
    try:
        pdfcontent = scraperwiki.scrape(pdfurl)
        parser.preprocess(pdfurl, pdfcontent)
        pdfcontent = None
    except ValueError, e:
        errors.append(e)

Example #2

File: postliste-universitetet-i-agder.py Project: flyeven/scraperwiki-scraper-vault

def process_pdf(parser, pdfurl, errors):
    postlistelib.exit_if_no_cpu_left(0, out_of_cpu, errors)
    try:
        pdfcontent = scraperwiki.scrape(pdfurl)
        parser.preprocess(pdfurl, pdfcontent)
        pdfcontent = None
    except ValueError, e:
        errors.append(e)

Example #3

File: postliste-ntnu.py Project: yuandra/scraperwiki-scraper-vault

def process_pdf(parser, pdfurl, errors):
    postlistelib.exit_if_no_cpu_left(0, out_of_cpu, errors)
    try:
        pdfcontent = scraperwiki.scrape(pdfurl)
        parser.preprocess(pdfurl, pdfcontent)
        pdfcontent = None
    except ValueError, e:  # Some PDFs can not be parsed!  This should be investigated
        print "PDF format problem"
        errors.append(e)

Example #4

File: postliste-ntnu.py Project: flyeven/scraperwiki-scraper-vault

def process_pdf(parser, pdfurl, errors):
    postlistelib.exit_if_no_cpu_left(0, out_of_cpu, errors)
    try:
        pdfcontent = scraperwiki.scrape(pdfurl)
        parser.preprocess(pdfurl, pdfcontent)
        pdfcontent = None
    except ValueError, e: # Some PDFs can not be parsed!  This should be investigated
        print "PDF format problem"
        errors.append(e)

Example #5

File: postliste-kristiansund.py Project: flyeven/scraperwiki-scraper-vault

def process_pdf(parser, pdfurl, errors):
    errors = []
    postlistelib.exit_if_no_cpu_left(0, callback=no_cpu_left, arg = errors)
    try:
        pdfcontent = lazycache.lazycache(pdfurl)
        parser.preprocess(pdfurl, pdfcontent)
#    except ValueError, e:
#        errors.append(e)
    except IndexError, e:
        errors.append(e)

Example #6

File: postliste-hadsel.py Project: yuandra/scraperwiki-scraper-vault

def process_pdf(parser, pdfurl, errors):
    errors = []
    postlistelib.exit_if_no_cpu_left(0, callback=no_cpu_left, arg = errors)
    try:
        pdfcontent = lazycache.lazycache(pdfurl)
        parser.preprocess(pdfurl, pdfcontent)
#    except ValueError, e:
#        errors.append(e)
    except IndexError, e:
        errors.append(e)

Example #7

def process_pdf(parser, pdfurl, errors):
    if parser.is_already_scraped(pdfurl):
        return
    postlistelib.exit_if_no_cpu_left(0, out_of_cpu, errors)
    try:
        pdfcontent = scraperwiki.scrape(pdfurl)
        parser.preprocess(pdfurl, pdfcontent)
        pdfcontent = None
    except ValueError, e:
        print e
        errors.append(e)

Example #8

File: postliste-nrk.py Project: flyeven/scraperwiki-scraper-vault

def process_pdf(parser, pdfurl, errors):
    if parser.is_already_scraped(pdfurl):
        return
    postlistelib.exit_if_no_cpu_left(0, out_of_cpu, errors)
    try:
        pdfcontent = scraperwiki.scrape(pdfurl)
        parser.preprocess(pdfurl, pdfcontent)
        pdfcontent = None
    except ValueError, e:
        print e
        errors.append(e)

Example #9

def fetch_and_preprocess(parser, pdfurl):
    pdfcontent = postlistelib.fetch_url_harder(pdfurl)
    parser.preprocess(pdfurl, pdfcontent)
    pdfcontent = None

Example #10

File: postliste-stortinget.py Project: flyeven/scraperwiki-scraper-vault

def fetch_and_preprocess(parser, pdfurl):
    pdfcontent = postlistelib.fetch_url_harder(pdfurl)
    parser.preprocess(pdfurl, pdfcontent)
    pdfcontent = None