def main(url):
MorphDatabase.init()
html = requests.get(url)
tree = lxml.html.fromstring(html.content)
print(tree)
sys.exit(0)
count_new = total = 0
for application_url in get_application_links(url):
if not application_url:
# Skipped entry...
total += 1
continue
# html = scraperwiki.scrape(url)
# XPath
#This will create a list of buyers:
buyers = tree.xpath('//div[@title="buyer-name"]/text()')
#This will create a list of prices
prices = tree.xpath('//span[@class="item-price"]/text()')
# Find something on the page using css selectors
# root.cssselect("div[align='left']")
#
data = extract_application_details(application_url)
application, created = DevelopmentApplication.get_or_create(**data)
total += 1
if not created:
print("* Skipping {0.council_reference}".format(application))
else:
print("Saved {0.council_reference}".format(application))
count_new += 1
print("Added {0} records out of {1} processed.".format(count_new, total))
def main(start_url):
MorphDatabase.init()
count_new = total = 0
for application_url in get_application_links(start_url):
if not application_url:
# Skipped entry...
total += 1
continue
da_info = extract_application_details(application_url)
application, created = DevelopmentApplication.get_or_create(**da_info)
total += 1
if not created:
print("* Skipping {0.council_reference}".format(application))
else:
print("Saved {0.council_reference}".format(application))
count_new += 1
print("Added {0} records out of {1} processed.".format(count_new, total))
