def handle(self, *args, **options):
url = 'http://www.vegguide.org/site/recent.rss?entries_only=1'
req = urllib.request.urlopen(url)
atom_data_entries = req.read().decode('utf-8')
e = etree.fromstring(atom_data_entries)
links = e.findall('channel/item/link')
for link in links:
source_id = int(link.text.split('entry/')[1])
get_entry_by_id(source_id, force=True)
def entry_map(request, entry_id):
entry = get_entry_by_id(entry_id)
cords = get_entry_geo(entry)
cords["c1lng"] = cords["lng"] + 0.01
cords["c2lng"] = cords["lng"] - 0.01
cords["c1lat"] = cords["lat"] + 0.01
cords["c2lat"] = cords["lat"] - 0.01
return render(request, "entry_map.html", {"cords": cords})
def entry_map(request, entry_id):
entry = get_entry_by_id(entry_id)
try:
cords = get_entry_geo(entry)
except IndexError:
return render(request, "entry_map.html")
#return HttpResponse('')
cords['c1lng'] = cords['lng']+0.01
cords['c2lng'] = cords['lng']-0.01
cords['c1lat'] = cords['lat']+0.01
cords['c2lat'] = cords['lat']-0.01
return render(request, "entry_map.html", {'cords':cords, 'name':entry.get_elem('name')})
def handle(self, *args, **options):
print(args)
for entry_id in args:
print("getting", entry_id)
entry = get_entry_by_id(entry_id)
print("got, back to cords", entry_id)
print(get_entry_geo(entry))
print(entry.get_cord())
print(get_entry_geo(entry))
print(entry.get_address_str())
print(settings.GOOGLE_GEOCODE_API_KEY)
self.stdout.write('Successfully got data')
def get_entry_by_vg_id(entry_id):
"""Get entry by VegGuide id.
"""
vg_region_type = ContentType.objects.get(app_label="transformer", model="region")
vg_entry_type = ContentType.objects.get(app_label="transformer", model="entry")
vg_entry = get_entry_by_id(entry_id)
try:
vs_entry= VeggieSailorEntry.objects.get(content_type=vg_entry_type, object_id=vg_entry.id)
except VeggieSailorEntry.DoesNotExist:
vs_entry = convert_entry(entry_id)
return vs_entry
def test_get_entry_by_id_force(self):
entry = get_entry_by_id(12188)
entry = get_entry_by_id(12188, force=False)
entry.set_obj()
self.assertEqual(entry.obj['name'], 'Gopal')
def test_geo(self, VegGuideParser):
vg_entry = get_entry_by_id(12188)
vg_entry.obj_geo = json.loads(ENTRY_12118_JSON)
cords = get_entry_geo(vg_entry)
self.assertEqual(cords['lat'],41.3803325)
self.assertEqual(cords['lng'],2.1772622)
def entry(request, entry_id):
force = request.GET.get('force', False)
entry_tmp = get_entry_by_id(entry_id, force)
entry_tmp.set_obj()
return HttpResponse(entry_tmp.results_source)
def convert_entry(entry_id, force=False):
"""Convert entry to the VeggieSailor object.
"""
vg_region_type = ContentType.objects.get(app_label="transformer", model="region")
vg_entry_type = ContentType.objects.get(app_label="transformer", model="entry")
vg_entry = get_entry_by_id(entry_id, force)
vg_region = vg_entry.region
#print ("Region", vg_region.source_id)
try:
vs_entry= VeggieSailorEntry.objects.get(content_type=vg_entry_type,
object_id=vg_entry.id)
except VeggieSailorEntry.DoesNotExist:
vs_entry= VeggieSailorEntry(content_type=vg_entry_type, object_id=vg_entry.id)
try:
vs_region = VeggieSailorRegion.objects.get(content_type=vg_region_type,
object_id=vg_region.source_id)
except VeggieSailorRegion.DoesNotExist:
vs_region = convert_region(vg_region.source_id)
vs_entry.name = vg_entry.get_name()
vs_entry.region = vs_region
vs_entry.short_description = vg_entry.get_short_description()
vs_entry.city = vg_entry.get_elem('city')
vs_entry.address1 = vg_entry.get_elem('address1')
vs_entry.address2 = vg_entry.get_elem('address2')
vs_entry.vg_object_id = vg_entry.source_id
vs_entry.rating = float(vg_entry.get_elem('weighted_rating', '0.0'))
vs_entry.rating_count = int(vg_entry.get_elem('rating_count', '0'))
vs_entry.level = int(vg_entry.get_elem('veg_level',0))
price_range = vg_entry.get_elem('price_range')
if price_range == '$ - inexpensive':
price = 1
elif price_range == '$$ - average':
price = 2
elif price_range == '$$$ - expensive':
price = 3
else:
price = 0
vs_entry.price = price
#print (vg_entry.source_id, price, price_range)
# List of images to download
#image_urls = []
vs_entry.description = vg_entry.get_long_description()
vs_entry.zipcode = vg_entry.get_postal_code()
#print ("smokesy",vg_entry.get_elem('allows_smoking'))
vs_entry.allows_smoking = vg_entry.get_elem('allows_smoking')
vs_entry.allows_reservations = vg_entry.get_elem('accepts_reservations')
vs_entry.save()
categories = vg_entry.get_elem('categories', [])
vs_categories = []
for category in categories:
vs_category, created = VeggieSailorCategory.objects.get_or_create(name=category)
vs_categories.append(vs_category)
if categories:
vs_entry.categories = vs_categories
vs_entry.save()
cuisines = vg_entry.get_elem('cuisines',[])
vs_cuisines = []
for cuisine in cuisines:
vs_cuisine, created = VeggieSailorCuisine.objects.get_or_create(name=cuisine)
vs_cuisines.append(vs_cuisine)
if cuisines:
vs_entry.cuisines = vs_cuisines
vs_entry.save()
cuisines = vg_entry.get_elem('cuisines',[])
vs_cuisines = []
for cuisine in cuisines:
vs_cuisine, created = VeggieSailorCuisine.objects.get_or_create(name=cuisine)
vs_cuisines.append(vs_cuisine)
if cuisines:
vs_entry.cuisines = vs_cuisines
vs_entry.save()
tags = vg_entry.get_elem('tags',[])
vs_tags = []
for tag in tags:
vs_tag, created = VeggieSailorTag.objects.get_or_create(name=tag)
vs_tags.append(vs_tag)
if tags:
vs_entry.tags = vs_tags
vs_entry.save()
tags = vg_entry.get_elem('tags',[])
vs_tags = []
for tag in tags:
vs_tag, created = VeggieSailorTag.objects.get_or_create(name=tag)
vs_tags.append(vs_tag)
if tags:
vs_entry.tags = vs_tags
vs_entry.save()
payments = vg_entry.get_elem('payment_options',[])
#print ("payments",payments)
vs_payments = []
for payment in payments:
vs_payment, created = VeggieSailorPayment.objects.get_or_create(name=payment)
vs_payments.append(vs_payment)
if payments:
vs_entry.payments = vs_payments
vs_entry.save()
payments = vg_entry.get_elem('payments',[])
vs_payments = []
for payment in payments:
vs_payment, created = VeggieSailorPayment.objects.get_or_create(name=payment)
vs_payments.append(vs_payment)
if payments:
vs_entry.payments = vs_payments
vs_entry.save()
#images = []
if not VeggieSailorImage.objects.filter(entry=vs_entry) or force is True:
if force is True:
VeggieSailorImage.objects.filter(entry=vs_entry).delete()
images_list = [x for x in vg_entry.get_elem('images',[]) ]
#print ("images", images_list)
# http://stackoverflow.com/questions/16174022/download-a-remote-image-and-save-it-to-a-django-model
for image_elem in images_list:
# Steam the image from the url
try:
title = image_elem['caption']
for image_file in image_elem['files']:
#print (image_file)
image_url = image_file['uri']
print ("MAKING IMAGE REQUEST %s", image_url)
new_name = md5(image_url.encode('utf-8')).hexdigest()
request = requests.get(image_url, stream=True)
width = image_file['width']
height = image_file['height']
# Was the request OK?
if request.status_code != requests.codes.ok:
# Nope, error handling, skip file etc etc etc
continue
# Get the filename from the url, used for saving later
url_file_name_ext = image_url.split('/')[-1].split(".")[-1]
file_name = '%s.%s' % (new_name, url_file_name_ext)
# Create a temporary file
lf = tempfile.NamedTemporaryFile()
# Read the streamed image in sections
for block in request.iter_content(1024 * 8):
# If no more file then stop
if not block:
break
# Write image block to temporary file
lf.write(block)
# Create the model you want to save the image to
try:
image = VeggieSailorImage()
image.title = title
image.entry = vs_entry
image.width = width
image.height = height
# Save the temporary image to the model#
# This saves the model so be sure that is it valid
image.photo.save(file_name, files.File(lf))
image.save()
except DataError:
print ("Error with the picture", vg_entry.source_id)
except KeyError:
print ("KeyError", vg_entry.source_id)
#print(vg_entry.source_id, VeggieSailorEntry.objects.all().count())
vs_entry.slug = '%s-%s' % (slugify(unidecode(vs_entry.name))[0:44] , vs_entry.id)
vs_entry.save()
return vs_entry
