def handle(self, *args, **options): url = 'http://www.vegguide.org/site/recent.rss?entries_only=1' req = urllib.request.urlopen(url) atom_data_entries = req.read().decode('utf-8') e = etree.fromstring(atom_data_entries) links = e.findall('channel/item/link') for link in links: source_id = int(link.text.split('entry/')[1]) get_entry_by_id(source_id, force=True)
def entry_map(request, entry_id): entry = get_entry_by_id(entry_id) cords = get_entry_geo(entry) cords["c1lng"] = cords["lng"] + 0.01 cords["c2lng"] = cords["lng"] - 0.01 cords["c1lat"] = cords["lat"] + 0.01 cords["c2lat"] = cords["lat"] - 0.01 return render(request, "entry_map.html", {"cords": cords})
def entry_map(request, entry_id): entry = get_entry_by_id(entry_id) try: cords = get_entry_geo(entry) except IndexError: return render(request, "entry_map.html") #return HttpResponse('') cords['c1lng'] = cords['lng']+0.01 cords['c2lng'] = cords['lng']-0.01 cords['c1lat'] = cords['lat']+0.01 cords['c2lat'] = cords['lat']-0.01 return render(request, "entry_map.html", {'cords':cords, 'name':entry.get_elem('name')})
def handle(self, *args, **options): print(args) for entry_id in args: print("getting", entry_id) entry = get_entry_by_id(entry_id) print("got, back to cords", entry_id) print(get_entry_geo(entry)) print(entry.get_cord()) print(get_entry_geo(entry)) print(entry.get_address_str()) print(settings.GOOGLE_GEOCODE_API_KEY) self.stdout.write('Successfully got data')
def get_entry_by_vg_id(entry_id): """Get entry by VegGuide id. """ vg_region_type = ContentType.objects.get(app_label="transformer", model="region") vg_entry_type = ContentType.objects.get(app_label="transformer", model="entry") vg_entry = get_entry_by_id(entry_id) try: vs_entry= VeggieSailorEntry.objects.get(content_type=vg_entry_type, object_id=vg_entry.id) except VeggieSailorEntry.DoesNotExist: vs_entry = convert_entry(entry_id) return vs_entry
def test_get_entry_by_id_force(self): entry = get_entry_by_id(12188) entry = get_entry_by_id(12188, force=False) entry.set_obj() self.assertEqual(entry.obj['name'], 'Gopal')
def test_geo(self, VegGuideParser): vg_entry = get_entry_by_id(12188) vg_entry.obj_geo = json.loads(ENTRY_12118_JSON) cords = get_entry_geo(vg_entry) self.assertEqual(cords['lat'],41.3803325) self.assertEqual(cords['lng'],2.1772622)
def entry(request, entry_id): force = request.GET.get('force', False) entry_tmp = get_entry_by_id(entry_id, force) entry_tmp.set_obj() return HttpResponse(entry_tmp.results_source)
def convert_entry(entry_id, force=False): """Convert entry to the VeggieSailor object. """ vg_region_type = ContentType.objects.get(app_label="transformer", model="region") vg_entry_type = ContentType.objects.get(app_label="transformer", model="entry") vg_entry = get_entry_by_id(entry_id, force) vg_region = vg_entry.region #print ("Region", vg_region.source_id) try: vs_entry= VeggieSailorEntry.objects.get(content_type=vg_entry_type, object_id=vg_entry.id) except VeggieSailorEntry.DoesNotExist: vs_entry= VeggieSailorEntry(content_type=vg_entry_type, object_id=vg_entry.id) try: vs_region = VeggieSailorRegion.objects.get(content_type=vg_region_type, object_id=vg_region.source_id) except VeggieSailorRegion.DoesNotExist: vs_region = convert_region(vg_region.source_id) vs_entry.name = vg_entry.get_name() vs_entry.region = vs_region vs_entry.short_description = vg_entry.get_short_description() vs_entry.city = vg_entry.get_elem('city') vs_entry.address1 = vg_entry.get_elem('address1') vs_entry.address2 = vg_entry.get_elem('address2') vs_entry.vg_object_id = vg_entry.source_id vs_entry.rating = float(vg_entry.get_elem('weighted_rating', '0.0')) vs_entry.rating_count = int(vg_entry.get_elem('rating_count', '0')) vs_entry.level = int(vg_entry.get_elem('veg_level',0)) price_range = vg_entry.get_elem('price_range') if price_range == '$ - inexpensive': price = 1 elif price_range == '$$ - average': price = 2 elif price_range == '$$$ - expensive': price = 3 else: price = 0 vs_entry.price = price #print (vg_entry.source_id, price, price_range) # List of images to download #image_urls = [] vs_entry.description = vg_entry.get_long_description() vs_entry.zipcode = vg_entry.get_postal_code() #print ("smokesy",vg_entry.get_elem('allows_smoking')) vs_entry.allows_smoking = vg_entry.get_elem('allows_smoking') vs_entry.allows_reservations = vg_entry.get_elem('accepts_reservations') vs_entry.save() categories = vg_entry.get_elem('categories', []) vs_categories = [] for category in categories: vs_category, created = VeggieSailorCategory.objects.get_or_create(name=category) vs_categories.append(vs_category) if categories: vs_entry.categories = vs_categories vs_entry.save() cuisines = vg_entry.get_elem('cuisines',[]) vs_cuisines = [] for cuisine in cuisines: vs_cuisine, created = VeggieSailorCuisine.objects.get_or_create(name=cuisine) vs_cuisines.append(vs_cuisine) if cuisines: vs_entry.cuisines = vs_cuisines vs_entry.save() cuisines = vg_entry.get_elem('cuisines',[]) vs_cuisines = [] for cuisine in cuisines: vs_cuisine, created = VeggieSailorCuisine.objects.get_or_create(name=cuisine) vs_cuisines.append(vs_cuisine) if cuisines: vs_entry.cuisines = vs_cuisines vs_entry.save() tags = vg_entry.get_elem('tags',[]) vs_tags = [] for tag in tags: vs_tag, created = VeggieSailorTag.objects.get_or_create(name=tag) vs_tags.append(vs_tag) if tags: vs_entry.tags = vs_tags vs_entry.save() tags = vg_entry.get_elem('tags',[]) vs_tags = [] for tag in tags: vs_tag, created = VeggieSailorTag.objects.get_or_create(name=tag) vs_tags.append(vs_tag) if tags: vs_entry.tags = vs_tags vs_entry.save() payments = vg_entry.get_elem('payment_options',[]) #print ("payments",payments) vs_payments = [] for payment in payments: vs_payment, created = VeggieSailorPayment.objects.get_or_create(name=payment) vs_payments.append(vs_payment) if payments: vs_entry.payments = vs_payments vs_entry.save() payments = vg_entry.get_elem('payments',[]) vs_payments = [] for payment in payments: vs_payment, created = VeggieSailorPayment.objects.get_or_create(name=payment) vs_payments.append(vs_payment) if payments: vs_entry.payments = vs_payments vs_entry.save() #images = [] if not VeggieSailorImage.objects.filter(entry=vs_entry) or force is True: if force is True: VeggieSailorImage.objects.filter(entry=vs_entry).delete() images_list = [x for x in vg_entry.get_elem('images',[]) ] #print ("images", images_list) # http://stackoverflow.com/questions/16174022/download-a-remote-image-and-save-it-to-a-django-model for image_elem in images_list: # Steam the image from the url try: title = image_elem['caption'] for image_file in image_elem['files']: #print (image_file) image_url = image_file['uri'] print ("MAKING IMAGE REQUEST %s", image_url) new_name = md5(image_url.encode('utf-8')).hexdigest() request = requests.get(image_url, stream=True) width = image_file['width'] height = image_file['height'] # Was the request OK? if request.status_code != requests.codes.ok: # Nope, error handling, skip file etc etc etc continue # Get the filename from the url, used for saving later url_file_name_ext = image_url.split('/')[-1].split(".")[-1] file_name = '%s.%s' % (new_name, url_file_name_ext) # Create a temporary file lf = tempfile.NamedTemporaryFile() # Read the streamed image in sections for block in request.iter_content(1024 * 8): # If no more file then stop if not block: break # Write image block to temporary file lf.write(block) # Create the model you want to save the image to try: image = VeggieSailorImage() image.title = title image.entry = vs_entry image.width = width image.height = height # Save the temporary image to the model# # This saves the model so be sure that is it valid image.photo.save(file_name, files.File(lf)) image.save() except DataError: print ("Error with the picture", vg_entry.source_id) except KeyError: print ("KeyError", vg_entry.source_id) #print(vg_entry.source_id, VeggieSailorEntry.objects.all().count()) vs_entry.slug = '%s-%s' % (slugify(unidecode(vs_entry.name))[0:44] , vs_entry.id) vs_entry.save() return vs_entry