def handle(self, *args, **options): regions_ids = (2218,844,2218,336,52, 583, 328,298,52, 301, 300,74,72,70, 106, 16,1,2, 3, 356, 355, 274, 273, ) for region_id in regions_ids: get_region_by_id(region_id) convert_region(region_id) data = serializers.serialize("json", Region.objects.filter(source_id__in=regions_ids)) file_json = open('/tmp/region_52.json', 'w') file_json.write(data) file_json.close() data = serializers.serialize("json", VeggieSailorRegion.objects.filter(source_region__source_id__in=regions_ids)) file_json = open('/tmp/vs_region_source_52.json', 'w') file_json.write(data) file_json.close()
def fetch_region_recursion(region_id, number): """Fetch regions in recursive way. Parameters ---------- region_id : int start region number : int number of recursive calls """ number = int(number) if number<1: return (region_id, 0) region = get_region_by_id(region_id) region.set_obj() if region_id==0: regions = region.obj['regions']['primary'] else: regions = region.obj.get('children', []) for region in regions: source_id = regex_region.findall(region['uri'])[0] is_cached = is_cached_region(source_id) if not is_cached: number -= int(1) sleep(randint(3,7)) # Be nice for VegGuide.org (region_id, number) = fetch_region_recursion(source_id, int(number)) if number==0: return (region_id, 0) return (region_id, number)
def convert_region_down(region_id, global_list_tmp=()): """Convert region. Parameters ---------- region_id """ global_list = list(global_list_tmp) #vg_region_type = ContentType.objects.get(app_label="transformer", model="region") vg_entry_type = ContentType.objects.get(app_label="transformer", model="entry") if region_id in global_list: #print ("Leaving", region_id) return None vs_region = convert_region(region_id) global_list.append(region_id) region = Region.objects.get(source_id=region_id) try: region = Region.objects.get(source_id=region_id) except Region.DoesNotExist: region = get_region_by_id(region_id) children = region.get_children() for child in children: convert_region_down(int(child), global_list) return (region, vs_region)
def test_convert_region_by_id(self): """Test convert region by id. """ region = get_region_by_id(52) regions_ids = (52, 583,66) for region_id in regions_ids: get_region_by_id(region_id) convert_region(region_id) vs_region = convert_region(52) region.set_obj() self.assertEqual(region.obj['name'], 'Europe') self.assertEqual(region.obj['name'], vs_region.name)
def test_get_region_by_id_force(self): region = get_region_by_id(66, force=False) region.set_obj() self.assertEqual(region.obj['name'], 'Spain') #entry = get_entry_by_id(12188) #entry.set_obj() #self.assertEqual(entry.obj['name'], 'Gopal') #entry = get_entry_by_id(12188) #entry = get_entry_by_id(12188, force=True) #entry.set_obj() #self.assertEqual(entry.obj['name'], 'Gopal') #region = get_region_by_id(66) #region.set_obj() #self.assertEqual(region.obj['name'], 'Spain') #c = Client() #r = c.get('/transformer/entry/12188/reviews') #region = get_region_by_id(66, force=False) #region.set_obj() #self.assertEqual(region.obj['name'], 'Spain') #c = Client() #r = c.get('/transformer/entry/12188') #self.assertEqual(r.content.decode('utf-8').find('Spain')>-1,True) #c = Client() ##r = c.get('/transformer/entry/12188?force=True') #self.assertEqual(r.content.decode('utf-8').find('Spain')>-1,True) #c = Client() #r = c.get('/transformer/region/66') #data = serializers.serialize("json", Region.objects.all()) #f = open('vegbasketapp/transformer/fixtures/region_transformer.json', 'w') #f.write(data) #f.close() #data = serializers.serialize("json", Entry.objects.all()) #f = open('vegbasketapp/transformer/fixtures/entry_transformer.json', 'w') #f.write(data) #f.close() #data = serializers.serialize("json", Reviews.objects.all()) #f = open('vegbasketapp/transformer/fixtures/reviews_transformer.json', 'w') #f.write(data) #f.close()
def handle(self, *args, **options): print(args) for region_id in args: try: print("getting region", region_id) region = get_region_by_id(region_id) region.set_obj() entries = get_entries(region.obj['uri']) for result in entries.results: print(region_id, result['uri']) get_entry(result['uri']) sleep_random(1, 5) sleep_random(4, 12) except KeyError: print("key error") self.stdout.write('Successfully got data')
def convert_region(region_id): """Convert single region and it's parent. Notes ----- Works in the recursive way. """ vg_region_type = ContentType.objects.get(app_label="transformer", model="region") #vg_entry_type = ContentType.objects.get(app_label="transformer", model="entry") try: region = Region.objects.get(source_id=region_id) except Region.DoesNotExist: region = get_region_by_id(region_id) region.set_obj() if VeggieSailorRegion.objects.filter(source_region=region).count()>0: vs_region = VeggieSailorRegion.objects.get(source_region=region) vs_region.content_type = vg_region_type vs_region.object_id = region.source_id vs_region.save() return vs_region vs_region = VeggieSailorRegion() vs_region.name = region.obj['name'] #try: #vs_region.name = region.obj['name'] #except KeyError: #from ipdb import set_trace; set_trace() vs_region.content_type = vg_region_type vs_region.object_id = region.source_id vs_region.source_region = region vs_region.save() if 'parent' in region.obj and 'uri' in region.obj['parent']: parent_id = get_region_id(region.obj['parent']['uri']) #print ("has parent", region_id, parent_id) vs_region_parent = convert_region(parent_id) vs_region.parent = vs_region_parent vs_region.save() return vs_region
def test_get_region_by_id(self): region = get_region_by_id(66) region.set_obj() self.assertEqual(region.obj['name'], 'Spain')
def test_request(self, VegGuideParser): VegGuideParser.return_value.result = json.loads(REGION_23) vg_region = (get_region_by_id(23)) vg_region.set_obj() self.assertEqual(vg_region.obj['name'], 'Ontario')
def region_root(request, region_id): force = request.GET.get('force', False) region_tmp = get_region_by_id(0, force) return HttpResponse(region_tmp.results_source)
def test_get_region_by_id_force(self): region = get_region_by_id(66, force=True) region.set_obj() self.assertEqual(region.obj['name'], 'Spain')
def region(request, region_id): force = request.GET.get("force", False) region_tmp = get_region_by_id(region_id, force) return HttpResponse(region_tmp.results_source)