def handle(self, *args, **options):
regions_ids = (2218,844,2218,336,52, 583, 328,298,52, 301, 300,74,72,70, 106, 16,1,2, 3, 356, 355, 274, 273, )
for region_id in regions_ids:
get_region_by_id(region_id)
convert_region(region_id)
data = serializers.serialize("json", Region.objects.filter(source_id__in=regions_ids))
file_json = open('/tmp/region_52.json', 'w')
file_json.write(data)
file_json.close()
data = serializers.serialize("json", VeggieSailorRegion.objects.filter(source_region__source_id__in=regions_ids))
file_json = open('/tmp/vs_region_source_52.json', 'w')
file_json.write(data)
file_json.close()
def fetch_region_recursion(region_id, number):
"""Fetch regions in recursive way.
Parameters
----------
region_id : int start region
number : int number of recursive calls
"""
number = int(number)
if number<1:
return (region_id, 0)
region = get_region_by_id(region_id)
region.set_obj()
if region_id==0:
regions = region.obj['regions']['primary']
else:
regions = region.obj.get('children', [])
for region in regions:
source_id = regex_region.findall(region['uri'])[0]
is_cached = is_cached_region(source_id)
if not is_cached:
number -= int(1)
sleep(randint(3,7)) # Be nice for VegGuide.org
(region_id, number) = fetch_region_recursion(source_id, int(number))
if number==0:
return (region_id, 0)
return (region_id, number)
def convert_region_down(region_id, global_list_tmp=()):
"""Convert region.
Parameters
----------
region_id
"""
global_list = list(global_list_tmp)
#vg_region_type = ContentType.objects.get(app_label="transformer", model="region")
vg_entry_type = ContentType.objects.get(app_label="transformer", model="entry")
if region_id in global_list:
#print ("Leaving", region_id)
return None
vs_region = convert_region(region_id)
global_list.append(region_id)
region = Region.objects.get(source_id=region_id)
try:
region = Region.objects.get(source_id=region_id)
except Region.DoesNotExist:
region = get_region_by_id(region_id)
children = region.get_children()
for child in children:
convert_region_down(int(child), global_list)
return (region, vs_region)
def test_convert_region_by_id(self):
"""Test convert region by id.
"""
region = get_region_by_id(52)
regions_ids = (52, 583,66)
for region_id in regions_ids:
get_region_by_id(region_id)
convert_region(region_id)
vs_region = convert_region(52)
region.set_obj()
self.assertEqual(region.obj['name'], 'Europe')
self.assertEqual(region.obj['name'], vs_region.name)
def test_get_region_by_id_force(self):
region = get_region_by_id(66, force=False)
region.set_obj()
self.assertEqual(region.obj['name'], 'Spain')
#entry = get_entry_by_id(12188)
#entry.set_obj()
#self.assertEqual(entry.obj['name'], 'Gopal')
#entry = get_entry_by_id(12188)
#entry = get_entry_by_id(12188, force=True)
#entry.set_obj()
#self.assertEqual(entry.obj['name'], 'Gopal')
#region = get_region_by_id(66)
#region.set_obj()
#self.assertEqual(region.obj['name'], 'Spain')
#c = Client()
#r = c.get('/transformer/entry/12188/reviews')
#region = get_region_by_id(66, force=False)
#region.set_obj()
#self.assertEqual(region.obj['name'], 'Spain')
#c = Client()
#r = c.get('/transformer/entry/12188')
#self.assertEqual(r.content.decode('utf-8').find('Spain')>-1,True)
#c = Client()
##r = c.get('/transformer/entry/12188?force=True')
#self.assertEqual(r.content.decode('utf-8').find('Spain')>-1,True)
#c = Client()
#r = c.get('/transformer/region/66')
#data = serializers.serialize("json", Region.objects.all())
#f = open('vegbasketapp/transformer/fixtures/region_transformer.json', 'w')
#f.write(data)
#f.close()
#data = serializers.serialize("json", Entry.objects.all())
#f = open('vegbasketapp/transformer/fixtures/entry_transformer.json', 'w')
#f.write(data)
#f.close()
#data = serializers.serialize("json", Reviews.objects.all())
#f = open('vegbasketapp/transformer/fixtures/reviews_transformer.json', 'w')
#f.write(data)
#f.close()
def handle(self, *args, **options):
print(args)
for region_id in args:
try:
print("getting region", region_id)
region = get_region_by_id(region_id)
region.set_obj()
entries = get_entries(region.obj['uri'])
for result in entries.results:
print(region_id, result['uri'])
get_entry(result['uri'])
sleep_random(1, 5)
sleep_random(4, 12)
except KeyError:
print("key error")
self.stdout.write('Successfully got data')
def convert_region(region_id):
"""Convert single region and it's parent.
Notes
-----
Works in the recursive way.
"""
vg_region_type = ContentType.objects.get(app_label="transformer", model="region")
#vg_entry_type = ContentType.objects.get(app_label="transformer", model="entry")
try:
region = Region.objects.get(source_id=region_id)
except Region.DoesNotExist:
region = get_region_by_id(region_id)
region.set_obj()
if VeggieSailorRegion.objects.filter(source_region=region).count()>0:
vs_region = VeggieSailorRegion.objects.get(source_region=region)
vs_region.content_type = vg_region_type
vs_region.object_id = region.source_id
vs_region.save()
return vs_region
vs_region = VeggieSailorRegion()
vs_region.name = region.obj['name']
#try:
#vs_region.name = region.obj['name']
#except KeyError:
#from ipdb import set_trace; set_trace()
vs_region.content_type = vg_region_type
vs_region.object_id = region.source_id
vs_region.source_region = region
vs_region.save()
if 'parent' in region.obj and 'uri' in region.obj['parent']:
parent_id = get_region_id(region.obj['parent']['uri'])
#print ("has parent", region_id, parent_id)
vs_region_parent = convert_region(parent_id)
vs_region.parent = vs_region_parent
vs_region.save()
return vs_region
def test_get_region_by_id(self):
region = get_region_by_id(66)
region.set_obj()
self.assertEqual(region.obj['name'], 'Spain')
def test_request(self, VegGuideParser):
VegGuideParser.return_value.result = json.loads(REGION_23)
vg_region = (get_region_by_id(23))
vg_region.set_obj()
self.assertEqual(vg_region.obj['name'], 'Ontario')
def region_root(request, region_id):
force = request.GET.get('force', False)
region_tmp = get_region_by_id(0, force)
return HttpResponse(region_tmp.results_source)
def test_get_region_by_id_force(self):
region = get_region_by_id(66, force=True)
region.set_obj()
self.assertEqual(region.obj['name'], 'Spain')
def region(request, region_id):
force = request.GET.get("force", False)
region_tmp = get_region_by_id(region_id, force)
return HttpResponse(region_tmp.results_source)