-
Notifications
You must be signed in to change notification settings - Fork 1
/
importer.py
107 lines (82 loc) · 3.19 KB
/
importer.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
#!/usr/bin/env python
from model import InstaStore
from google.appengine.ext import db
import config
import datetime
import json
import random
import urllib2
import webapp2
API_ENDPOINT = 'https://api.instagram.com/v1/'
MAX_IMPORT = 20
class ImportHandler(webapp2.RequestHandler):
# Returns true if image exists
def does_image_id_exist(self, image_id):
q = InstaStore.all(keys_only=True)
q.filter('image_id =', image_id)
result = q.get()
if result:
return True
return False
# Gets the contents of a given url
def get_url_contents(self, url):
req = urllib2.Request(url)
return urllib2.urlopen(req)
# Adds one image into the datastore
def insert_to_datastore(self, image):
try:
caption = image['caption']['text']
except TypeError:
caption = ''
insta_img = InstaStore(
caption=caption,
created=datetime.datetime.fromtimestamp(int(image['created_time'])),
image_id=image['id'],
image_photo_url=image['images']['standard_resolution']['url'],
image_photo_thumbnail_url=image['images']['thumbnail']['url'],
image_url=image['link'],
user_avatar_url=image['user']['profile_picture'],
user_id=image['user']['id'],
user_name=image['user']['username'],
user_real_name=image['user']['full_name'],
user_url='http://instagram.com/%s' % image['user']['username']
)
insta_img.put()
print 'Added %s to the datastore' % image['link']
def get(self, filter):
self.response.write('Importing...')
i = 0
if filter == 'tag':
url = '%stags/%s/media/recent?client_id=%s' % (API_ENDPOINT, random.choice(config.tags), config.client_id)
elif filter == 'user':
url = '%susers/%s/media/recent?client_id=%s' % (API_ENDPOINT, config.user_id, config.client_id)
else:
url = '%slocations/%s/media/recent?client_id=%s' % (API_ENDPOINT, config.location_id, config.client_id)
print url
while i < MAX_IMPORT:
response = self.get_url_contents(url)
jason = json.loads(response.read())
images = jason['data']
flag = False
for image in images:
print image['id']
if self.does_image_id_exist(image['id']):
continue
self.insert_to_datastore(image)
flag = True
self.response.write('<hr />')
self.response.write('<div style="display: block;">')
self.response.write('<a href="' + image['link'] + '" target="_blank"><img src="' + image['images']['thumbnail']['url'] + '"/></a><br />')
self.response.write('</div>')
if flag is False:
i = MAX_IMPORT
try:
url = jason['pagination']['next_url']
except KeyError:
i = MAX_IMPORT
i += 1
class TruncateData(webapp2.RequestHandler):
def get(self):
query = InstaStore.all(keys_only=True)
entries =query.fetch(1000)
db.delete(entries)