class BaseTestCase(unittest.TestCase):
def setUp(self):
super(BaseTestCase, self).setUp()
self.handler = OAuthHandler(self.consumer_key, self.consumer_secret)
self.handler.set_access_token(self.oauth_token, self.oauth_token_secret)
self.api = FiveHundredPXAPI(auth_handler=self.handler)
self.unauthorized_api = FiveHundredPXAPI()
self.follower_id = '925306' # test user id
self.user_id = '727199' # this is akirahrkw's id
self.photo = None # sample photo for test
def tearDown(self):
pass
def _get_sample_photo(self):
if not self.photo:
json = self.api.photos(require_auth=True, feature='user', user_id=self.follower_id, rpp=1)
self.assertIsNotNone(json)
self.photo = json['photos'][0]
return self.photo
class BaseTestCase(unittest.TestCase):
def setUp(self):
super(BaseTestCase, self).setUp()
self.handler = OAuthHandler(self.consumer_key, self.consumer_secret)
self.handler.set_access_token(self.oauth_token,
self.oauth_token_secret)
self.api = FiveHundredPXAPI(auth_handler=self.handler)
self.unauthorized_api = FiveHundredPXAPI()
self.follower_id = '925306' # test user id
self.user_id = '727199' # this is akirahrkw's id
self.photo = None # sample photo for test
def tearDown(self):
pass
def _get_sample_photo(self):
if not self.photo:
json = self.api.photos(require_auth=True,
feature='user',
user_id=self.follower_id,
rpp=1)
self.assertIsNotNone(json)
self.photo = json['photos'][0]
return self.photo
import numpy as np
import csv
CONSUMER_KEY = environ['PX_CONSUMER_KEY']
api = FiveHundredPXAPI()
feature_list = ['fresh_today']
maxlimit = 1000
allurls = np.chararray((maxlimit * len(feature_list), ), itemsize=300)
allratings = np.zeros((maxlimit * len(feature_list), ))
for idx, feature_type in enumerate(feature_list):
photos = api.photos(consumer_key=CONSUMER_KEY,
feature=feature_type,
image_size=21,
sort_direction='asc',
page=1,
rpp=maxlimit)
urls = map(lambda x: x['image_url'], photos['photos'])
ratings = map(lambda x: x['highest_rating'], photos['photos'])
allurls[idx * maxlimit:(idx + 1) * maxlimit] = urls
allratings[idx * maxlimit:(idx + 1) * maxlimit] = ratings
_, idx = np.unique(allurls, return_index=True)
img = allurls[idx]
rat = allratings[idx]
sz = len(img)
p = np.random.permutation(len(img))
class FivehundredPXTestCase(unittest.TestCase):
def setUp(self):
super(FivehundredPXTestCase, self).setUp()
self.consumer_key = CONSUMER_KEY
self.consumer_secret = CONSUMER_SECRET
self.oauth_token = OAUTH_TOKEN
self.oauth_token_secret = OAUTH_TOKEN_SECRET
self.handler = OAuthHandler(self.consumer_key,self.consumer_secret)
self.handler.set_access_token(self.oauth_token,self.oauth_token_secret)
self.api = FiveHundredPXAPI(auth_handler=self.handler)
self.unauthorized_api = FiveHundredPXAPI()
if SECOND_USER:
self.second_handler = OAuthHandler(self.consumer_key,self.consumer_secret)
self.second_oauth_token = SECOND_OAUTH_TOKEN
self.second_oauth_token_secret = SECOND_OAUTH_TOKEN_SECRET
self.second_handler.set_access_token(self.second_oauth_token, self.second_oauth_token_secret)
self.second_api = FiveHundredPXAPI(auth_handler=self.second_handler)
self.follower_id = '925306'
self.user_id = '642049'
self.filepath = 'images/africa.jpg'
def tearDown(self):
pass
def testusers(self):
self.api.users()
def testusersshow(self):
self.unauthorized_api.users_show(consumer_key=self.consumer_key, id='727199')
self.unauthorized_api.users_show(consumer_key=self.consumer_key, username='******')
self.unauthorized_api.users_show(consumer_key=self.consumer_key, email='*****@*****.**')
self.api.users_show(require_auth=True,id='727199')
self.api.users_show(require_auth=True,username='******')
self.api.users_show(require_auth=True,email='*****@*****.**')
def testuserfriends(self):
self.unauthorized_api.users_friends(consumer_key=self.consumer_key,id=self.user_id,rpp=5,page=2)
self.api.users_friends(require_auth=True,id=self.user_id,rpp=5,page=2)
self.api.users_friends_post(require_auth=True,id=self.follower_id)
self.api.users_friends_delete(require_auth=True,id=self.follower_id)
def testuserfollowers(self):
self.unauthorized_api.users_followers(consumer_key=self.consumer_key,id=self.user_id,rpp=5,page=2)
self.api.users_followers(require_auth=True,id=self.user_id,rpp=5,page=2)
def testblogs(self):
self.unauthorized_api.blogs(consumer_key=self.consumer_key,rpp=2,page=1)
blogs = self.api.blogs(require_auth=True,rpp=2,page=1)
for blog in blogs['blog_posts']:
blog = self.api.blogs_id(require_auth=True,id=blog['id'])
comments = self.api.blogs_comments(require_auth=True,id=blog['id'])
def testblogspost(self):
blog = self.api.blogs_post(title='title test', body='body test', tags='akira,hirakawa')
time.sleep(2)
uid = blog['id']
self.api.blogs_update(id=uid, title='title test 2', body='body test 2', tags='akira,hirakawa')
self.api.blogs_comments_post(id=uid, body='test blog comment')
self.api.blogs_delete(id=uid)
def testcollections(self):
collection = self.api.collections_post(require_auth=True, title='collection title', path='test')
self.api.collections_update(require_auth=True, id=collection['id'], title='collection title 2', path='test2')
collections = self.api.collections(require_auth=True,rpp=2,page=1)
for collection in collections['collections']:
self.api.collections_id(require_auth=True, id=collection['id'])
self.api.collections_delete(id=collection['id'])
def testphotos(self):
json = self.unauthorized_api.photos(feature='popular',consumer_key=self.consumer_key)
json = self.api.photos(require_auth=True, feature='popular',rpp=3)
self.assert_( len(json['photos']) == 3 )
for photo in json['photos']:
photo_id = photo['id']
self.unauthorized_api.photos_id(id=photo_id,consumer_key=self.consumer_key)
self.api.photos_id(require_auth=True,id=photo_id)
self.unauthorized_api.photos_comments(id=photo_id,consumer_key=self.consumer_key)
def testphotosgenerator(self):
gen = self.api.photos(require_auth=True, feature='popular',rpp=1, as_generator=True)
for g in gen: g
def testphotossearch(self):
json = self.unauthorized_api.photos_search(consumer_key=self.consumer_key,term='test',rpp=1,page=1)
json = self.api.photos_search(require_auth=True, term='test',rpp=1,page=1)
self.assert_( len(json['photos']) == 1 )
def _update_photo(self,photo_id):
self.api.photos_comments_post(id=photo_id,body='test comment')
self.api.photos_favorite_post(id=photo_id)
self.api.photos_favorite_delete(id=photo_id)
self.api.photos_tags_post(id=photo_id,tags='test,test2')
self.api.photos_tags_delete(id=photo_id,tags='test,test2')
if SECOND_USER:
self.api.photos_vote_post(id=photo_id,vote='1')
self.api.photos_update(id=photo_id,name='akira test',description='akira description')
def testphotospost(self):
api = self.api if not SECOND_USER else self.second_api
access_key = self.oauth_token if not SECOND_USER else self.second_oauth_token
json = api.photos_post()
photo_id = json['photo']['id']
upload_key = str(json['upload_key'])
api.upload_photo(
photo_id=photo_id,
filename=self.filepath,
consumer_key=self.consumer_key,
upload_key=upload_key,
access_key=access_key
)
time.sleep(3)
self._update_photo(photo_id)
time.sleep(1)
api.photos_delete(id=photo_id)
json = api.photos_post()
photo_id = json['photo']['id']
upload_key = str(json['upload_key'])
file_type = mimetypes.guess_type(self.filepath)
fp = open(self.filepath,'rb')
api.upload_photo(
photo_id=photo_id,
fp=fp,
file_type=file_type[0],
consumer_key=self.consumer_key,
upload_key=upload_key,
access_key=access_key
)
fp.close()
time.sleep(3)
api.photos_delete(id=photo_id)
import numpy as np
import csv
CONSUMER_KEY = environ['PX_CONSUMER_KEY']
api = FiveHundredPXAPI()
feature_list = ['popular','highest_rated','upcoming','editors','fresh_today','fresh_yesterday','fresh_week']
maxlimit = 1000
allurls = np.chararray((maxlimit*len(feature_list),),itemsize=300)
allratings = np.zeros((maxlimit*len(feature_list),))
for idx, feature_type in enumerate(feature_list):
photos = api.photos(consumer_key=CONSUMER_KEY,
feature=feature_type,
image_size=21,
sort_direction='asc',
page=1,
rpp=maxlimit)
urls = map(lambda x: x['image_url'], photos['photos'])
ratings = map(lambda x: x['highest_rating'], photos['photos'])
allurls[idx*maxlimit:(idx+1)*maxlimit] = urls
allratings[idx*maxlimit:(idx+1)*maxlimit] = ratings
_, idx = np.unique(allurls, return_index=True)
img = allurls[idx]
rat = allratings[idx]
sz = len(img)
p = np.random.permutation(len(img))
class FivehundredPXTestCase(unittest.TestCase):
def setUp(self):
super(FivehundredPXTestCase, self).setUp()
self.consumer_key = CONSUMER_KEY
self.consumer_secret = CONSUMER_SECRET
self.oauth_token = OAUTH_TOKEN
self.oauth_token_secret = OAUTH_TOKEN_SECRET
self.handler = OAuthHandler(self.consumer_key, self.consumer_secret)
self.handler.set_access_token(self.oauth_token,
self.oauth_token_secret)
self.api = FiveHundredPXAPI(auth_handler=self.handler)
self.unauthorized_api = FiveHundredPXAPI()
if SECOND_USER:
self.second_handler = OAuthHandler(self.consumer_key,
self.consumer_secret)
self.second_oauth_token = SECOND_OAUTH_TOKEN
self.second_oauth_token_secret = SECOND_OAUTH_TOKEN_SECRET
self.second_handler.set_access_token(
self.second_oauth_token, self.second_oauth_token_secret)
self.second_api = FiveHundredPXAPI(
auth_handler=self.second_handler)
self.follower_id = '925306'
self.user_id = '642049'
self.filepath = 'images/africa.jpg'
def tearDown(self):
pass
def testusers(self):
self.api.users()
def testusersshow(self):
self.unauthorized_api.users_show(consumer_key=self.consumer_key,
id='727199')
self.unauthorized_api.users_show(consumer_key=self.consumer_key,
username='******')
self.unauthorized_api.users_show(consumer_key=self.consumer_key,
email='*****@*****.**')
self.api.users_show(require_auth=True, id='727199')
self.api.users_show(require_auth=True, username='******')
self.api.users_show(require_auth=True, email='*****@*****.**')
def testuserfriends(self):
self.unauthorized_api.users_friends(consumer_key=self.consumer_key,
id=self.user_id,
rpp=5,
page=2)
self.api.users_friends(require_auth=True,
id=self.user_id,
rpp=5,
page=2)
self.api.users_friends_post(require_auth=True, id=self.follower_id)
self.api.users_friends_delete(require_auth=True, id=self.follower_id)
def testuserfollowers(self):
self.unauthorized_api.users_followers(consumer_key=self.consumer_key,
id=self.user_id,
rpp=5,
page=2)
self.api.users_followers(require_auth=True,
id=self.user_id,
rpp=5,
page=2)
def testblogs(self):
self.unauthorized_api.blogs(consumer_key=self.consumer_key,
rpp=2,
page=1)
blogs = self.api.blogs(require_auth=True, rpp=2, page=1)
for blog in blogs['blog_posts']:
blog = self.api.blogs_id(require_auth=True, id=blog['id'])
comments = self.api.blogs_comments(require_auth=True,
id=blog['id'])
def testblogspost(self):
blog = self.api.blogs_post(title='title test',
body='body test',
tags='akira,hirakawa')
time.sleep(2)
uid = blog['id']
self.api.blogs_update(id=uid,
title='title test 2',
body='body test 2',
tags='akira,hirakawa')
self.api.blogs_comments_post(id=uid, body='test blog comment')
self.api.blogs_delete(id=uid)
def testcollections(self):
collection = self.api.collections_post(require_auth=True,
title='collection title',
path='test')
self.api.collections_update(require_auth=True,
id=collection['id'],
title='collection title 2',
path='test2')
collections = self.api.collections(require_auth=True, rpp=2, page=1)
for collection in collections['collections']:
self.api.collections_id(require_auth=True, id=collection['id'])
self.api.collections_delete(id=collection['id'])
def testphotos(self):
json = self.unauthorized_api.photos(feature='popular',
consumer_key=self.consumer_key)
json = self.api.photos(require_auth=True, feature='popular', rpp=3)
self.assert_(len(json['photos']) == 3)
for photo in json['photos']:
photo_id = photo['id']
self.unauthorized_api.photos_id(id=photo_id,
consumer_key=self.consumer_key)
self.api.photos_id(require_auth=True, id=photo_id)
self.unauthorized_api.photos_comments(
id=photo_id, consumer_key=self.consumer_key)
def testphotosgenerator(self):
gen = self.api.photos(require_auth=True,
feature='popular',
rpp=1,
as_generator=True)
for g in gen:
g
def testphotossearch(self):
json = self.unauthorized_api.photos_search(
consumer_key=self.consumer_key, term='test', rpp=1, page=1)
json = self.api.photos_search(require_auth=True,
term='test',
rpp=1,
page=1)
self.assert_(len(json['photos']) == 1)
def _update_photo(self, photo_id):
self.api.photos_comments_post(id=photo_id, body='test comment')
self.api.photos_favorite_post(id=photo_id)
self.api.photos_favorite_delete(id=photo_id)
self.api.photos_tags_post(id=photo_id, tags='test,test2')
self.api.photos_tags_delete(id=photo_id, tags='test,test2')
if SECOND_USER:
self.api.photos_vote_post(id=photo_id, vote='1')
self.api.photos_update(id=photo_id,
name='akira test',
description='akira description')
def testphotospost(self):
api = self.api if not SECOND_USER else self.second_api
access_key = self.oauth_token if not SECOND_USER else self.second_oauth_token
json = api.photos_post()
photo_id = json['photo']['id']
upload_key = str(json['upload_key'])
api.upload_photo(photo_id=photo_id,
filename=self.filepath,
consumer_key=self.consumer_key,
upload_key=upload_key,
access_key=access_key)
time.sleep(3)
self._update_photo(photo_id)
time.sleep(1)
api.photos_delete(id=photo_id)
json = api.photos_post()
photo_id = json['photo']['id']
upload_key = str(json['upload_key'])
file_type = mimetypes.guess_type(self.filepath)
fp = open(self.filepath, 'rb')
api.upload_photo(photo_id=photo_id,
fp=fp,
file_type=file_type[0],
consumer_key=self.consumer_key,
upload_key=upload_key,
access_key=access_key)
fp.close()
time.sleep(3)
api.photos_delete(id=photo_id)
class CrawlData():
def __init__(self,searchEngine):
self.searchEngine = searchEngine
self.count = 0
if searchEngine == 'google':
self.baseDir = 'googleImageResults'
self.rootUrl = 'https://ajax.googleapis.com/ajax/services/search/images?v=1.0&q='
self.apiKey = 'replace your api key here' # not needed if want < 64 images. Have not implemented the paid account version
self.opUrlKey = 'unescapedUrl'
elif searchEngine == 'bing':
self.baseDir = 'bingImageResults'
self.rootUrl = 'https://api.datamarket.azure.com/Bing/Search/v1/Image?Query='
self.apiKey = 'replace your api key here'
self.opUrlKey = 'MediaUrl'
elif searchEngine == '500px':
self.baseDir = '500pxImageResults'
self.CONSUMER_KEY = 'your consumer key here'
self.CONSUMER_SECRET = 'your consumer secret here'
self.opUrlKey = 'image_url'
# oauth details
self.handler = OAuthHandler(self.CONSUMER_KEY,self.CONSUMER_SECRET)
self.requestToken = self.handler.get_request_token()
self.handler.set_request_token(self.requestToken.key,self.requestToken.secret)
username = raw_input("Input your username: "******"Press Enter")
self.api.get_token_part_two((self.token,self.frob))
# self.cc_licenses = '1, 2, 3, 4, 5, 6, 7' for cc license search
self.cc_licenses = ''
# you have to generate the urls yourself for flickr
def grabDataFlickr(self,dataInfo):
for j in range(len(dataInfo)):
currFlickrMeta = dataInfo[j]
currUrl = 'http://farm1.staticflickr.com/{0}/{1}_{2}_z.jpg'.format(
currFlickrMeta['server'],
currFlickrMeta['id'],
currFlickrMeta['secret'])
opFileName = '{0}/Image_{1:010d}.jpg'.format(self.opDir,self.count)
print opFileName
urllib.urlretrieve(currUrl,opFileName)
self.count = self.count + 1
def grabData(self,dataInfo):
for j in range(len(dataInfo)):
currUrl = dataInfo[j][self.opUrlKey]
opFileName = '{0}/Image_{1:010d}.jpg'.format(self.opDir,self.count)
print opFileName
urllib.urlretrieve(currUrl,opFileName)
self.count = self.count + 1
def doSearch(self,queryTerm,pageNumber):
self.opDir = self.baseDir + '/' + queryTerm
if not os.path.exists(self.opDir):
os.makedirs(self.opDir)
if self.searchEngine == 'google':
searchUrl = self.rootUrl + urllib.quote(queryTerm) +'&start='+str(pageNumber*8)+'&userip=MyIP&rsz=8&imgtype=photo'
try:
response = requests.get(searchUrl).json()
dataInfo = response['responseData']['results']
except (IndexError,TypeError,ValueError,NameError):
print 'skipping'
return
elif self.searchEngine == 'bing':
searchUrl = self.rootUrl + '%27' + urllib.quote(queryTerm) + '%27&$format=json&$skip=' + str(pageNumber*10)
try:
response = requests.get(searchUrl, auth=(self.apiKey,self.apiKey)).json()
dataInfo = response['d']['results']
except (IndexError,TypeError,ValueError,NameError):
print 'skipping'
return
elif self.searchEngine == 'flickr':
# documentation available at http://www.flickr.com/services/api/flickr.photos.search.html
try:
if not self.cc_licenses:
responseUnStripped = self.api.photos_search(text=queryTerm,
content_type=1,
page=(pageNumber+1),
format='json')
else:
responseUnStripped = self.api.photos_search(text=queryTerm,
content_type=1,
page =(pageNumber+1),
license=self.cc_licenses,
format='json')
response = simplejson.loads(re.search(r'jsonFlickrApi\(>?(.+)\)', responseUnStripped).group(1))
self.grabDataFlickr(response['photos']['photo'])
except (IndexError,TypeError,ValueError,NameError):
print 'skipping'
return
elif self.searchEngine == '500px':
try:
response = self.api.photos_search(require_auth=True,
tag=queryTerm,
page=(pageNumber+1),
image_size=5)
dataInfo = response['photos']
except (IndexError,TypeError,ValueError,NameError):
print 'skipping'
return
self.grabData(dataInfo)
def doSearchPopular(self,pageNumber):
self.opDir = self.baseDir + '/popular on ' + datetime.datetime.now().strftime('%d-%m-%Y')
if not os.path.exists(self.opDir):
os.makedirs(self.opDir)
if self.searchEngine == '500px':
try:
currPage = pageNumber+1
response = self.api.photos(require_auth=True,
feature='popular',
sort='rating',
image_size=5,
page=currPage)
dataInfo = response['photos']
except (IndexError,TypeError,ValueError,NameError):
print 'skipping'
return
self.grabData(dataInfo)
