def scrape(self, user_data):
"""
Scrape photos from google photos using the following API:
https://developers.google.com/drive/v3/reference/
"""
try:
oauth = user_data.services['google']
except KeyError:
return False
if 'denied' in oauth:
return False
credentials = client.OAuth2Credentials(
access_token=oauth['access_token'],
client_id=CONFIG.get('google_client_id'),
client_secret=CONFIG.get('google_client_secret'),
refresh_token=oauth.get('refresh_token', None),
token_uri=client.GOOGLE_TOKEN_URI,
token_expiry=oauth.get('expires_in', None),
user_agent='QS-server-agent/1.0',
id_token=oauth.get('id_token', None))
http = credentials.authorize(httplib2.Http())
gplus = build('drive', 'v3', http=http)
photos = list(
apiclient_paginate(gplus.files(),
'list', {
'spaces': 'photos',
'fields': 'files,kind,nextPageToken',
},
max_results=self.num_images_per_user))
for photo in photos:
faces = yield find_faces_url(photo['thumbnailLink'], upsample=2)
photo['faces'] = faces
return photos
def scrape(self, user_data):
"""
Scrape photos from google photos using the following API:
https://developers.google.com/drive/v3/reference/
"""
try:
oauth = user_data.services['google']
except KeyError:
return False
if 'denied' in oauth:
return False
credentials = client.OAuth2Credentials(
access_token=oauth['access_token'],
client_id=CONFIG.get('google_client_id'),
client_secret=CONFIG.get('google_client_secret'),
refresh_token=oauth.get('refresh_token', None),
token_uri=client.GOOGLE_TOKEN_URI,
token_expiry=oauth.get('expires_in', None),
user_agent='QS-server-agent/1.0',
id_token=oauth.get('id_token', None)
)
http = credentials.authorize(httplib2.Http())
gplus = build('drive', 'v3', http=http)
photos = list(apiclient_paginate(gplus.files(), 'list', {
'spaces': 'photos',
'fields': 'files,kind,nextPageToken',
}, max_results=self.num_images_per_user))
for photo in photos:
faces = yield find_faces_url(photo['thumbnailLink'], upsample=2)
photo['faces'] = faces
return photos
def scrape(self, user_data):
"""
Scrapes text and contacts from user gmail account
https://developers.google.com/gmail/api/v1/reference/
"""
try:
oauth = user_data.services['google']
except KeyError:
return False
if 'denied' in oauth:
return False
creds = client.OAuth2Credentials(
access_token=oauth['access_token'],
client_id=CONFIG.get('google_client_id'),
client_secret=CONFIG.get('google_client_secret'),
refresh_token=oauth.get('refresh_token', None),
token_uri=client.GOOGLE_TOKEN_URI,
token_expiry=oauth.get('expires_in', None),
user_agent='QS-server-agent/1.0',
id_token=oauth.get('id_token', None)
)
# Set up client
http = creds.authorize(httplib2.Http())
gmail = build('gmail', 'v1', http=http)
# Get seed language tokens
# Go through each token, seed a search to find threads we want to
# search through
thread_ids_per_token = {}
for token in self.tokens:
res = gmail.users().messages().list(
userId='me',
q='in:sent {0}'.format(token)).execute()
thread_ids_per_token[token] = self.paginate_messages(
gmail,
res,
max_results=self.num_threads
)
equalize_dict(thread_ids_per_token, self.num_threads)
threads = set(thread for threads in thread_ids_per_token.values()
for thread in threads)
data = { "text" : [],
"snippets" : [],
"people" : [] }
for i, thread in enumerate(threads):
email, snippet = self.get_beg_thread(gmail, thread)
body = None
if email['body'] is not None:
body = email['body']
people = self.get_recipient(email)
data['text'].append(body)
data['snippets'].append(snippet)
data['people'].append(people)
return data
def scrape(self, user_data):
"""
Scrapes text and contacts from user gmail account
https://developers.google.com/gmail/api/v1/reference/
"""
try:
oauth = user_data.services['google']
except KeyError:
return False
if 'denied' in oauth:
return False
creds = client.OAuth2Credentials(
access_token=oauth['access_token'],
client_id=CONFIG.get('google_client_id'),
client_secret=CONFIG.get('google_client_secret'),
refresh_token=oauth.get('refresh_token', None),
token_uri=client.GOOGLE_TOKEN_URI,
token_expiry=oauth.get('expires_in', None),
user_agent='QS-server-agent/1.0',
id_token=oauth.get('id_token', None))
# Set up client
http = creds.authorize(httplib2.Http())
gmail = build('gmail', 'v1', http=http)
# Get seed language tokens
# Go through each token, seed a search to find threads we want to
# search through
thread_ids_per_token = {}
for token in self.tokens:
res = gmail.users().messages().list(
userId='me', q='in:sent {0}'.format(token)).execute()
thread_ids_per_token[token] = self.paginate_messages(
gmail, res, max_results=self.num_threads)
equalize_dict(thread_ids_per_token, self.num_threads)
threads = set(thread for threads in thread_ids_per_token.values()
for thread in threads)
data = {"text": [], "snippets": [], "people": []}
for i, thread in enumerate(threads):
email, snippet = self.get_beg_thread(gmail, thread)
body = None
if email['body'] is not None:
body = email['body']
people = self.get_recipient(email)
data['text'].append(body)
data['snippets'].append(snippet)
data['people'].append(people)
return data
def get(self):
showid = self.get_argument('showtime_id')
shares = self.get_arguments('share')
passphrase = self.get_argument('passphrase', None)
ticket_api = CONFIG.get('ticket_api')
# we could also just pass the raw arguments, but this is more explicit
# and 'self documenting'
params = [('showtime_id', showid)]
if passphrase:
params += [('passphrase', passphrase)]
elif shares:
params += [('share', s) for s in shares]
url = url_concat(ticket_api + '/api/showtimes/keys', params)
show_data_raw = yield httpclient.fetch(url)
if show_data_raw.code != 200:
return self.error(show_data_raw.code, show_data_raw.body)
show_data = json.loads(show_data_raw.body)
show_date = show_data['data']['date']
rfidb = yield RFIDB.get_global()
result = yield rfidb.unlock_show(showid, show_date,
show_data['data']['users'])
return self.api_response(result)
def get(self):
date_raw = self.get_argument('date')
available_tickets = int(self.get_argument('tickets', 40))
timezone = tz.gettz(CONFIG.get('timezone'))
date = date_parser.parse(date_raw).replace(tzinfo=timezone)
showid = yield create_showtime(date, available_tickets)
return self.api_response({'showid': showid})
def get(self):
date_raw = self.get_argument('date')
available_tickets = int(self.get_argument('tickets', 40))
timezone = tz.gettz(CONFIG.get('timezone'))
date = date_parser.parse(date_raw).replace(tzinfo=timezone)
showid = yield create_showtime(date, available_tickets)
return self.api_response({'showid': showid})
def require_basic_auth(handler, kwargs):
auth_header = handler.request.headers.get('Authorization')
if auth_header is None or not auth_header.startswith('Basic '):
handler.set_status(401)
handler.set_header('WWW-Authenticate',
'Basic realm=Restricted')
handler._transforms = []
handler.finish()
return False
auth_decoded = base64.b64decode(auth_header[6:]).decode('utf-8')
user, password = auth_decoded.split(':', 2)
hashed_password = CONFIG.get('admin_user_pass').encode('utf-8')
encoded_password = password.encode('utf-8')
if (user == CONFIG.get('admin_user_id') and bcrypt.hashpw(
encoded_password, hashed_password) == hashed_password):
return True
handler.set_status(401)
handler._transforms = []
handler.finish()
return False
def require_basic_auth(handler, kwargs):
auth_header = handler.request.headers.get('Authorization')
if auth_header is None or not auth_header.startswith('Basic '):
handler.set_status(401)
handler.set_header('WWW-Authenticate',
'Basic realm=Restricted')
handler._transforms = []
handler.finish()
return False
auth_decoded = base64.b64decode(auth_header[6:]).decode('utf-8')
user, password = auth_decoded.split(':', 2)
hashed_password = CONFIG.get('admin_user_pass').encode('utf-8')
encoded_password = password.encode('utf-8')
if (user == CONFIG.get('admin_user_id') and
bcrypt.hashpw(encoded_password, hashed_password) ==
hashed_password):
return True
handler.set_status(401)
handler._transforms = []
handler.finish()
return False
def get(self):
showid = self.get_argument('showtime_id')
shares = self.get_arguments('share')
passphrase = self.get_argument('passphrase', None)
reprocess = bool(self.get_argument('reprocess', None) is not None)
ticket_api = CONFIG.get('ticket_api')
# we could also just pass the raw arguments, but this is more explicit
# and 'self documenting'
params = [('showtime_id', showid)]
if passphrase:
params += [('passphrase', passphrase)]
elif shares:
params += [('share', s) for s in shares]
url = url_concat(ticket_api + '/api/showtimes/access_tokens', params)
show_data_raw = yield httpclient.fetch(url, request_timeout=180)
if show_data_raw.code != 200:
return self.error(show_data_raw.code, show_data_raw.body)
exhibitperms = yield ExhibitPermissions.get_global()
show_data = json.loads(show_data_raw.body)
show_date = show_data['data']['date']
users_added = []
for user_data in show_data['data']['users']:
userid = user_data.pop('id')
perms = yield exhibitperms.get_permissions(userid)
if perms and not reprocess:
users_added.append({'userid': userid,
'permissions': perms,
'process': False})
publickey = user_data['publickey']
privatekey = user_data.get('privatekey')
meta = user_data.get('meta') or {}
meta.update({'showid': showid, 'permissions': perms,
'showdate': show_date})
user = User(userid, publickey, services=user_data['services'],
privatekey_pem=privatekey, meta=meta)
users_added.append({'userid': userid, 'process': True})
ioloop.IOLoop.current().add_callback(partial(userprocess, user))
return self.api_response(users_added)
def get(self):
# inefficient databse query, figure out how to perform this query in
# db.
# remove all expired tickets
yield remove_expired_tickets()
showtimes = yield get_showtimes()
reservations = yield get_reservations()
showtime_map = {}
for reservation in reservations:
ticket_id = reservation["showtime_id"]
if ticket_id in showtime_map:
showtime_map[ticket_id] += 1
else:
showtime_map[ticket_id] = 1
result = []
timezone = tz.gettz(CONFIG.get('timezone'))
timeformat = "%A %d, %B - %I:%M%p"
for showtime in showtimes:
showid = showtime["id"]
dateString = showtime["date"]. \
astimezone(timezone). \
strftime(timeformat)
if showid in showtime_map:
available_tickets = \
showtime["max_booking"] - showtime_map[showid]
if available_tickets < 0:
available_tickets = 0
else:
available_tickets = showtime["max_booking"]
result.append({
"id": showid,
"date": dateString,
"available_tickets": available_tickets,
})
self.api_response(result)
def get(self):
# inefficient databse query, figure out how to perform this query in
# db.
# remove all expired tickets
yield remove_expired_tickets()
showtimes = yield get_showtimes()
reservations = yield get_reservations()
showtime_map = {}
for reservation in reservations:
ticket_id = reservation["showtime_id"]
if ticket_id in showtime_map:
showtime_map[ticket_id] += 1
else:
showtime_map[ticket_id] = 1
result = []
timezone = tz.gettz(CONFIG.get('timezone'))
timeformat = "%A %d, %B - %I:%M%p"
for showtime in showtimes:
showid = showtime["id"]
dateString = showtime["date"]. \
astimezone(timezone). \
strftime(timeformat)
if showid in showtime_map:
available_tickets = \
showtime["max_booking"] - showtime_map[showid]
if available_tickets < 0:
available_tickets = 0
else:
available_tickets = showtime["max_booking"]
result.append({
"id": showid,
"date": dateString,
"available_tickets": available_tickets,
})
self.api_response(result)
from lib.eta_callback_receiver import ETaCallbackReceiverInstaller
from lib.eta_callback_filter import ETaCallbackFilterInstaller
from lib.eta_callback_sender import ETaCallbackSenderInstaller
from lib.eta_callback_filter_app import ETaCallbackFilterAppInstaller
from lib.eta_callback_sender_app import ETaCallbackSenderAppInstaller
from lib.ephi import EPHIInstaller
from lib.epcilon import EpcILoNInstaller
from lib.ons import ONSConfigurer
from lib.alfa import ALfAInstaller
from lib.omega import OMeGaInstaller
from lib.ypsilon import YPSilonInstaller
from lib.lambda_iota import LaMBDaInstaller
if __name__ == "__main__":
if "--accept-defaults" in sys.argv:
CONFIG.set("global", "accept_defaults", "true")
lib.utils.putWarning("Installing with defaults values from `resources/install.ini")
else:
CONFIG.set("global", "accept_defaults", "false")
lib.utils.putTitle(" IoTa Installer")
GlobalConfigurer().run()
CertConfigurer().run()
TomcatInstaller().run()
DBConfigurer().run()
SigMaCertConfigurer().run()
SigMAInstaller().run()
EpcisInstaller().run()
LDAPConfigurer().run()
YPSilonInstaller().run()
ETaInstaller().run()
EPHIInstaller().run()
def scrape(self, user_data):
try:
tokens = user_data.services['reddit']
except KeyError:
return False
if 'denied' in tokens:
return False
client_id = CONFIG.get("reddit_client_id")
client_secret = CONFIG.get("reddit_client_secret")
r = praw.Reddit(user_agent="QS-agent/1.0")
r.set_oauth_app_info(
client_id=client_id,
client_secret=client_secret,
redirect_uri="http://iamadatapoint.com:8085/auth/reddit")
tokens = r.refresh_access_information(
refresh_token=tokens['refresh_token'])
r.set_access_credentials(**tokens)
data = {}
subs = []
count = 0
for i in r.get_my_subreddits():
if count > self.num_scrape:
break
load = {}
load['name'] = i.display_name
load['url'] = i.url
subs.append(load)
count += 1
data['subs'] = subs
msgs = []
count = 0
for i in r.get_inbox():
if count > self.num_scrape:
break
load = {}
load['body'] = i.body
if i.author:
load['author'] = i.author.name
load['comment'] = i.was_comment
msgs.append(load)
count += 1
data['text'] = msgs
me = r.get_me()
# May want to grab own profile name
karma_c = me.comment_karma
karma_l = me.link_karma
submissions = []
count = 0
for i in me.get_submitted():
if count > self.num_scrape:
break
load = {}
load['title'] = i.title
load['url'] = i.url
load['score'] = i.score
load['subreddit'] = i.subreddit.title
submissions.append(load)
count += 1
data['submissions'] = submissions
data['comment_karma'] = karma_c
data['link_karma'] = karma_l
likes = []
count = 0
for i in me.get_upvoted():
if count > self.num_scrape:
break
load = {}
load['title'] = i.title
load['url'] = i.url
likes.append(load)
count += 1
data['likes'] = likes
dislikes = []
count = 0
for i in me.get_downvoted():
if count > self.num_scrape:
break
load = {}
load['title'] = i.title
load['url'] = i.url
dislikes.append(load)
count += 1
data['dislikes'] = dislikes
return data
def scrape(self, user_data):
try:
oauth = user_data.services['google']
except KeyError:
return False
if 'denied' in oauth:
return False
credentials = client.OAuth2Credentials(
access_token=oauth['access_token'],
client_id=CONFIG.get('google_client_id'),
client_secret=CONFIG.get('google_client_secret'),
refresh_token=oauth.get('refresh_token', None),
token_uri=client.GOOGLE_TOKEN_URI,
token_expiry=oauth.get('expires_in', None),
user_agent='QS-server-agent/1.0',
id_token=oauth.get('id_token', None)
)
http = credentials.authorize(httplib2.Http())
try:
youtube = build('youtube', 'v3', http=http)
except:
return False
userinfo = list(apiclient_paginate(youtube.channels(), 'list', {
'part': 'statistics,contentDetails',
'mine': True,
}))[0]
special_videos = {}
user_special_playlists = userinfo['contentDetails']['relatedPlaylists']
for name, playlistid in user_special_playlists.items():
try:
videos = list(
apiclient_paginate(
youtube.playlistItems(),
'list',
{
'part': 'snippet',
'playlistId': playlistid,
}, max_results=self.num_videos_per_playlist
)
)
special_videos[name] = videos
except Exception:
self.logger.exception("Error getting special playlists")
continue
playlists = list(apiclient_paginate(youtube.playlists(), 'list', {
'part': 'id,snippet',
'mine': True,
}))
subscriptions = list(apiclient_paginate(
youtube.subscriptions(), 'list', {
'part': 'snippet',
'mine': True,
})
)
result = {
'subscriptions': subscriptions,
'playlists': playlists,
'userinfo': userinfo,
'special_videos': special_videos,
}
return result
def scrape(self, user_data):
try:
oauth = user_data.services['google']
except KeyError:
return False
if 'denied' in oauth:
return False
credentials = client.OAuth2Credentials(
access_token=oauth['access_token'],
client_id=CONFIG.get('google_client_id'),
client_secret=CONFIG.get('google_client_secret'),
refresh_token=oauth.get('refresh_token', None),
token_uri=client.GOOGLE_TOKEN_URI,
token_expiry=oauth.get('expires_in', None),
user_agent='QS-server-agent/1.0',
id_token=oauth.get('id_token', None))
http = credentials.authorize(httplib2.Http())
try:
youtube = build('youtube', 'v3', http=http)
except:
return False
userinfo = list(
apiclient_paginate(youtube.channels(), 'list', {
'part': 'statistics,contentDetails',
'mine': True,
}))[0]
special_videos = {}
user_special_playlists = userinfo['contentDetails']['relatedPlaylists']
for name, playlistid in user_special_playlists.items():
try:
videos = list(
apiclient_paginate(
youtube.playlistItems(),
'list', {
'part': 'snippet',
'playlistId': playlistid,
},
max_results=self.num_videos_per_playlist))
special_videos[name] = videos
except Exception:
self.logger.exception("Error getting special playlists")
continue
playlists = list(
apiclient_paginate(youtube.playlists(), 'list', {
'part': 'id,snippet',
'mine': True,
}))
subscriptions = list(
apiclient_paginate(youtube.subscriptions(), 'list', {
'part': 'snippet',
'mine': True,
}))
result = {
'subscriptions': subscriptions,
'playlists': playlists,
'userinfo': userinfo,
'special_videos': special_videos,
}
return result
def scrape(self, user_data):
try:
access = user_data.services['tumblr']['access_token']
secret = user_data.services['tumblr']['access_token_secret']
except KeyError:
return False
if 'denied' in access:
return False
consumer_key = CONFIG.get("tumblr_client_id")
consumer_secret = CONFIG.get("tumblr_client_secret")
client = pytumblr.TumblrRestClient(consumer_key, consumer_secret,
access, secret)
tumblr_data = {
"hosted_blogs": [],
"following": [],
"likes": [],
"suggestions": []
}
profile = client.info()
total_likes = profile['user']['likes']
total_following = profile['user']['following']
# Should we be doing anything in case we get garbage/shell accounts?
# Write this comment in GitHub
if total_likes == 0 and total_following == 0:
return "Empty Account"
hosts = []
for i in profile['user']['blogs']:
blog = {}
blog['total_posts'] = i['total_posts']
blog['followers'] = i['followers']
blog['description'] = i['description']
blog['url'] = i['url']
blog['name'] = i['name']
blog['title'] = i['title']
hosts.append(blog)
tumblr_data['hosted_blogs'] = hosts
dash = client.dashboard()
dash_data = []
for data in dash['posts']:
goods = {}
goods['source_title'] = data.get('source_title', None)
goods['post_url'] = data.get('post_url', None)
goods['date'] = data.get('date', None)
goods['summary'] = data.get('summary', None)
goods['photos'] = data.get('photos', None)
goods['blog_name'] = data.get('blog_name', None)
goods['text'] = data.get('text', None)
goods['caption'] = data.get('caption', None)
goods['source_url'] = data.get('source_url', None)
goods['content'] = data.get('content', None)
dash_data.append(goods)
tumblr_data['suggestions'] = dash_data
like_data = []
likes = client.likes()
total = likes['liked_count']
like_data = likes['liked_posts']
if total > 20:
full_set = yield self.paginate_likes(client, total)
like_data.append(full_set)
if len(like_data) > 0:
tumblr_data['likes'] = self.clean_likes(like_data)
follows = client.following()
tumblr_data['follows'] = follows
return tumblr_data
