def scrape(self, user_data): """ Scrape photos from google photos using the following API: https://developers.google.com/drive/v3/reference/ """ try: oauth = user_data.services['google'] except KeyError: return False if 'denied' in oauth: return False credentials = client.OAuth2Credentials( access_token=oauth['access_token'], client_id=CONFIG.get('google_client_id'), client_secret=CONFIG.get('google_client_secret'), refresh_token=oauth.get('refresh_token', None), token_uri=client.GOOGLE_TOKEN_URI, token_expiry=oauth.get('expires_in', None), user_agent='QS-server-agent/1.0', id_token=oauth.get('id_token', None)) http = credentials.authorize(httplib2.Http()) gplus = build('drive', 'v3', http=http) photos = list( apiclient_paginate(gplus.files(), 'list', { 'spaces': 'photos', 'fields': 'files,kind,nextPageToken', }, max_results=self.num_images_per_user)) for photo in photos: faces = yield find_faces_url(photo['thumbnailLink'], upsample=2) photo['faces'] = faces return photos
def scrape(self, user_data): """ Scrape photos from google photos using the following API: https://developers.google.com/drive/v3/reference/ """ try: oauth = user_data.services['google'] except KeyError: return False if 'denied' in oauth: return False credentials = client.OAuth2Credentials( access_token=oauth['access_token'], client_id=CONFIG.get('google_client_id'), client_secret=CONFIG.get('google_client_secret'), refresh_token=oauth.get('refresh_token', None), token_uri=client.GOOGLE_TOKEN_URI, token_expiry=oauth.get('expires_in', None), user_agent='QS-server-agent/1.0', id_token=oauth.get('id_token', None) ) http = credentials.authorize(httplib2.Http()) gplus = build('drive', 'v3', http=http) photos = list(apiclient_paginate(gplus.files(), 'list', { 'spaces': 'photos', 'fields': 'files,kind,nextPageToken', }, max_results=self.num_images_per_user)) for photo in photos: faces = yield find_faces_url(photo['thumbnailLink'], upsample=2) photo['faces'] = faces return photos
def scrape(self, user_data): """ Scrapes text and contacts from user gmail account https://developers.google.com/gmail/api/v1/reference/ """ try: oauth = user_data.services['google'] except KeyError: return False if 'denied' in oauth: return False creds = client.OAuth2Credentials( access_token=oauth['access_token'], client_id=CONFIG.get('google_client_id'), client_secret=CONFIG.get('google_client_secret'), refresh_token=oauth.get('refresh_token', None), token_uri=client.GOOGLE_TOKEN_URI, token_expiry=oauth.get('expires_in', None), user_agent='QS-server-agent/1.0', id_token=oauth.get('id_token', None) ) # Set up client http = creds.authorize(httplib2.Http()) gmail = build('gmail', 'v1', http=http) # Get seed language tokens # Go through each token, seed a search to find threads we want to # search through thread_ids_per_token = {} for token in self.tokens: res = gmail.users().messages().list( userId='me', q='in:sent {0}'.format(token)).execute() thread_ids_per_token[token] = self.paginate_messages( gmail, res, max_results=self.num_threads ) equalize_dict(thread_ids_per_token, self.num_threads) threads = set(thread for threads in thread_ids_per_token.values() for thread in threads) data = { "text" : [], "snippets" : [], "people" : [] } for i, thread in enumerate(threads): email, snippet = self.get_beg_thread(gmail, thread) body = None if email['body'] is not None: body = email['body'] people = self.get_recipient(email) data['text'].append(body) data['snippets'].append(snippet) data['people'].append(people) return data
def scrape(self, user_data): """ Scrapes text and contacts from user gmail account https://developers.google.com/gmail/api/v1/reference/ """ try: oauth = user_data.services['google'] except KeyError: return False if 'denied' in oauth: return False creds = client.OAuth2Credentials( access_token=oauth['access_token'], client_id=CONFIG.get('google_client_id'), client_secret=CONFIG.get('google_client_secret'), refresh_token=oauth.get('refresh_token', None), token_uri=client.GOOGLE_TOKEN_URI, token_expiry=oauth.get('expires_in', None), user_agent='QS-server-agent/1.0', id_token=oauth.get('id_token', None)) # Set up client http = creds.authorize(httplib2.Http()) gmail = build('gmail', 'v1', http=http) # Get seed language tokens # Go through each token, seed a search to find threads we want to # search through thread_ids_per_token = {} for token in self.tokens: res = gmail.users().messages().list( userId='me', q='in:sent {0}'.format(token)).execute() thread_ids_per_token[token] = self.paginate_messages( gmail, res, max_results=self.num_threads) equalize_dict(thread_ids_per_token, self.num_threads) threads = set(thread for threads in thread_ids_per_token.values() for thread in threads) data = {"text": [], "snippets": [], "people": []} for i, thread in enumerate(threads): email, snippet = self.get_beg_thread(gmail, thread) body = None if email['body'] is not None: body = email['body'] people = self.get_recipient(email) data['text'].append(body) data['snippets'].append(snippet) data['people'].append(people) return data
def get(self): showid = self.get_argument('showtime_id') shares = self.get_arguments('share') passphrase = self.get_argument('passphrase', None) ticket_api = CONFIG.get('ticket_api') # we could also just pass the raw arguments, but this is more explicit # and 'self documenting' params = [('showtime_id', showid)] if passphrase: params += [('passphrase', passphrase)] elif shares: params += [('share', s) for s in shares] url = url_concat(ticket_api + '/api/showtimes/keys', params) show_data_raw = yield httpclient.fetch(url) if show_data_raw.code != 200: return self.error(show_data_raw.code, show_data_raw.body) show_data = json.loads(show_data_raw.body) show_date = show_data['data']['date'] rfidb = yield RFIDB.get_global() result = yield rfidb.unlock_show(showid, show_date, show_data['data']['users']) return self.api_response(result)
def get(self): date_raw = self.get_argument('date') available_tickets = int(self.get_argument('tickets', 40)) timezone = tz.gettz(CONFIG.get('timezone')) date = date_parser.parse(date_raw).replace(tzinfo=timezone) showid = yield create_showtime(date, available_tickets) return self.api_response({'showid': showid})
def require_basic_auth(handler, kwargs): auth_header = handler.request.headers.get('Authorization') if auth_header is None or not auth_header.startswith('Basic '): handler.set_status(401) handler.set_header('WWW-Authenticate', 'Basic realm=Restricted') handler._transforms = [] handler.finish() return False auth_decoded = base64.b64decode(auth_header[6:]).decode('utf-8') user, password = auth_decoded.split(':', 2) hashed_password = CONFIG.get('admin_user_pass').encode('utf-8') encoded_password = password.encode('utf-8') if (user == CONFIG.get('admin_user_id') and bcrypt.hashpw( encoded_password, hashed_password) == hashed_password): return True handler.set_status(401) handler._transforms = [] handler.finish() return False
def require_basic_auth(handler, kwargs): auth_header = handler.request.headers.get('Authorization') if auth_header is None or not auth_header.startswith('Basic '): handler.set_status(401) handler.set_header('WWW-Authenticate', 'Basic realm=Restricted') handler._transforms = [] handler.finish() return False auth_decoded = base64.b64decode(auth_header[6:]).decode('utf-8') user, password = auth_decoded.split(':', 2) hashed_password = CONFIG.get('admin_user_pass').encode('utf-8') encoded_password = password.encode('utf-8') if (user == CONFIG.get('admin_user_id') and bcrypt.hashpw(encoded_password, hashed_password) == hashed_password): return True handler.set_status(401) handler._transforms = [] handler.finish() return False
def get(self): showid = self.get_argument('showtime_id') shares = self.get_arguments('share') passphrase = self.get_argument('passphrase', None) reprocess = bool(self.get_argument('reprocess', None) is not None) ticket_api = CONFIG.get('ticket_api') # we could also just pass the raw arguments, but this is more explicit # and 'self documenting' params = [('showtime_id', showid)] if passphrase: params += [('passphrase', passphrase)] elif shares: params += [('share', s) for s in shares] url = url_concat(ticket_api + '/api/showtimes/access_tokens', params) show_data_raw = yield httpclient.fetch(url, request_timeout=180) if show_data_raw.code != 200: return self.error(show_data_raw.code, show_data_raw.body) exhibitperms = yield ExhibitPermissions.get_global() show_data = json.loads(show_data_raw.body) show_date = show_data['data']['date'] users_added = [] for user_data in show_data['data']['users']: userid = user_data.pop('id') perms = yield exhibitperms.get_permissions(userid) if perms and not reprocess: users_added.append({'userid': userid, 'permissions': perms, 'process': False}) publickey = user_data['publickey'] privatekey = user_data.get('privatekey') meta = user_data.get('meta') or {} meta.update({'showid': showid, 'permissions': perms, 'showdate': show_date}) user = User(userid, publickey, services=user_data['services'], privatekey_pem=privatekey, meta=meta) users_added.append({'userid': userid, 'process': True}) ioloop.IOLoop.current().add_callback(partial(userprocess, user)) return self.api_response(users_added)
def get(self): # inefficient databse query, figure out how to perform this query in # db. # remove all expired tickets yield remove_expired_tickets() showtimes = yield get_showtimes() reservations = yield get_reservations() showtime_map = {} for reservation in reservations: ticket_id = reservation["showtime_id"] if ticket_id in showtime_map: showtime_map[ticket_id] += 1 else: showtime_map[ticket_id] = 1 result = [] timezone = tz.gettz(CONFIG.get('timezone')) timeformat = "%A %d, %B - %I:%M%p" for showtime in showtimes: showid = showtime["id"] dateString = showtime["date"]. \ astimezone(timezone). \ strftime(timeformat) if showid in showtime_map: available_tickets = \ showtime["max_booking"] - showtime_map[showid] if available_tickets < 0: available_tickets = 0 else: available_tickets = showtime["max_booking"] result.append({ "id": showid, "date": dateString, "available_tickets": available_tickets, }) self.api_response(result)
from lib.eta_callback_receiver import ETaCallbackReceiverInstaller from lib.eta_callback_filter import ETaCallbackFilterInstaller from lib.eta_callback_sender import ETaCallbackSenderInstaller from lib.eta_callback_filter_app import ETaCallbackFilterAppInstaller from lib.eta_callback_sender_app import ETaCallbackSenderAppInstaller from lib.ephi import EPHIInstaller from lib.epcilon import EpcILoNInstaller from lib.ons import ONSConfigurer from lib.alfa import ALfAInstaller from lib.omega import OMeGaInstaller from lib.ypsilon import YPSilonInstaller from lib.lambda_iota import LaMBDaInstaller if __name__ == "__main__": if "--accept-defaults" in sys.argv: CONFIG.set("global", "accept_defaults", "true") lib.utils.putWarning("Installing with defaults values from `resources/install.ini") else: CONFIG.set("global", "accept_defaults", "false") lib.utils.putTitle(" IoTa Installer") GlobalConfigurer().run() CertConfigurer().run() TomcatInstaller().run() DBConfigurer().run() SigMaCertConfigurer().run() SigMAInstaller().run() EpcisInstaller().run() LDAPConfigurer().run() YPSilonInstaller().run() ETaInstaller().run() EPHIInstaller().run()
def scrape(self, user_data): try: tokens = user_data.services['reddit'] except KeyError: return False if 'denied' in tokens: return False client_id = CONFIG.get("reddit_client_id") client_secret = CONFIG.get("reddit_client_secret") r = praw.Reddit(user_agent="QS-agent/1.0") r.set_oauth_app_info( client_id=client_id, client_secret=client_secret, redirect_uri="http://iamadatapoint.com:8085/auth/reddit") tokens = r.refresh_access_information( refresh_token=tokens['refresh_token']) r.set_access_credentials(**tokens) data = {} subs = [] count = 0 for i in r.get_my_subreddits(): if count > self.num_scrape: break load = {} load['name'] = i.display_name load['url'] = i.url subs.append(load) count += 1 data['subs'] = subs msgs = [] count = 0 for i in r.get_inbox(): if count > self.num_scrape: break load = {} load['body'] = i.body if i.author: load['author'] = i.author.name load['comment'] = i.was_comment msgs.append(load) count += 1 data['text'] = msgs me = r.get_me() # May want to grab own profile name karma_c = me.comment_karma karma_l = me.link_karma submissions = [] count = 0 for i in me.get_submitted(): if count > self.num_scrape: break load = {} load['title'] = i.title load['url'] = i.url load['score'] = i.score load['subreddit'] = i.subreddit.title submissions.append(load) count += 1 data['submissions'] = submissions data['comment_karma'] = karma_c data['link_karma'] = karma_l likes = [] count = 0 for i in me.get_upvoted(): if count > self.num_scrape: break load = {} load['title'] = i.title load['url'] = i.url likes.append(load) count += 1 data['likes'] = likes dislikes = [] count = 0 for i in me.get_downvoted(): if count > self.num_scrape: break load = {} load['title'] = i.title load['url'] = i.url dislikes.append(load) count += 1 data['dislikes'] = dislikes return data
def scrape(self, user_data): try: oauth = user_data.services['google'] except KeyError: return False if 'denied' in oauth: return False credentials = client.OAuth2Credentials( access_token=oauth['access_token'], client_id=CONFIG.get('google_client_id'), client_secret=CONFIG.get('google_client_secret'), refresh_token=oauth.get('refresh_token', None), token_uri=client.GOOGLE_TOKEN_URI, token_expiry=oauth.get('expires_in', None), user_agent='QS-server-agent/1.0', id_token=oauth.get('id_token', None) ) http = credentials.authorize(httplib2.Http()) try: youtube = build('youtube', 'v3', http=http) except: return False userinfo = list(apiclient_paginate(youtube.channels(), 'list', { 'part': 'statistics,contentDetails', 'mine': True, }))[0] special_videos = {} user_special_playlists = userinfo['contentDetails']['relatedPlaylists'] for name, playlistid in user_special_playlists.items(): try: videos = list( apiclient_paginate( youtube.playlistItems(), 'list', { 'part': 'snippet', 'playlistId': playlistid, }, max_results=self.num_videos_per_playlist ) ) special_videos[name] = videos except Exception: self.logger.exception("Error getting special playlists") continue playlists = list(apiclient_paginate(youtube.playlists(), 'list', { 'part': 'id,snippet', 'mine': True, })) subscriptions = list(apiclient_paginate( youtube.subscriptions(), 'list', { 'part': 'snippet', 'mine': True, }) ) result = { 'subscriptions': subscriptions, 'playlists': playlists, 'userinfo': userinfo, 'special_videos': special_videos, } return result
def scrape(self, user_data): try: oauth = user_data.services['google'] except KeyError: return False if 'denied' in oauth: return False credentials = client.OAuth2Credentials( access_token=oauth['access_token'], client_id=CONFIG.get('google_client_id'), client_secret=CONFIG.get('google_client_secret'), refresh_token=oauth.get('refresh_token', None), token_uri=client.GOOGLE_TOKEN_URI, token_expiry=oauth.get('expires_in', None), user_agent='QS-server-agent/1.0', id_token=oauth.get('id_token', None)) http = credentials.authorize(httplib2.Http()) try: youtube = build('youtube', 'v3', http=http) except: return False userinfo = list( apiclient_paginate(youtube.channels(), 'list', { 'part': 'statistics,contentDetails', 'mine': True, }))[0] special_videos = {} user_special_playlists = userinfo['contentDetails']['relatedPlaylists'] for name, playlistid in user_special_playlists.items(): try: videos = list( apiclient_paginate( youtube.playlistItems(), 'list', { 'part': 'snippet', 'playlistId': playlistid, }, max_results=self.num_videos_per_playlist)) special_videos[name] = videos except Exception: self.logger.exception("Error getting special playlists") continue playlists = list( apiclient_paginate(youtube.playlists(), 'list', { 'part': 'id,snippet', 'mine': True, })) subscriptions = list( apiclient_paginate(youtube.subscriptions(), 'list', { 'part': 'snippet', 'mine': True, })) result = { 'subscriptions': subscriptions, 'playlists': playlists, 'userinfo': userinfo, 'special_videos': special_videos, } return result
def scrape(self, user_data): try: access = user_data.services['tumblr']['access_token'] secret = user_data.services['tumblr']['access_token_secret'] except KeyError: return False if 'denied' in access: return False consumer_key = CONFIG.get("tumblr_client_id") consumer_secret = CONFIG.get("tumblr_client_secret") client = pytumblr.TumblrRestClient(consumer_key, consumer_secret, access, secret) tumblr_data = { "hosted_blogs": [], "following": [], "likes": [], "suggestions": [] } profile = client.info() total_likes = profile['user']['likes'] total_following = profile['user']['following'] # Should we be doing anything in case we get garbage/shell accounts? # Write this comment in GitHub if total_likes == 0 and total_following == 0: return "Empty Account" hosts = [] for i in profile['user']['blogs']: blog = {} blog['total_posts'] = i['total_posts'] blog['followers'] = i['followers'] blog['description'] = i['description'] blog['url'] = i['url'] blog['name'] = i['name'] blog['title'] = i['title'] hosts.append(blog) tumblr_data['hosted_blogs'] = hosts dash = client.dashboard() dash_data = [] for data in dash['posts']: goods = {} goods['source_title'] = data.get('source_title', None) goods['post_url'] = data.get('post_url', None) goods['date'] = data.get('date', None) goods['summary'] = data.get('summary', None) goods['photos'] = data.get('photos', None) goods['blog_name'] = data.get('blog_name', None) goods['text'] = data.get('text', None) goods['caption'] = data.get('caption', None) goods['source_url'] = data.get('source_url', None) goods['content'] = data.get('content', None) dash_data.append(goods) tumblr_data['suggestions'] = dash_data like_data = [] likes = client.likes() total = likes['liked_count'] like_data = likes['liked_posts'] if total > 20: full_set = yield self.paginate_likes(client, total) like_data.append(full_set) if len(like_data) > 0: tumblr_data['likes'] = self.clean_likes(like_data) follows = client.following() tumblr_data['follows'] = follows return tumblr_data