def get_or_create_content_from_request_async(prefix='content', get_if_needed=False, **kwargs): if prefix and not prefix.endswith('_'): prefix = prefix + '_' elif not prefix: prefix = '' content_id = flask_extras.get_parameter(prefix + 'id') if content_id: try: kwargs.setdefault('content_id', int(content_id)) except: raise errors.InvalidArgument('Invalid %sid value' % (prefix, )) kwargs.setdefault( 'creator', flask_extras.get_parameter(prefix + 'creator_identifier')) kwargs.setdefault('url', flask_extras.get_parameter(prefix + 'url')) duration = flask_extras.get_parameter(prefix + 'duration') if duration: try: kwargs.setdefault('duration', int(duration)) except: raise errors.InvalidArgument('Invalid %sduration value' % (prefix, )) kwargs.setdefault('thumb_url', flask_extras.get_parameter(prefix + 'thumb_url')) kwargs.setdefault('title', flask_extras.get_parameter(prefix + 'title')) kwargs.setdefault('video_url', flask_extras.get_parameter(prefix + 'video_url')) key, content = yield get_or_create_content_async(**kwargs) if key and get_if_needed and not content: content = yield key.get_async() raise ndb.Return((key, content))
def auth_async(code): try: result = yield _fetch_async( 'https://www.googleapis.com/oauth2/v4/token', method='POST', payload=urllib.urlencode({ 'client_id': '_REMOVED_', 'code': code, 'grant_type': 'authorization_code', 'redirect_uri': 'cam.reaction.ReactionCam:/youtube_oauth2redirect', }), follow_redirects=False, deadline=10) data = json.loads(result.content) except Exception as e: logging.exception('YouTube token exchange failed.') raise errors.ServerError() if result.status_code != 200: logging.debug('Could not exchange code: %r', data) raise errors.InvalidArgument('Failed to exchange code for token') try: id_token = data.pop('id_token') id_hdr, id_bdy, id_sig = (p + ('=' * (-len(p) % 4)) for p in id_token.split('.')) profile = json.loads(base64.b64decode(id_bdy)) assert 'sub' in profile except: logging.debug('Could not extract profile data: %r', data) raise errors.InvalidArgument('Missing profile in token response') raise ndb.Return((data, profile))
def get_video_async(video_id, snippet=True, statistics=False): if not (snippet or statistics): raise errors.InvalidArgument( 'Specify at least one of snippet, statistics') parts = [] if snippet: parts.append('snippet') if statistics: parts.append('statistics') try: qs = urllib.urlencode({ 'id': video_id, 'key': config.YOUTUBE_API_KEY, 'part': ','.join(parts), }) result = yield _fetch_async( 'https://www.googleapis.com/youtube/v3/videos?%s' % (qs, ), follow_redirects=False, deadline=10) data = json.loads(result.content) except Exception as e: logging.exception('YouTube call failed.') raise errors.ServerError() if result.status_code == 404: raise errors.ResourceNotFound('That video does not exist') elif result.status_code != 200: logging.debug('Could not get YouTube video (%d): %r', result.status_code, data) raise errors.InvalidArgument('Invalid video id') if data['pageInfo']['totalResults'] == 0: raise ndb.Return(None) if not data['items']: logging.warning('Expected at least one item in data: %r', data) raise ndb.Return(None) raise ndb.Return(data['items'][0])
def get(self, others, all_chunks=False, create=False, disable_autojoin=False, reason='unknown', solo=False, title=None, **kwargs): try: accounts = models.Account.resolve_list([self.account] + others) if len(accounts) == 2 and accounts[0].key == accounts[1].key: # The user specified themselves as the second user so this should be solo. accounts = accounts[:1] solo = True if create: for a in accounts: if a.key in self.account.blocked_by: logging.info('Blocked stream creation by %s (with %s)', self.account.key, a.key) raise errors.InvalidArgument('Could not get stream') assert 'owners' not in kwargs kwargs['owners'] = { a.key: self.account.key for a in accounts if a.is_bot } stream, new = models.Stream.get_or_create(title, accounts, solo=solo, **kwargs) else: stream, new = models.Stream.get(accounts, solo=solo, title=title), False except ValueError: raise errors.InvalidArgument('Got one or more invalid account(s)') if not stream: return None handler = MutableStream(self.account, stream, account_map={a.key: a for a in accounts}, disable_autojoin=disable_autojoin) if new: # Notify all participants that a new stream including them was created. handler.notify(notifs.ON_STREAM_NEW, add_stream=True, reason=reason) handler._report('created') else: # The stream already existed - update properties. image = kwargs.get('image') if image: handler.set_image(image) if not new and all_chunks: handler.load_all_chunks() return handler
def post_action(): data = json.loads(request.form['payload']) if data['token'] != 'lwitiqXtcufDL1vXiMj3f36w': raise errors.InvalidArgument('Invalid data') if data['callback_id'] != 'review_content': raise errors.InvalidArgument('Unsupported callback_id') try: attachment = data['original_message']['attachments'][0] except: logging.exception('Missing attachment') raise errors.InvalidArgument('Missing attachment') try: action = data['actions'][0] assert action['name'] == 'quality' assert action['type'] == 'button' pieces = action['value'].split(':') account_id, content_id, quality = pieces account_id = int(account_id) content_id = int(content_id) if quality == 'hide': return {'text': ''} quality = int(quality) except: logging.exception('Error in action logic') raise errors.InvalidArgument('Unsupported action') if quality == 0: quality_label = u'1️⃣' elif quality == 1: quality_label = u'2️⃣' elif quality == 2: quality_label = u'3️⃣' elif quality == 3: quality_label = u'4️⃣' elif quality == 4: quality_label = u'🤩' else: quality_label = u'❓' title = attachment['title'] username = data['user']['name'] account_future = models.Account.get_by_id_async(account_id) taskqueue.add(url='/_ah/jobs/set_quality', params={ 'account_id': account_id, 'quality': quality }, queue_name=config.INTERNAL_QUEUE) account = account_future.get_result() return { 'text': u'%s rated %s: %s' % (username, slack_api.admin(account), quality_label), }
def remove_chunk(self, chunk_id): # TODO: Only allow sender to remove their own chunk? try: chunk_id = int(chunk_id) except (TypeError, ValueError): raise errors.InvalidArgument('Invalid chunk id') self._tx(models.Stream.remove_chunk, chunk_id)
def set_played_until(self, timestamp, report=True): # Convert the timestamp if necessary. if not isinstance(timestamp, datetime): try: timestamp = convert.from_unix_timestamp_ms(int(timestamp)) except (TypeError, ValueError): raise errors.InvalidArgument('Invalid timestamp') if abs(timestamp - self.last_chunk_end) < timedelta(seconds=1): # Ignore sub-second differences between "played until" and end of stream. timestamp = self.last_chunk_end old_played_until = self.played_until was_unplayed = not self.is_played self._tx(models.Stream.set_played_until, self.participant.account, timestamp) if self.played_until != old_played_until: for chunk in self.chunks: if chunk.end <= old_played_until: continue sender_stream = self.for_participant(chunk.sender) sender_stream.notify_first_play(chunk, player=self.account) self.notify_change(notifs.ON_STREAM_PLAY, old_played_until=old_played_until, played_until=self.played_until) if not report: return # Report the user updated their played state and seconds played. duration = sum((min(c.end, timestamp) - max(c.start, old_played_until) for c in self.chunks if c.sender != self.account.key and c.end > old_played_until and c.start < timestamp), timedelta()) self._report('played', duration=duration.total_seconds(), unplayed=was_unplayed)
def change_identifier(self, old, new, notify_connect=True, primary=False): new, identifier_type = identifiers.parse(new) if not new: logging.warning('%r is invalid', new) raise errors.InvalidArgument('That identifier is not valid') if old not in self.identifiers: raise errors.ForbiddenAction('That identifier belongs to another account') if old == new: return # Get the service, team, resource from the new identifier. try: service, team, resource = identifiers.parse_service(new) new_team = not self.is_on_team(service, team) except: service, team, resource = (None, None, None) new_team = True identity, account = models.Identity.change( old, new, assert_account_key=self.account.key, primary=primary) if not identity: raise errors.AlreadyExists('That identifier is already in use') # Update in-memory instance to reflect reality. if account: self.account.populate(**account.to_dict()) if self.account.is_activated and service == 'email' and new_team: # Connect the email "service" (if the user is not already on this domain). self.connect_service(service, team, resource, notify=notify_connect) # TODO: We should also disconnect service if the old identifier was a service. self._notify_account_change()
def join_service_content(self, service_content_id, autocreate=True, **kwargs): # Variable name should stay "service_content_id" to prevent duplicate in kwargs. try: service, team, resource = identifiers.parse_service( service_content_id) content_id = identifiers.build_service(service, team, resource) except: raise errors.InvalidArgument('Invalid service identifier') q = models.Stream.query(models.Stream.service_content_id == content_id) stream = q.get() if not stream: if not autocreate: raise errors.ResourceNotFound('Cannot join that stream') handler = self.get_or_create([], service_content_id=content_id, **kwargs) logging.debug('Created stream for %s (%r)', content_id, handler.title) return handler handler = Stream(stream).join(self.account, do_not_bump=True) if 'title' in kwargs and handler.title != kwargs['title']: handler.set_title(kwargs['title']) if not handler.visible: handler.show() logging.debug('Joined stream for %s (%r)', content_id, handler.title) members = kwargs.get('service_members') if members and set(handler.service_members) != set(members): handler._stream.service_members = members handler._stream.put() logging.debug('Updated member list of %s (%r)', content_id, handler.title) return handler
def __init__(self, client, identifier): self.client = client self.identifier = identifiers.clean(identifier) if not self.identifier: logging.warning('Cleaning %r resulted in empty value', identifier) raise errors.InvalidArgument('That identifier is not valid') # Team will be set to preferred team after challenge is complete. self.team = None
def post_transcode_complete(): try: content_id = int(flask_extras.get_parameter('content_id')) except: raise errors.InvalidArgument('Invalid content_id parameter') stream_url = flask_extras.get_parameter('stream_url') if not stream_url: raise errors.InvalidArgument('Invalid stream_url parameter') content = models.Content.get_by_id(content_id) if not content: raise errors.ResourceNotFound('Content not found') if not content.metadata: content.metadata = {} content.metadata['raw_video_url'] = content.video_url content.video_url = stream_url content.put() return {'success': True}
def get_or_create_async(self, identifier_list): try: account_keys = yield models.Account.resolve_keys_async( identifier_list) except: raise errors.InvalidArgument('Got one or more invalid identifiers') account_keys.add(self.account_key) thread = yield models.Thread.lookup_async(account_keys) raise ndb.Return(ThreadWithAccount(self.account_key, thread))
def grant_type_authorization_code(client): code = flask_extras.get_parameter('code') if not code: raise errors.MissingArgument('A code is required') redirect_uri = flask_extras.get_parameter('redirect_uri') if not redirect_uri: redirect_uri = None if redirect_uri and redirect_uri not in client.redirect_uris: raise errors.InvalidArgument('Invalid redirect_uri value') session = auth.Session.from_auth_code(code, client.key.id(), redirect_uri) return accounts.get_handler(session.account)
def block(self, identifier): blocked_account = models.Account.resolve_key(identifier) if blocked_account == self.account.key: raise errors.InvalidArgument('You cannot block yourself') models.Account.add_block(blocked_account, self.account.key) f1 = models.AccountFollow.unfollow_async(self.account.key, blocked_account) f2 = models.AccountFollow.unfollow_async(blocked_account, self.account.key) stream = self.streams.get([blocked_account]) if stream: stream.hide() ndb.Future.wait_all([f1, f2])
def set_username(self, new_username): new_username, identifier_type = identifiers.parse(new_username) if identifier_type != identifiers.USERNAME: # A user may not use this endpoint to add a phone number/e-mail. raise errors.InvalidArgument('A valid username must be provided') # Switch out the old username if it exists, otherwise just add the new one. old_username = self.username if old_username and self.account.primary_set: self.change_identifier(old_username, new_username, primary=True) else: self.add_identifier(new_username, primary=True)
def _upload(original_filename, data, persist): if not data: raise errors.InvalidArgument('Empty data') if persist: upload_func = _s3_upload else: upload_func = _gcs_upload_shortlived # Create a unique filename from the data and file extension. _, extension = os.path.splitext(original_filename) data_sha256 = hashlib.sha256(data).hexdigest() mime_type, _ = mimetypes.guess_type(original_filename) return upload_func(data, data_sha256, extension.lower(), mime_type)
def _media_service(service, fields={}, persist=False, **kwargs): kwargs['bucket'] = config.BUCKET_PERSISTENT if persist else config.BUCKET_SHORTLIVED url = '{}?{}'.format(service, urllib.urlencode(kwargs)) logging.debug('POST %s', url) result = http.request('POST', url, fields=fields, timeout=60) if result.status != 200: logging.warning('HTTP %d: %s', result.status, result.data) if result.status == 404: raise errors.InvalidArgument('The provided URL could not be loaded') raise errors.ExternalError('Failed to process media') info = json.loads(result.data.decode('utf-8')) path = '/' + info['bucket_path'] return path, info['duration']
def _validate_status_transition(old_status, new_status): """Validates that a status may change from a certain value to another.""" can_change = False if old_status is not None else True for tier in config.VALID_STATUS_TRANSITIONS: if old_status in tier: can_change = True if new_status in tier: if not can_change: raise errors.ForbiddenAction('Cannot change status from "%s" to "%s"' % ( old_status, new_status)) break else: raise errors.InvalidArgument('Invalid status')
def get_by_id(self, stream_id, all_chunks=False, **kwargs): try: if all_chunks: stream, chunks = models.Stream.get_by_id_with_chunks( int(stream_id)) else: stream = models.Stream.get_by_id(int(stream_id)) chunks = None except (TypeError, ValueError): raise errors.InvalidArgument('Invalid stream id') if not stream: raise errors.ResourceNotFound('That stream does not exist') return MutableStream(self.account, stream, chunks=chunks, **kwargs)
def validate(self, secret): result = models.Challenge.validate(self.client, self.identifier, secret) if result == models.Challenge.SUCCESS: return True elif result == models.Challenge.INVALID_SECRET: raise errors.InvalidArgument('An invalid secret was provided') elif result == models.Challenge.TOO_MANY_ATTEMPTS: raise errors.ResourceNotFound('Too many attempts') elif result == models.Challenge.EXPIRED: raise errors.ResourceNotFound('Challenge has expired') # Unexpected result. raise errors.ServerError()
def get_or_create_account_key(self, create_status='temporary', origin_account=None): # Identifier types that may have an account created. creatable_types = (identifiers.EMAIL, identifiers.PHONE, identifiers.SERVICE_ID) # Try to match the destination to an existing account. best_route = None for route in self.routes: key = models.Account.resolve_key(route.value) if key: return key if route.type in creatable_types and not best_route: best_route = route # Account not found, create one based on first usable contact detail. # TODO: This should check properly if route is externally verifiable (e.g., SMS). if not best_route: logging.warning('Failed to create an account for one of %s', self.routes) return None identifier = best_route.value # Locally verified accounts can be created immediately. if best_route.type != identifiers.SERVICE_ID: handler = create(identifier, status=create_status) return handler.account.key # Verify that this user is on the same service/team as the origin account. if not origin_account: raise errors.InvalidArgument('Cannot use third-party accounts') service_key, team_key, resource = models.Service.parse_identifier(identifier) if not origin_account.is_on_team(service_key, team_key): raise errors.InvalidArgument('Invalid third-party account') # Look up the third-party account. # TODO: Support multiple types of services dynamically. if service_key.id() != 'slack': raise errors.NotSupported('Only Slack accounts are supported') auth = origin_account.get_auth_key(service_key, team_key).get() # TODO: Put this API call elsewhere! info = slack_api.get_user_info(resource, auth.access_token) ids = [identifier, info['user']['profile']['email']] handler = get_or_create( *ids, display_name=info['user']['real_name'] or info['user']['name'], image=info['user']['profile'].get('image_original'), status=create_status) return handler.account.key
def itunes(receipt_data, url='https://buy.itunes.apple.com/verifyReceipt'): try: result = urlfetch.fetch(url=url, method=urlfetch.POST, payload=json.dumps( {'receipt-data': receipt_data}), deadline=30) data = json.loads(result.content) except Exception: logging.exception('Could not get result from Apple payment server.') raise errors.ServerError() if result.status_code != 200: logging.error('Apple payment server HTTP %d: %r', result.status_code, data) raise errors.InvalidArgument( 'Failed to validate receipt data with Apple') status = data.get('status') if not isinstance(status, (int, long)): logging.error('Could not get status: %r', data) raise errors.ServerError() if status == 0: return data elif status in (21000, 21002, 21003): raise errors.InvalidArgument('Invalid receipt data provided') elif status == 21005: raise errors.ExternalError() elif status == 21007: return itunes(receipt_data, url='https://sandbox.itunes.apple.com/verifyReceipt') elif status == 21008: return itunes(receipt_data) elif status == 21010: raise errors.InvalidArgument('Invalid purchase') elif 21100 <= status <= 21199: logging.error('Internal data access error: %r', data) raise errors.InvalidArgument('Invalid receipt data provided') logging.error('Unsupported status: %r', data) raise errors.NotSupported()
def get_challenger(client, identifier, call=False): """ Returns a handler for creating a challenge for the given identifier. """ identifier, identifier_type = identifiers.parse(identifier) if identifier in config.DEMO_ACCOUNTS: # Demo accounts. return DummyChallenger(client, identifier) if identifier_type == identifiers.PHONE and call: return CallChallenger(client, identifier) challenger = _challengers.get(identifier_type) if not challenger: logging.warning('Failed to get a challenger for %r', identifier) raise errors.InvalidArgument('That identifier is not valid') return challenger(client, identifier)
def set_display_name(self, display_name): if not isinstance(display_name, basestring): raise TypeError('Display name must be a string') # TODO: Validate display name more. display_name = display_name.strip() if not display_name: raise errors.InvalidArgument('Invalid display name') if display_name == self.account.display_name: return self.account.display_name = display_name self.account.put() if not self.account.primary_set: base = identifiers.clean_username(self.account.display_name) if base: logging.debug('User has no username, autosetting one') self.generate_username(base) self._notify_account_change()
def _s3_upload(data, data_sha256, extension, mime_type): filename = data_sha256 + extension try: result = _aws4_signed_fetch(region='us-east-1', service='s3', headers={'x-amz-acl': 'public-read', 'x-amz-content-sha256': data_sha256, 'Content-Type': mime_type}, method='PUT', host='s3.amazonaws.com', path='/%s/%s' % (config.S3_BUCKET, filename), payload=data, payload_sha256=data_sha256) except: logging.exception('Could not upload to S3.') raise errors.ServerError() if result.status_code != 200: logging.debug('Could not upload file: %r', result.content) raise errors.InvalidArgument('Failed to upload file') return config.S3_BUCKET_CDN + filename
def set_seen_until(self, seen_until): t = self._thread.key.get() for m in t.messages: if m.message_id == seen_until: break else: # TODO: Have another look at this logic. raise errors.InvalidArgument( 'Message id must be one of 10 most recent') for a in t.accounts: if a.account == self.account_key: break else: raise errors.ForbiddenAction('Cannot update that thread') if a.seen_until and a.seen_until.id() >= seen_until: return a.seen_until = m.key_with_parent(self.key) t.put() self._thread = t
def create_auth_code(self, client_id, redirect_uri=None): secret = random.base62(config.AUTH_CODE_LENGTH) expires = datetime.utcnow() + config.AUTH_CODE_TTL client = services.get_client(client_id) if redirect_uri: if redirect_uri not in client.redirect_uris: raise errors.InvalidArgument('Invalid redirect_uri value') elif len(client.redirect_uris) == 1: # Figure out the redirect URI from the client. redirect_uri = client.redirect_uris[0] if not redirect_uri: # Ensure that it's not the empty string. redirect_uri = None code = models.AuthCode(id=secret, account=self.account.key, expires=expires, client_id=client_id, redirect_uri=redirect_uri) code.put() return code
def post_chunk_played(): # Only internal websites can use this endpoint. try: payload = security.decrypt(config.WEB_ENCRYPTION_KEY, flask_extras.get_parameter('payload'), block_segments=True) data = json.loads(payload) fingerprint = data['fingerprint'] stream_key = ndb.Key('Stream', data['stream_id']) chunk_key = ndb.Key('Chunk', data['chunk_id'], parent=stream_key) except: raise errors.InvalidArgument('Invalid payload') cache_key = 'external_plays:%d:%d:%s' % (stream_key.id(), chunk_key.id(), fingerprint) if memcache.get(cache_key): logging.debug( 'Repeat chunk play for fingerprint %s (stream %d chunk %d)', fingerprint, stream_key.id(), chunk_key.id()) return {'success': True} memcache.set(cache_key, True, 172800) stream, chunk = ndb.get_multi([stream_key, chunk_key]) if not stream or not chunk: raise errors.ResourceNotFound('That chunk does not exist') chunk.external_plays += 1 chunk.put() for local_chunk in stream.chunks: if local_chunk.chunk_id == chunk.key.id(): local_chunk.external_plays = chunk.external_plays stream.put() break logging.debug('New chunk play for fingerprint %s (stream %d chunk %d)', fingerprint, stream_key.id(), chunk_key.id()) logging.debug('Total external chunk plays is now %d', chunk.external_plays) handler = streams.MutableStream(chunk.sender, stream) if chunk.external_plays == 1: handler.notify_first_play(chunk) handler.notify(notifs.ON_STREAM_CHUNK_EXTERNAL_PLAY, add_stream=True, chunk=chunk) return {'success': True}
def upload_and_send(stream): extras = flask_extras.get_flag_dict('allow_duplicate', 'export', 'mute_notification', 'persist') # Export everything by default. extras.setdefault('export', True) persist = extras.get('persist', False) show_in_recents = flask_extras.get_flag('show_in_recents') if show_in_recents is not None: extras['show_for_sender'] = show_in_recents # Timed transcriptions. text_segments = flask_extras.get_parameter('text_segments') if text_segments: extras['text_segments'] = [ models.TextSegment(**s) for s in json.loads(text_segments) ] # Payload can either be a file + duration or a URL to download. if g.api_version >= 29: payload = request.files.get('payload') url = flask_extras.get_parameter('url') else: payload = request.files.get('audio') url = flask_extras.get_parameter('audio_url') if payload: try: duration = int(flask_extras.get_parameter('duration')) except (TypeError, ValueError): raise errors.InvalidArgument( 'Duration should be milliseconds as an int') path = files.upload(payload.filename, payload.stream, persist=persist) elif url: if url.startswith(config.STORAGE_URL_HOST): path = '/' + url[len(config.STORAGE_URL_HOST):] try: duration = int(flask_extras.get_parameter('duration')) except (TypeError, ValueError): raise errors.InvalidArgument( 'Duration should be milliseconds as an int') else: raise errors.InvalidArgument('Cannot use that URL') else: return False # Upload file attachments. extras['attachments'] = [] attachments = request.files.getlist('attachment') for attachment in attachments: p = files.upload(attachment.filename, attachment.stream, persist=persist) extras['attachments'].append( models.ChunkAttachment(title=attachment.filename, url=files.storage_url(p))) # Add URL attachments. attachment_titles = flask_extras.get_parameter_list('attachment_title') attachment_urls = flask_extras.get_parameter_list('attachment_url') try: # Validate URLs. parsed_urls = map(urlparse.urlparse, attachment_urls) except: raise errors.InvalidArgument('Invalid attachment URL provided') if not attachment_titles: attachment_titles = [u''] * len(attachment_urls) for i, title in enumerate(attachment_titles): if title: continue # Default empty titles to URL hostname. attachment_titles[i] = parsed_urls[i].hostname if len(attachment_titles) != len(attachment_urls): raise errors.InvalidArgument('Attachment title/URL count mismatch') for title, url in zip(attachment_titles, attachment_urls): extras['attachments'].append( models.ChunkAttachment(title=title, url=url)) # Support creating public links to chunk. if extras.get('export', False): custom_content_id = flask_extras.get_parameter('external_content_id') if custom_content_id: # TODO: WARNING, this can be abused to take over existing shared links! if not re.match(r'[a-zA-Z0-9]{21}$', custom_content_id): raise errors.InvalidArgument( 'Custom external_content_id must be 21 base62 digits') extras['external_content_id'] = custom_content_id client_id, _ = auth.get_client_details() extras.setdefault('client_id', client_id) stream.send(path, duration, token=flask_extras.get_parameter('chunk_token'), **extras) # Ping IFTTT to fetch the new data. # TODO: Do this in some nicer way. Maybe on_new_chunk for services? if stream.service_id == 'ifttt' and stream.account.username != 'ifttt': ping_ifttt(stream.service_owner or stream.account) return True
def get_client(client_id): client = models.ServiceClient.get_by_id(client_id) if not client: raise errors.InvalidArgument('Invalid client') return Client(client)