def remove_hidden_groups(user, permission): if not user.organization or user.is_admin: return permission org = user.organization.get() not_finished = True fake_ids_mapping = {} fake_ids = [] while not_finished: for op in permission: for group_id in permission[op]['shared']: group = ndb.Key('Group', group_id) if org.is_hidden_group(group) and not user.in_group(group): if group_id in fake_ids_mapping: fake_id = fake_ids_mapping[group_id] else: fake_id = memcache.get(str(group_id) + '_fake_id') if fake_id is None: fake_id = server.create_uuid() memcache.add(key=str(group_id) + '_fake_id', value=fake_id) fake_ids_mapping[group_id] = fake_id fake_ids.append(fake_id) permission[op]['shared'][fake_id] = permission[op]['shared'][group_id] del permission[op]['shared'][group_id] break for group_id in permission[op]['required']: group = ndb.Key('Group', group_id) if org.is_hidden_group(group) and not user.in_group(group): if group_id in fake_ids_mapping: fake_id = fake_ids_mapping[group_id] else: fake_id = server.create_uuid() fake_ids_mapping[group_id] = fake_id fake_ids.append(fake_id) permission[op]['required'][fake_id] = permission[op]['required'][group_id] del permission[op]['required'][group_id] break not_finished = False if len(fake_ids) > 0: permission['hidden'] = fake_ids return permission
def get_comment_index_doc(self, comment): concept = self.concept.get() project = concept.project fields = [ ttindex.ATOMFIELD, 'typ', 'anno_reply', ttindex.ATOMFIELD, 'anno', self.key.id(), ttindex.TEXTFIELD, 'reply', comment.comment, ttindex.ATOMFIELD, 'pro', project.id(), ttindex.ATOMFIELD, 'doc', self.document.id(), ttindex.ATOMFIELD, 'con', concept.key.id(), ttindex.DATEFIELD, 'date', comment.created_ts, ] return create_uuid(), fields
def get(self): if not self.request.get('project', None): raise HttpErrorException.bad_request('invalid project id') project = Project.get_by_id(self.request.get('project')) if not project: raise HttpErrorException.bad_request('invalid project id') channel_id = server.create_uuid() client_auth_token = auth.create_custom_token(self.user.key.id()) color = self.get_previous_color(project) if not color: color = ChannelToken.generate_color() channel_token = ChannelToken( key=ChannelToken.create_key(channel_id), project=project.key, user=self.user.key, color=color, ) channel_token.set_status(True) channel_token.send_message({'channel_op': 'ping'}) channel_token.put() self.write_json_response({ 'channel_id': channel_id, 'auth_token': client_auth_token, })
def new(verified=None): verify_id = server.create_uuid() if not verified: if server.GlobalConfig.get_configs().require_email_verification: verified = False else: verified = True return EmailVerification(verify_id=verify_id, verified=verified)
def _add_perm(self): group, required = self.project.validate_add_perm_request( self.json_request, self.user) self.project.add_group_perm(group, self.json_request.get('operation'), self.json_request.get('permission'), required) if self.json_request.get('operation') == 'read': self.project.record_analytic('pro_perm', self.analytic_session) action_data = { 'group': group.key.id(), 'operation': self.json_request.get('operation'), 'permission': self.json_request.get('permission'), 'type': 'required' if required else 'shared', 'hidden': False, } trans = Transaction(action='pro_perm_add', user=self.user.key, artifact=self.project.key, project=self.project.key, action_data=action_data) trans.put() org = self.project.organization.get() # Get project channel tokens channel_tokens = ChannelToken.get_by_project_key( self.project.key, self.user_channel_token) # Loop through each channel token to test permissions on next sibling for channel_token in channel_tokens: ach_user = channel_token.user.get() if (not self.project.has_permission_read(channel_token.user.get()) and not self.project.had_permission_read( channel_token.user.get())): continue if not ach_user.is_super_admin and not \ (ach_user.is_org_admin and org.key == ach_user.organization) and \ org.is_hidden_group(group): fake_id = memcache.get(group.key.id() + '_fake_id') if fake_id is None: fake_id = server.create_uuid() memcache.add(key=group.key.id() + '_fake_id', value=fake_id) trans.action_data['group'] = fake_id trans.action_data['hidden'] = True message = { 'user': self.get_user_channel_data(), 'transaction': trans.to_dict(self.user) } ChannelToken.broadcast_message([channel_token], message)
def _remove_group(self): group = self.phrasing.validate_rm_group_request( self.json_request, self.user) for op in self.phrasing.operations_list: self.phrasing.remove_group_perm(group, op) self.phrasing.remove_group_perm(group, op, required=True) project = self.phrasing.concept.get().project.get() self.phrasing.broadcast_rm_grp(project, group, self.user, 'phr_rm_grp') action_data = { 'group': group.key.id(), 'hidden': False, } trans = Transaction(action='phr_grp_rmv', user=self.user.key, artifact=self.phrasing.key, project=project.key, action_data=action_data) trans.put() org = project.organization.get() # Get project channel tokens channel_tokens = ChannelToken.get_by_project_key( project.key, self.user_channel_token) # Loop through each channel token to test permissions on next sibling for channel_token in channel_tokens: ach_user = channel_token.user.get() if (not self.phrasing.has_permission_read(channel_token.user.get()) and not self.phrasing.had_permission_read( channel_token.user.get())): continue if not ach_user.is_super_admin and not \ (ach_user.is_org_admin and org.key == ach_user.organization) and \ org.is_hidden_group(group): fake_id = memcache.get(group.key.id() + '_fake_id') if fake_id is None: fake_id = server.create_uuid() memcache.add(key=group.key.id() + '_fake_id', value=fake_id) trans.action_data['group'] = fake_id trans.action_data['hidden'] = True message = { 'user': self.get_user_channel_data(), 'transaction': trans.to_dict(self.user) } ChannelToken.broadcast_message([channel_token], message)
def emit(self, record): try: rc_msg = record.getMessage() msg = record.__dict__.get( 'dict_msg', { 'msg_short': rc_msg[:MSG_SHORT_LEN] + '...' if len(rc_msg) > MSG_SHORT_LEN else rc_msg, 'msg': record.getMessage() }) request_user = record.__dict__.get('request_user', None) affected_user = record.__dict__.get('affected_user', None) artifact = record.__dict__.get('artifact', None) log_type = record.__dict__.get('log_type', DEFAULT) request = record.__dict__.get('request', None) org = record.__dict__.get('org', None) client_log = record.__dict__.get('client_log', False) if not org: org = record.__dict__.get('organization', None) lg = TTLogRecord( id=server.create_uuid(), log_level=record.levelno, log_level_name=record.levelname, logger_name=record.name, logger_pathname=record.pathname, logger_lineno=record.lineno, logger_func=record.funcName, log_type=log_type, msg_short=msg.get('msg_short', ''), msg=msg.get('msg', ''), client_log=client_log, ) if artifact: if isinstance(artifact, ndb.Model): artifact = artifact.key lg.artifact = artifact if org: if isinstance(org, ndb.Model): org = org.key lg.organization = org if request_user: lg.request_user = request_user.key if affected_user: lg.affected_user = affected_user.key if request: lg.request = stringify_request(request) if request else '' lg.put() except: return
def _create_analytic_session(self): analytic_session = AnalyticsSession( id=server.create_uuid(), ip_addr=self.request.remote_addr, user_agent=self.request.user_agent, host=self.request.host, host_url=self.request.host_url, referer=self.request.referer, ) if not self.user.is_world_user(): analytic_session.user = self.user.key analytic_session.put() memcache.add(analytic_session.key.id(), analytic_session, namespace='analytics') self.analytic_session = analytic_session
def send_password_reset(user, gc): env = Environment() user.password_reset_secret = server.create_uuid() user.put() reset_url = (server.server_url + '/account/password/reset/' + user.password_reset_secret) temp_var = { 'username': user.username, 'first_name': user.first_name, 'last_name': user.last_name, 'reset_url': reset_url, } email_temp = env.from_string(source=gc.reset_password_template) msg = MIMEText(email_temp.render(temp_var), 'html') msg['Subject'] = gc.reset_password_template_subject msg['From'] = gc.smtp_username msg['To'] = user.email send_email(user.email, msg, gc) log.info('Sent password resest email to user: %s', user.email, extra={'user': user})
def add_channel_token(self): token_id = str(self.username) + str(server.create_uuid()) token = ChannelToken(id=token_id, token=channel.create_channel(token_id)) client = memcache.Client() while True: # Retry Loop user_token_list = client.gets(key=self.key.id(), namespace='user_tokens') if user_token_list is None: if not memcache.add(key=self.key.id(), value=[token], namespace='user_tokens'): memcache.set(key=self.key.id(), value=[token], namespace='user_tokens') break user_token_list.append(token) if client.cas(key=self.key.id(), value=user_token_list, namespace='user_tokens'): break return token
def get_world_user(): return User(id=server.create_uuid(), user_id=server.create_uuid(), groups=[ndb.Key('Group', 'world')], username='******')
def create_uuid(): return server.create_uuid()
def create_key(cls, id_=None): return ndb.Key(cls.__name__, id_ if id_ else server.create_uuid())
def get_hidden_group_id(group_id): fake_id = memcache.get(str(group_id) + '_fake_id') if fake_id is None: fake_id = server.create_uuid() memcache.add(key=str(group_id) + '_fake_id', value=fake_id, time=config.memcache_fake_id_timeout) return fake_id
def get(self, project, document, group): if not project: raise HttpErrorException.bad_request('invalid project given') project = Project.get_by_id(project) if not project: raise HttpErrorException.bad_request('invalid project given') if not project.has_permission_read(self.user): raise HttpErrorException.forbidden() if not document: raise HttpErrorException.bad_request('invalid document given') document = Document.get_by_id(document) if not document: raise HttpErrorException.bad_request('invalid document given') if not document.has_permission_read(self.user): raise HttpErrorException.forbidden() if not group: raise HttpErrorException.bad_request('invalid group given') group = Group.get_by_id(group) if not group: raise HttpErrorException.bad_request('invalid group given') temp_user = User() temp_user.groups = [group.key, Group.get_worldshare().key] org = self.user.organization if self.user.organization else None if org: temp_user.organization = org if not document.has_permission_read(temp_user): raise HttpErrorException.bad_request( 'Group does not have permission to read the document') if document.key not in project.documents and document.key != project.distilled_document: raise HttpErrorException.bad_request( 'document does not belong to project') pubs = document.get_published(group=group) version_int = PublishDocument.get_largest_version(pubs) if version_int is None: version_int = 1 else: version_int += 1 version = self.request.get('version', str(version_int)) if version == 'latest': raise HttpErrorException.bad_request('invalid version given') pubs = document.get_published(group=group) for pub in pubs: if pub.version == version: raise HttpErrorException.bad_request( 'version name already taken') publisher = DocumentPublisherThread() publisher.id = server.create_uuid() publisher.request = self publisher.project = project publisher.document = document publisher.group = group publisher.user = self.user publisher.version = version if publisher.is_lock(): raise HttpErrorException.bad_request('publisher already running') publisher.start() self.write_json_response({'id': publisher.id}) self.get_analytic_session() document.record_analytic('doc_publish', self.analytic_session, project=project.key)
def import_images(self): concepts = [] deleted_concept = [] image_concept = [] for img_tag in self.image_tags: try: tag = img_tag['img'] src = HTMLImporter.normalize_image_url(self.url, tag['src']) alt = tag.get('alt', None) concept = img_tag['concept'] r = requests.get(src, stream=True) concept.media_id = server.create_uuid() filename = '/' + server.GCS_BUCKET_NAME + '/' + concept.media_id image_data = StringIO(r.content).getvalue() image = images.Image(image_data=image_data) if 'image' not in r.headers['content-type']: deleted_concept.append({ 'concept': concept.key.id(), 'parent_child_count': concept.parent.get().num_of_children(self.user) if concept.parent else None, 'parent': concept.parent.id() if concept.parent else None }) concept.remove_from_parent(concept.project) concept.delete(self.user) continue if image.width < 100 or image.height < 100: deleted_concept.append({ 'concept': concept.key.id(), 'parent_child_count': concept.parent.get().num_of_children(self.user) if concept.parent else None, 'parent': concept.parent.id() if concept.parent else None }) concept.remove_from_parent(concept.project) concept.delete(self.user) else: f = gcs.open(filename, mode='w', content_type=r.headers['content-type']) f.write(image_data) f.close() concept.media_blob = blobstore.create_gs_key('/gs' + filename) concept.media_ready = True concept.media_mime_type = r.headers['content-type'] if alt and alt.rstrip() != '': phr_perm = Permission( permissions=Permission.init_perm_struct( Phrasing.operations_list), key=Permission.create_key(), project=self.project.key) phrasing = Phrasing( key=Phrasing.create_key(), text=alt, concept=concept.key, owner=[self.user.key], permissions=phr_perm.key, originating_document=self.document.key, project=self.project.key) phr_perm.artifact = phrasing.key ndb.put_multi([phrasing, phr_perm]) concept.phrasings.append(phrasing.key) concept.distilled_phrasing = phrasing.key image_concept.append(concept.key.id()) concepts.append(concept) concept.put() except Exception as e: # raise type_, value_, traceback_ = sys.exc_info() ex = traceback.format_exception(type_, value_, traceback_) lr = tt_logging.construct_log( msg_short='Error Importing Image', msg=e.message + '\n\n' + str(ex), log_type=tt_logging.USER, artifact=self.project, request_user=self.user) log.error(lr['dict_msg']['msg'], extra=lr)
def restore_concept_from_json_v2(self, project, concepts, id_mapping, request_user, model_array, parent, documents, zip_file): concepts_key_array = [] for con_json in concepts: con_json['distilled_phrasing_text'] = con_json[ 'distilled_phrasing']['text'] phr_perm = Permission(permissions=Permission.init_perm_struct( Phrasing.operations_list), key=Permission.create_key(), project=project.key) con_perm = Permission(permissions=Permission.init_perm_struct( Concept.operations_list), key=Permission.create_key(), project=project.key) distilled_phrasing = Phrasing( key=Phrasing.create_key(), text=con_json['distilled_phrasing']['text'], owner=[request_user.key], permissions=phr_perm.key, originating_document=project.distilled_document, project=project.key) concept = Concept(key=Concept.create_key(), owner=[request_user.key], parent=parent.key if parent else None, project=project.key, distilled_phrasing=distilled_phrasing.key, phrasings=[distilled_phrasing.key], permissions=con_perm.key) if parent: parent.children.append(concept.key) concept.parent_perms = parent.parent_perms concept.parent_perms.append(parent.permissions) else: project.children.append(concept.key) concept.parent_perms.append(project.permissions) phr_perm.artifact = distilled_phrasing.key con_perm.artifact = concept.key distilled_phrasing.concept = concept.key if request_user.in_org(): distilled_phrasing.organization = request_user.organization concept.organization = request_user.organization model_array += [phr_perm, con_perm, distilled_phrasing, concept] id_mapping[con_json['id']] = concept.key.id() id_mapping[con_json['distilled_phrasing'] ['id']] = distilled_phrasing.key.id() if con_json['id'] == "c3598ccc0b5511e4b7839bfd6cb32178": pass for attr in con_json['attributes']: try: attribute = Attributes( key=Attributes.create_key(), project=project.key, document=documents[id_mapping[attr['document']]].key, attributes=attr['attributes']) concept.attributes.append(attribute.key) model_array.append(attribute) except KeyError: continue for crawl in con_json['crawlcontexts']: try: crawlcontext = CrawlContext( key=CrawlContext.create_key(), project=project.key, document=documents[id_mapping[crawl['document']]].key, crawl=crawl['crawl']) concept.crawlcontext.append(crawlcontext.key) model_array.append(crawlcontext) except KeyError: continue new_phrasing_dict = {} for phrasing_dict in con_json['phrasings']: if phrasing_dict['id'] != con_json['distilled_phrasing']['id']: try: phr_perm = Permission( permissions=Permission.init_perm_struct( Phrasing.operations_list), key=Permission.create_key(), project=project.key) phrasing = Phrasing( key=Phrasing.create_key(), concept=concept.key, text=phrasing_dict['text'], owner=[request_user.key], permissions=phr_perm.key, originating_document=project.distilled_document, project=project.key) phr_perm.artifact = phrasing.key new_phrasing_dict[phrasing.key.id()] = phrasing concept.phrasings.append(phrasing.key) id_mapping[phrasing_dict['id']] = phrasing.key.id() model_array.append(phrasing) model_array.append(phr_perm) except KeyError: continue else: new_phrasing_dict[ distilled_phrasing.key.id()] = distilled_phrasing for sel_phr in con_json['selected_phrasings']: try: selected_phrasing = SelectedPhrasing( key=SelectedPhrasing.create_key(), project=project.key, document=documents[id_mapping[ sel_phr['document']]].key, phrasing=new_phrasing_dict[id_mapping[ sel_phr['phrsing']]].key) concept.selected_phrasings.append(selected_phrasing.key) model_array.append(selected_phrasing) except KeyError: continue if 'is_media' in con_json.keys(): if con_json['is_media']: if 'image' in con_json['content_type']: concept.media_id = server.create_uuid() image = zip_file.open('images/' + con_json['id'], 'r') filename = '/' + server.GCS_BUCKET_NAME + '/' + concept.media_id f = gcs.open(filename, mode='w', content_type=con_json['content_type']) f.write(image.read()) f.close() concept.media_blob = blobstore.create_gs_key('/gs' + filename) if con_json['children'] is not None: self.restore_concept_from_json_v2(project, con_json['children'], id_mapping, request_user, model_array, concept, documents, zip_file) concepts_key_array.append(concept.key) return concepts_key_array
super_admin.active = True super_admin.put() # Create Corpus.io group org = Group(key=Group.create_key(), name='corpus.io', description='corpus.io organization group', organization=corpus.key) org.active = True org.organization = corpus.key corpus.groups.append(org.key) corpus.org_group = org.key org.put() # Setup Super Admin Accounts username = server.create_uuid() print('Super user Andrew, username: '******'username': username, 'password': '******', 'first_name': 'Andrew', 'last_name': 'Miller', 'email': '*****@*****.**', 'organization': corpus.key.id(), 'phone_numbers': { 'main': '304-816-2003' } }, verify_email=False) amiller_su = User.get_by_id(username)
def get(self, project, document, group): if not project: raise HttpErrorException.bad_request('invalid project given') project = Project.get_by_id(project) if not project: raise HttpErrorException.bad_request('invalid project given') if not project.has_permission_read(self.user): raise HttpErrorException.forbidden() if not document: raise HttpErrorException.bad_request('invalid document given') document = Document.get_by_id(document) if not document: raise HttpErrorException.bad_request('invalid document given') if not document.has_permission_read(self.user): raise HttpErrorException.forbidden() if not group: raise HttpErrorException.bad_request('invalid group given') group = Group.get_by_id(group) if not group: raise HttpErrorException.bad_request('invalid group given') if document.key not in project.documents and document.key != project.distilled_document: raise HttpErrorException.bad_request( 'document does not belong to project') temp_user = User() temp_user.groups = [group.key, Group.get_worldshare().key] org = self.user.organization if self.user.organization else None if org: temp_user.organization = org if not document.has_permission_read(temp_user): raise HttpErrorException.bad_request( 'Group does not have permission to read the document') pubs = document.get_presentation_published(group=group) version_int = PublishDocument.get_largest_version(pubs) if version_int is None: version_int = 1 else: version_int += 1 version = self.request.get('version', str(version_int)) if version == 'latest': raise HttpErrorException.bad_request('invalid version given') for pub in pubs: if pub.version == version: raise HttpErrorException.bad_request( 'version name already taken') try: slide_count = int(self.request.get('slide_count', 15)) except ValueError: raise HttpErrorException.bad_request( 'invalid slide count, must be integer') if slide_count < 1: raise HttpErrorException.bad_request( 'invalid slide_count given min 1') if slide_count > 100: raise HttpErrorException.bad_request( 'invalid slide_count given max 100') try: min_bullet = int(self.request.get('min_bullet', 4)) except ValueError: raise HttpErrorException.bad_request( 'invalid min bullet, must be integer') if min_bullet < 1: raise HttpErrorException.bad_request( 'invalid min_bullet given min 1') if min_bullet > 15: raise HttpErrorException.bad_request( 'invalid min_bullet given max 15') try: max_bullet = int(self.request.get('max_bullet', 6)) except ValueError: raise HttpErrorException.bad_request( 'invalid max bullet, must be integer') if max_bullet < 1: raise HttpErrorException.bad_request( 'invalid max_bullet given min 1') if max_bullet > 15: raise HttpErrorException.bad_request( 'invalid max_bullet given max 15') if min_bullet > max_bullet: raise HttpErrorException.bad_request( 'min_bullet can not be greater than max_bullet') publisher = PresentationPublisherThread() publisher.id = server.create_uuid() publisher.request = self publisher.project = project publisher.document = document publisher.slide_count = slide_count publisher.min_bullet = min_bullet publisher.max_bullet = max_bullet publisher.group = group publisher.user = self.user publisher.version = version if publisher.is_lock(): raise HttpErrorException.bad_request('publisher already running') publisher.start() self.write_json_response({'id': publisher.id}) self.get_analytic_session() document.record_analytic('pres_publish', self.analytic_session, project=project.key)
def new(request_user, verify_email=True, request=None, worldshare_group=None, organization=None): if not request_user.get('username'): raise HttpErrorException.bad_request('no username given') if request_user.get('username') == 'anonymous': raise HttpErrorException.bad_request('reserved username') if len(request_user.get('username')) > 75: raise HttpErrorException.bad_request( 'username to long, max 75 characters') if User.get_by_id(request_user.get('username')): raise HttpErrorException.bad_request('username not available') if not request_user.get('password'): raise HttpErrorException.bad_request('no password given') if not request_user.get('first_name'): raise HttpErrorException.bad_request('no first name given') if not request_user.get('last_name'): raise HttpErrorException.bad_request('no last name given') if not request_user.get('email'): raise HttpErrorException.bad_request('no email given') if not request_user.get('phone_numbers'): raise HttpErrorException.bad_request('no phone number given') if not request_user.get('payment_plan') and not organization: raise HttpErrorException.bad_request('no payment plan given') creds = GenericCredentials.from_request(request_user) user = User(id=request_user.get('username')) user.user_id = server.create_uuid() user.username = request_user.get('username') user.display_name = request_user.get( 'display_name', request_user.get('first_name') + ' ' + request_user.get('last_name')) user.password = Password(creds.password) user.first_name = request_user.get('first_name') user.last_name = request_user.get('last_name') user.email = request_user.get('email') user.email_changed = datetime.datetime.now() user.groups.append(worldshare_group) for num_type in request_user.get('phone_numbers'): user.phone_numbers.append( {num_type: request_user.get('phone_numbers').get(num_type)}) if request_user.get('address'): address = request_user.get('address') user.address = { 'street1': address['street1'], 'city': address['city'], 'state': address['state'], 'zip_code': address['zip_code'] } if 'street2' in address: user.address['street2'] = address['street2'] if request_user.get('birthday'): user.birthday = request_user.get('birthday') if organization: user.organization = organization.key user.groups.append(organization.org_group) coup = request_user.get('coupon_code') if coup != '' and coup is not None: try: coup = coupon.Coupon.get_coupon( request_user.get('coupon_code').lower()) user.coupon = coup.key except coupon.InvalidCouponCodeException as e: raise HttpErrorException.bad_request(e.message) if user.organization: user.account_status = payment.ORG user.account_type = payment.ORG user.account_expire_data = datetime.datetime.max else: if not request_user.get('merchant'): raise HttpErrorException.bad_request('no merchant given') payment.init_payment(user, request_user.get('payment_plan'), request_user.get('merchant')) if verify_email: user.email_verification = EmailVerification.new() import smtplib try: mail.send_email_verification(user, GlobalConfig.get_configs()) except smtplib.SMTPException as e: log.error('Could not send verification email') log.exception(e) else: user.email_verification = EmailVerification.new(verified=True) lr = tt_logging.construct_log(msg_short='New User Registered', msg='%s %s has registered an account' % (user.first_name, user.last_name), log_type=tt_logging.REGISTRATION, request_user=user, request=request) log.info(lr['dict_msg']['msg'], extra=lr) user.put() return user