def post(self, phrasing_id=None):
self.get_channel_token()
if not Phrasing.valid_id(phrasing_id):
raise HttpErrorException.bad_request('invalid phrasing id')
self.phrasing = Phrasing.get_by_id(phrasing_id)
if not self.phrasing:
raise HttpErrorException.bad_request('invalid phrasing id')
if not self.phrasing.has_permission_read(self.user):
raise HttpErrorException.forbidden()
if self.json_request.get('text'):
self._set_text()
if self.json_request.get('permission'):
self._add_perm()
if not self.json_request.get('permission') and self.json_request.get(
'group_id'):
self._rm_perm()
if self.json_request.get('remove_group'):
self._remove_group()
self.phrasing.modified_ts = datetime.datetime.now()
project = self.phrasing.project.get()
project.pw_modified_ts = datetime.datetime.now()
ndb.put_multi([project, self.phrasing])
def create_new_concept(self, phrasing_text, parent=None, attribute=None):
phr_perm = Permission(permissions=Permission.init_perm_struct(
Phrasing.operations_list),
key=Permission.create_key(),
project=self.project.key)
con_perm = Permission(permissions=Permission.init_perm_struct(
Concept.operations_list),
key=Permission.create_key(),
project=self.project.key)
phrasing = Phrasing(key=Phrasing.create_key(),
text=phrasing_text,
owner=[self.user.key],
permissions=phr_perm.key,
originating_document=self.document.key,
project=self.project.key)
crawlcontext = CrawlContext(key=CrawlContext.create_key(),
project=self.project.key,
document=self.document.key,
crawl=True)
concept = Concept(key=Concept.create_key(),
owner=[self.user.key],
parent=parent.key if parent else None,
project=self.project.key,
distilled_phrasing=phrasing.key,
phrasings=[phrasing.key],
crawlcontext=[crawlcontext.key],
permissions=con_perm.key)
self.last_concept = concept
crawlcontext.concept = concept.key
if attribute:
concept.attributes = [attribute.key]
attribute.concept = concept.key
if parent:
parent.children.append(concept.key)
concept.parent_perms = parent.parent_perms
concept.parent_perms.append(parent.permissions)
else:
self.project.children.append(concept.key)
concept.parent_perms.append(self.project.permissions)
phr_perm.artifact = phrasing.key
con_perm.artifact = concept.key
phrasing.concept = concept.key
if self.user.in_org():
phrasing.organization = self.user.organization
concept.organization = self.user.organization
self.concept_array[concept.key.id()] = concept
self.crawlcontext_array.append(crawlcontext)
self.phrasing_array.append(phrasing)
self.prems_array += [phr_perm, con_perm]
return concept
def post(self, phrasing_id=None):
if not phrasing_id:
raise HttpErrorException.bad_request('no phrasing_id given')
phrasing = Phrasing.get_by_id(phrasing_id)
if not phrasing:
raise HttpErrorException.bad_request('invalid phrasing_id given')
if not self.json_request.get('document'):
raise HttpErrorException.bad_request('no document given')
document = Document.get_by_id(self.json_request.get('document'))
if not document:
raise HttpErrorException.bad_request('invalid document given')
if not document.has_permission(self.user, 'manage_phrasings'):
raise HttpErrorException.forbidden()
if not document.presentation_document:
raise HttpErrorException.bad_request(
'document does not have a presentation')
sum_doc = document.presentation_document.get()
if not sum_doc:
raise HttpErrorException.bad_request(
'document does not have a presentation')
concept = phrasing.concept.get()
if concept.has_presentation_selected_phrasing(document=sum_doc):
selected_phrasing = concept.get_presentation_selected_phrasing(
sum_doc)
selected_phrasing.phrasing = phrasing.key
selected_phrasing.put()
else:
selected_phrasing = PresentationSelectedPhrasing(
id=PresentationSelectedPhrasing.create_uuid(),
project=concept.project,
document=sum_doc.key,
phrasing=phrasing.key)
selected_phrasing.put()
concept.presentation_selected_phrasings.append(
selected_phrasing.key)
concept.put()
project = document.project.get(
) # Don't want concept's project, this could be a linked concept
project.pw_modified_ts = datetime.datetime.now()
project.put()
self.get_analytic_session()
concept.record_analytic('con_phr_cha', self.analytic_session)
if selected_phrasing:
self.write_json_response(selected_phrasing.to_dict())
def get(self, selected_phrasing_id=None):
selected_phrasing_dict = {}
q = SelectedPhrasing.query()
if self.request.get('project_id').strip() != '':
q = q.filter(SelectedPhrasing.project == Project.get_by_id(
self.request.get('project_id').strip()).key)
if self.request.get('document_id').strip() != '':
q = q.filter(SelectedPhrasing.document == Document.get_by_id(
self.request.get('document_id').strip()).key)
if self.request.get('phrasing_id').strip() != '':
q = q.filter(SelectedPhrasing.phrasing == Phrasing.get_by_id(
self.request.get('phrasing_id').strip()).key)
for phrase in q.iter():
phrase_dict = phrase.to_dict()
selected_phrasing_dict[phrase.key.id()] = phrase_dict
self.write_json_response(selected_phrasing_dict)
def delete(self, phrasing_id=None):
if not phrasing_id:
raise HttpErrorException.bad_request('no phrasing_id given')
phrasing = Phrasing.get_by_id(phrasing_id)
if not phrasing:
raise HttpErrorException.bad_request('invalid phrasing_id given')
if not phrasing.has_permission_write(self.user):
raise HttpErrorException.forbidden()
concept = phrasing.concept.get()
if phrasing.key == concept.distilled_phrasing:
raise HttpErrorException.forbidden()
project = phrasing.project.get()
project.pw_modified_ts = datetime.datetime.now()
project.put()
trans = Transaction(action='phr_del',
user=self.user.key,
artifact=phrasing.key,
project=project.key)
trans.put()
self.get_channel_token()
channel_tokens = ChannelToken.get_by_project_key(
project.key, self.user_channel_token)
channel_tokens = ChannelToken.remove_unauthorized_users(
channel_tokens, [concept, phrasing])
message = {
'user': self.get_user_channel_data(),
'transaction': trans.to_dict(self.user)
}
ChannelToken.broadcast_message(channel_tokens, message)
phrasing.delete(concept, self.user)
self.get_analytic_session()
concept.record_analytic('con_phr_del', self.analytic_session)
def put(self, phrasing_id=None):
self.get_analytic_session()
phrasing_text = self.json_request.get('text')
if not phrasing_text:
raise HttpErrorException.bad_request('invalid phrasing text')
if not self.json_request.get('concept') and not Concept.valid_id(
self.json_request.get('concept')):
raise HttpErrorException.bad_request('invalid concept id')
concept = Concept.get_by_id(self.json_request.get('concept'))
if not concept:
raise HttpErrorException.bad_request('invalid concept id')
if not self.json_request.get('document') and not Document.valid_id(
self.json_request.get('document')):
raise HttpErrorException.bad_request('invalid document id')
document = Document.get_by_id(self.json_request.get('document'))
if not document:
raise HttpErrorException.bad_request('invalid document id')
sum_doc = None
if self.json_request.get('summary', False):
if not document.summary_document:
HttpErrorException.bad_request(
'no document does not have summary')
sum_doc = document.summary_document.get()
if not sum_doc:
HttpErrorException.bad_request(
'no document does not have summary')
pres_doc = None
if self.json_request.get('presentation', False):
if not document.presentation_document:
HttpErrorException.bad_request(
'no document does not have presentation')
pres_doc = document.presentation_document.get()
if not pres_doc:
HttpErrorException.bad_request(
'no document does not have presentation')
phr_perm = Permission(permissions=Permission.init_perm_struct(
Phrasing.operations_list),
key=Permission.create_key(),
project=document.project)
phrasing = Phrasing(key=Phrasing.create_key(),
text=phrasing_text,
concept=concept.key,
owner=[self.user.key],
permissions=phr_perm.key,
project=document.project)
phr_perm.artifact = phrasing.key
concept.phrasings.append(phrasing.key)
sel_phr = None
if (document.is_distilled_document()
and document.project == concept.project and sum_doc is None
and pres_doc is None):
concept.distilled_phrasing = phrasing.key
else:
if sum_doc is not None:
if concept.has_summary_selected_phrasing(document=sum_doc):
sel_phr = concept.get_summary_selected_phrasing(sum_doc)
sel_phr.phrasing = phrasing.key
else:
sel_phr = SummarySelectedPhrasing(
key=SummarySelectedPhrasing.create_key(),
project=concept.project,
document=sum_doc.key,
phrasing=phrasing.key)
concept.summary_selected_phrasings.append(sel_phr.key)
elif pres_doc is not None:
if concept.has_presentation_selected_phrasing(
document=pres_doc):
sel_phr = concept.get_presentation_selected_phrasing(
pres_doc)
sel_phr.phrasing = phrasing.key
else:
sel_phr = PresentationSelectedPhrasing(
key=PresentationSelectedPhrasing.create_key(),
project=concept.project,
document=pres_doc.key,
phrasing=phrasing.key)
concept.presentation_selected_phrasings.append(sel_phr.key)
else:
if concept.has_selected_phrasing(document=document):
sel_phr = concept.get_selected_phrasing(document=document)
sel_phr.phrasing = phrasing.key
else:
sel_phr = SelectedPhrasing(
key=SelectedPhrasing.create_key(),
document=document.key,
phrasing=phrasing.key,
project=document.project)
concept.selected_phrasings.append(sel_phr.key)
sel_phr.put()
phrasing.originating_document = document.key
project = phrasing.project.get()
project.pw_modified_ts = datetime.datetime.now()
ndb.put_multi([phr_perm, phrasing, concept, project])
index = self.user.get_put_index()
phrasing.index(index, concept=concept)
concept.record_analytic('con_phr_new', self.analytic_session)
self.write_json_response({
'phrasing':
phrasing.to_dict(self.user),
'selected_phrasing':
sel_phr.to_dict() if sel_phr else None
})
action_data = {
'concept': concept.key.id(),
'document': document.key.id(),
'phrasing': phrasing.to_dict(self.user)
}
trans = Transaction(action='phr_new',
user=self.user.key,
artifact=phrasing.key,
project=project.key,
action_data=action_data)
trans.put()
self.get_channel_token()
channel_tokens = ChannelToken.get_by_project_key(
project.key, self.user_channel_token)
channel_tokens = ChannelToken.remove_unauthorized_users(
channel_tokens, [concept, phrasing, document])
message = {
'user': self.get_user_channel_data(),
'transaction': trans.to_dict(self.user)
}
ChannelToken.broadcast_message(channel_tokens, message)
def import_images(self):
concepts = []
deleted_concept = []
image_concept = []
for img_tag in self.image_tags:
try:
tag = img_tag['img']
src = HTMLImporter.normalize_image_url(self.url, tag['src'])
alt = tag.get('alt', None)
concept = img_tag['concept']
r = requests.get(src, stream=True)
concept.media_id = server.create_uuid()
filename = '/' + server.GCS_BUCKET_NAME + '/' + concept.media_id
image_data = StringIO(r.content).getvalue()
image = images.Image(image_data=image_data)
if 'image' not in r.headers['content-type']:
deleted_concept.append({
'concept':
concept.key.id(),
'parent_child_count':
concept.parent.get().num_of_children(self.user)
if concept.parent else None,
'parent':
concept.parent.id() if concept.parent else None
})
concept.remove_from_parent(concept.project)
concept.delete(self.user)
continue
if image.width < 100 or image.height < 100:
deleted_concept.append({
'concept':
concept.key.id(),
'parent_child_count':
concept.parent.get().num_of_children(self.user)
if concept.parent else None,
'parent':
concept.parent.id() if concept.parent else None
})
concept.remove_from_parent(concept.project)
concept.delete(self.user)
else:
f = gcs.open(filename,
mode='w',
content_type=r.headers['content-type'])
f.write(image_data)
f.close()
concept.media_blob = blobstore.create_gs_key('/gs' +
filename)
concept.media_ready = True
concept.media_mime_type = r.headers['content-type']
if alt and alt.rstrip() != '':
phr_perm = Permission(
permissions=Permission.init_perm_struct(
Phrasing.operations_list),
key=Permission.create_key(),
project=self.project.key)
phrasing = Phrasing(
key=Phrasing.create_key(),
text=alt,
concept=concept.key,
owner=[self.user.key],
permissions=phr_perm.key,
originating_document=self.document.key,
project=self.project.key)
phr_perm.artifact = phrasing.key
ndb.put_multi([phrasing, phr_perm])
concept.phrasings.append(phrasing.key)
concept.distilled_phrasing = phrasing.key
image_concept.append(concept.key.id())
concepts.append(concept)
concept.put()
except Exception as e:
# raise
type_, value_, traceback_ = sys.exc_info()
ex = traceback.format_exception(type_, value_, traceback_)
lr = tt_logging.construct_log(
msg_short='Error Importing Image',
msg=e.message + '\n\n' + str(ex),
log_type=tt_logging.USER,
artifact=self.project,
request_user=self.user)
log.error(lr['dict_msg']['msg'], extra=lr)
def restore_v1(self, project_json, request_user, zip_file):
doc_perm = Permission(key=Permission.create_key(),
permissions=Permission.init_perm_struct(
Document.operations_list))
pro_perm = Permission(key=Permission.create_key(),
permissions=Permission.init_perm_struct(
Project.operations_list))
document = Document(
key=Document.create_key(),
title=project_json['distilled_document']['title'],
author=project_json['distilled_document']['author'],
subtitle=project_json['distilled_document']['subtitle'],
date=project_json['distilled_document']['date'],
copyright_text=project_json['distilled_document']
['copyright_text'],
description=project_json['distilled_document']['description'],
owner=[request_user.key],
permissions=doc_perm.key)
project = Project(key=Project.create_key(),
title=project_json['title'],
distilled_document=document.key,
permissions=pro_perm.key,
owner=[request_user.key])
document.project = project.key
document.parent_perms.append(pro_perm.key)
if request_user.in_org():
document.organization = request_user.organization
project.organization = request_user.organization
doc_perm.artifact = document.key
doc_perm.project = project.key
pro_perm.artifact = project.key
pro_perm.project = project.key
ndb.put_multi([doc_perm, pro_perm, document, project])
# We do not want to re-use the old project, document, concept, ... ids so we will create a mapping
# from old to new :)
id_mapping = {
project_json['id']:
project.key.id(),
project_json['distilled_document']['id']:
project.distilled_document.id()
}
documents = {
project.distilled_document.id(): project.distilled_document.get()
}
for doc_json in project_json['documents']:
doc_json['project_id'] = project.key.id()
doc_perm = Permission(key=Permission.create_key(),
project=project.key,
permissions=Permission.init_perm_struct(
Document.operations_list))
doc_obj = Document(key=Document.create_key(),
title=doc_json['title'],
author=doc_json['author'],
subtitle=doc_json['subtitle'],
date=doc_json['date'],
copyright_text=doc_json['copyright_text'],
description=doc_json['description'],
owner=[request_user.key],
permissions=doc_perm.key,
project=project.key)
doc_perm.artifact = doc_obj.key
doc_obj.put()
doc_perm.put()
project.documents.append(doc_obj.key)
id_mapping[doc_json['id']] = doc_obj.key.id()
documents[doc_obj.key.id()] = doc_obj
project.put()
model_array = []
project.children = self.restore_concept_from_json_v1(
project, project_json['concepts'], id_mapping, request_user,
model_array, None, documents, zip_file)
ndb.put_multi(model_array)
Phrasing.index_multi(model_array, request_user)
project.put()
return project
def restore_concept_from_json_v2(self, project, concepts, id_mapping,
request_user, model_array, parent,
documents, zip_file):
concepts_key_array = []
for con_json in concepts:
con_json['distilled_phrasing_text'] = con_json[
'distilled_phrasing']['text']
phr_perm = Permission(permissions=Permission.init_perm_struct(
Phrasing.operations_list),
key=Permission.create_key(),
project=project.key)
con_perm = Permission(permissions=Permission.init_perm_struct(
Concept.operations_list),
key=Permission.create_key(),
project=project.key)
distilled_phrasing = Phrasing(
key=Phrasing.create_key(),
text=con_json['distilled_phrasing']['text'],
owner=[request_user.key],
permissions=phr_perm.key,
originating_document=project.distilled_document,
project=project.key)
concept = Concept(key=Concept.create_key(),
owner=[request_user.key],
parent=parent.key if parent else None,
project=project.key,
distilled_phrasing=distilled_phrasing.key,
phrasings=[distilled_phrasing.key],
permissions=con_perm.key)
if parent:
parent.children.append(concept.key)
concept.parent_perms = parent.parent_perms
concept.parent_perms.append(parent.permissions)
else:
project.children.append(concept.key)
concept.parent_perms.append(project.permissions)
phr_perm.artifact = distilled_phrasing.key
con_perm.artifact = concept.key
distilled_phrasing.concept = concept.key
if request_user.in_org():
distilled_phrasing.organization = request_user.organization
concept.organization = request_user.organization
model_array += [phr_perm, con_perm, distilled_phrasing, concept]
id_mapping[con_json['id']] = concept.key.id()
id_mapping[con_json['distilled_phrasing']
['id']] = distilled_phrasing.key.id()
if con_json['id'] == "c3598ccc0b5511e4b7839bfd6cb32178":
pass
for attr in con_json['attributes']:
try:
attribute = Attributes(
key=Attributes.create_key(),
project=project.key,
document=documents[id_mapping[attr['document']]].key,
attributes=attr['attributes'])
concept.attributes.append(attribute.key)
model_array.append(attribute)
except KeyError:
continue
for crawl in con_json['crawlcontexts']:
try:
crawlcontext = CrawlContext(
key=CrawlContext.create_key(),
project=project.key,
document=documents[id_mapping[crawl['document']]].key,
crawl=crawl['crawl'])
concept.crawlcontext.append(crawlcontext.key)
model_array.append(crawlcontext)
except KeyError:
continue
new_phrasing_dict = {}
for phrasing_dict in con_json['phrasings']:
if phrasing_dict['id'] != con_json['distilled_phrasing']['id']:
try:
phr_perm = Permission(
permissions=Permission.init_perm_struct(
Phrasing.operations_list),
key=Permission.create_key(),
project=project.key)
phrasing = Phrasing(
key=Phrasing.create_key(),
concept=concept.key,
text=phrasing_dict['text'],
owner=[request_user.key],
permissions=phr_perm.key,
originating_document=project.distilled_document,
project=project.key)
phr_perm.artifact = phrasing.key
new_phrasing_dict[phrasing.key.id()] = phrasing
concept.phrasings.append(phrasing.key)
id_mapping[phrasing_dict['id']] = phrasing.key.id()
model_array.append(phrasing)
model_array.append(phr_perm)
except KeyError:
continue
else:
new_phrasing_dict[
distilled_phrasing.key.id()] = distilled_phrasing
for sel_phr in con_json['selected_phrasings']:
try:
selected_phrasing = SelectedPhrasing(
key=SelectedPhrasing.create_key(),
project=project.key,
document=documents[id_mapping[
sel_phr['document']]].key,
phrasing=new_phrasing_dict[id_mapping[
sel_phr['phrsing']]].key)
concept.selected_phrasings.append(selected_phrasing.key)
model_array.append(selected_phrasing)
except KeyError:
continue
if 'is_media' in con_json.keys():
if con_json['is_media']:
if 'image' in con_json['content_type']:
concept.media_id = server.create_uuid()
image = zip_file.open('images/' + con_json['id'], 'r')
filename = '/' + server.GCS_BUCKET_NAME + '/' + concept.media_id
f = gcs.open(filename,
mode='w',
content_type=con_json['content_type'])
f.write(image.read())
f.close()
concept.media_blob = blobstore.create_gs_key('/gs' +
filename)
if con_json['children'] is not None:
self.restore_concept_from_json_v2(project,
con_json['children'],
id_mapping, request_user,
model_array, concept,
documents, zip_file)
concepts_key_array.append(concept.key)
return concepts_key_array
def post(self, phrasing_id=None):
if not phrasing_id:
raise HttpErrorException.bad_request('no phrasing_id given')
phrasing = Phrasing.get_by_id(phrasing_id)
if not phrasing:
raise HttpErrorException.bad_request('invalid phrasing_id given')
if not self.json_request.get('document'):
raise HttpErrorException.bad_request('no document given')
document = Document.get_by_id(self.json_request.get('document'))
if not document:
raise HttpErrorException.bad_request('invalid document given')
if not document.has_permission(self.user, 'manage_phrasings'):
lr = tt_logging.construct_log(
msg_short='User does not have manage_phrasing perm',
log_type=tt_logging.SECURITY,
request=self.request,
artifact=document,
request_user=self.user)
log.info(lr['dict_msg']['msg'], extra=lr)
raise HttpErrorException.forbidden()
concept = phrasing.concept.get()
project = concept.project.get()
selected_phrasing = None
if document.is_distilled_document(
) and project.key == document.project:
concept.distilled_phrasing = phrasing.key
concept.put()
else:
if concept.has_selected_phrasing(document=document):
selected_phrasing = concept.get_selected_phrasing(document)
selected_phrasing.phrasing = phrasing.key
selected_phrasing.put()
else:
selected_phrasing = SelectedPhrasing(id=Phrasing.create_uuid(),
project=concept.project,
document=document.key,
phrasing=phrasing.key)
selected_phrasing.put()
concept.selected_phrasings.append(selected_phrasing.key)
concept.put()
project = document.project.get(
) # Don't want concept's project, this could be a linked concept
project.pw_modified_ts = datetime.datetime.now()
project.put()
self.get_analytic_session()
concept.record_analytic('con_phr_cha', self.analytic_session)
if selected_phrasing:
self.write_json_response(selected_phrasing.to_dict())
action_data = {'document': document.key.id()}
trans = Transaction(action='phr_chg',
user=self.user.key,
artifact=phrasing.key,
project=project.key,
action_data=action_data)
trans.put()
self.get_channel_token()
channel_tokens = ChannelToken.get_by_project_key(
project.key, self.user_channel_token)
channel_tokens = ChannelToken.remove_unauthorized_users(
channel_tokens, [concept, document, phrasing])
message = {
'user': self.get_user_channel_data(),
'transaction': trans.to_dict(self.user)
}
ChannelToken.broadcast_message(channel_tokens, message)
def post(self):
query_dict = self.json_request.get('query')
if not query_dict:
raise HttpErrorException.bad_request('no query given')
if not query_dict['pro']:
raise HttpErrorException.bad_request('no project id given')
project = Project.get_by_id(query_dict['pro'])
if not project:
raise HttpErrorException.bad_request('invalid project id given')
if not project.has_permission(self.user, 'read'):
raise HttpErrorException.forbidden()
index = project.get_indexes(create_new=False)
self.get_analytic_session()
query_string = 'pro=%s typ=(phr OR anno_reply) (reply=(%s) OR phr=(%s))' % (
project.key.id(), query_dict.get('string',
''), query_dict.get('string', ''))
if index is not None and len(index) > 0:
search_results = ttindex.ttsearch(index,
query_string,
limit=1000,
use_cursor=False,
user=self.user)
concepts = []
concept_ids = []
while len(concepts) < 20 and search_results is not None:
for sr in search_results:
if sr['fields']['con'] not in concept_ids:
concept_ids.append(sr['fields']['con'])
if sr['fields']['typ'] == 'phr':
concept = Concept.get_by_id(sr['fields']['con'])
phrasing = Phrasing.get_by_id(sr['id'])
if concept.has_permission_read(
self.user
) and phrasing.has_permission_read(self.user):
concepts.append({
'con_id': sr['fields']['con'],
'phr_text': sr['fields']['phr']
})
elif sr['fields']['typ'] == 'anno_reply':
concept = Concept.get_by_id(sr['fields']['con'])
document = Document.get_by_id(sr['fields']['doc'])
if concept.has_permission_read(self.user) and \
document.has_permission_read(self.user) and \
document.has_permission_annotation_read(self.user):
concepts.append({
'con_id':
sr['fields']['con'],
'phr_text':
sr['fields']['reply']
})
if len(concepts) >= 20:
break
else:
search_results = ttindex.ttsearch(index,
query_dict,
limit=1000,
user=self.user)
self.write_json_response({'concepts': concepts})
else:
self.write_json_response({'concepts': ''})
project.record_analytic('pro_search',
self.analytic_session,
meta_data={'sch_query': str(query_dict)})