def create_new_concept(self, phrasing_text, parent=None, attribute=None):
phr_perm = Permission(permissions=Permission.init_perm_struct(
Phrasing.operations_list),
key=Permission.create_key(),
project=self.project.key)
con_perm = Permission(permissions=Permission.init_perm_struct(
Concept.operations_list),
key=Permission.create_key(),
project=self.project.key)
phrasing = Phrasing(key=Phrasing.create_key(),
text=phrasing_text,
owner=[self.user.key],
permissions=phr_perm.key,
originating_document=self.document.key,
project=self.project.key)
crawlcontext = CrawlContext(key=CrawlContext.create_key(),
project=self.project.key,
document=self.document.key,
crawl=True)
concept = Concept(key=Concept.create_key(),
owner=[self.user.key],
parent=parent.key if parent else None,
project=self.project.key,
distilled_phrasing=phrasing.key,
phrasings=[phrasing.key],
crawlcontext=[crawlcontext.key],
permissions=con_perm.key)
self.last_concept = concept
crawlcontext.concept = concept.key
if attribute:
concept.attributes = [attribute.key]
attribute.concept = concept.key
if parent:
parent.children.append(concept.key)
concept.parent_perms = parent.parent_perms
concept.parent_perms.append(parent.permissions)
else:
self.project.children.append(concept.key)
concept.parent_perms.append(self.project.permissions)
phr_perm.artifact = phrasing.key
con_perm.artifact = concept.key
phrasing.concept = concept.key
if self.user.in_org():
phrasing.organization = self.user.organization
concept.organization = self.user.organization
self.concept_array[concept.key.id()] = concept
self.crawlcontext_array.append(crawlcontext)
self.phrasing_array.append(phrasing)
self.prems_array += [phr_perm, con_perm]
return concept
def put(self, phrasing_id=None):
self.get_analytic_session()
phrasing_text = self.json_request.get('text')
if not phrasing_text:
raise HttpErrorException.bad_request('invalid phrasing text')
if not self.json_request.get('concept') and not Concept.valid_id(
self.json_request.get('concept')):
raise HttpErrorException.bad_request('invalid concept id')
concept = Concept.get_by_id(self.json_request.get('concept'))
if not concept:
raise HttpErrorException.bad_request('invalid concept id')
if not self.json_request.get('document') and not Document.valid_id(
self.json_request.get('document')):
raise HttpErrorException.bad_request('invalid document id')
document = Document.get_by_id(self.json_request.get('document'))
if not document:
raise HttpErrorException.bad_request('invalid document id')
sum_doc = None
if self.json_request.get('summary', False):
if not document.summary_document:
HttpErrorException.bad_request(
'no document does not have summary')
sum_doc = document.summary_document.get()
if not sum_doc:
HttpErrorException.bad_request(
'no document does not have summary')
pres_doc = None
if self.json_request.get('presentation', False):
if not document.presentation_document:
HttpErrorException.bad_request(
'no document does not have presentation')
pres_doc = document.presentation_document.get()
if not pres_doc:
HttpErrorException.bad_request(
'no document does not have presentation')
phr_perm = Permission(permissions=Permission.init_perm_struct(
Phrasing.operations_list),
key=Permission.create_key(),
project=document.project)
phrasing = Phrasing(key=Phrasing.create_key(),
text=phrasing_text,
concept=concept.key,
owner=[self.user.key],
permissions=phr_perm.key,
project=document.project)
phr_perm.artifact = phrasing.key
concept.phrasings.append(phrasing.key)
sel_phr = None
if (document.is_distilled_document()
and document.project == concept.project and sum_doc is None
and pres_doc is None):
concept.distilled_phrasing = phrasing.key
else:
if sum_doc is not None:
if concept.has_summary_selected_phrasing(document=sum_doc):
sel_phr = concept.get_summary_selected_phrasing(sum_doc)
sel_phr.phrasing = phrasing.key
else:
sel_phr = SummarySelectedPhrasing(
key=SummarySelectedPhrasing.create_key(),
project=concept.project,
document=sum_doc.key,
phrasing=phrasing.key)
concept.summary_selected_phrasings.append(sel_phr.key)
elif pres_doc is not None:
if concept.has_presentation_selected_phrasing(
document=pres_doc):
sel_phr = concept.get_presentation_selected_phrasing(
pres_doc)
sel_phr.phrasing = phrasing.key
else:
sel_phr = PresentationSelectedPhrasing(
key=PresentationSelectedPhrasing.create_key(),
project=concept.project,
document=pres_doc.key,
phrasing=phrasing.key)
concept.presentation_selected_phrasings.append(sel_phr.key)
else:
if concept.has_selected_phrasing(document=document):
sel_phr = concept.get_selected_phrasing(document=document)
sel_phr.phrasing = phrasing.key
else:
sel_phr = SelectedPhrasing(
key=SelectedPhrasing.create_key(),
document=document.key,
phrasing=phrasing.key,
project=document.project)
concept.selected_phrasings.append(sel_phr.key)
sel_phr.put()
phrasing.originating_document = document.key
project = phrasing.project.get()
project.pw_modified_ts = datetime.datetime.now()
ndb.put_multi([phr_perm, phrasing, concept, project])
index = self.user.get_put_index()
phrasing.index(index, concept=concept)
concept.record_analytic('con_phr_new', self.analytic_session)
self.write_json_response({
'phrasing':
phrasing.to_dict(self.user),
'selected_phrasing':
sel_phr.to_dict() if sel_phr else None
})
action_data = {
'concept': concept.key.id(),
'document': document.key.id(),
'phrasing': phrasing.to_dict(self.user)
}
trans = Transaction(action='phr_new',
user=self.user.key,
artifact=phrasing.key,
project=project.key,
action_data=action_data)
trans.put()
self.get_channel_token()
channel_tokens = ChannelToken.get_by_project_key(
project.key, self.user_channel_token)
channel_tokens = ChannelToken.remove_unauthorized_users(
channel_tokens, [concept, phrasing, document])
message = {
'user': self.get_user_channel_data(),
'transaction': trans.to_dict(self.user)
}
ChannelToken.broadcast_message(channel_tokens, message)
def put(self, document_id=None):
if not self.json_request.get('project') and not Project.valid_id(
self.json_request.get('project')):
raise HttpErrorException.bad_request('invalid project id')
pro = Project.get_by_id(self.json_request.get('project'))
if not pro:
raise HttpErrorException.bad_request('invalid project id')
if not pro.has_permission_read(self.user):
raise HttpErrorException.forbidden()
if not self.json_request.get('title'):
raise HttpErrorException.bad_request('invalid title')
doc = Document(key=Document.create_key())
doc.project = pro.key
doc.title = self.json_request.get('title')
doc.subtitle = self.json_request.get('subtitle')
doc.author = self.json_request.get('author')
doc.version = self.json_request.get('version')
doc.date = self.json_request.get('date')
doc.copyright_text = self.json_request.get('copyright')
doc.description = self.json_request.get('description')
doc.owner.append(self.user.key)
doc_perm = Permission(permissions=Permission.init_perm_struct(
Document.operations_list),
key=Permission.create_key(),
project=pro.key)
doc_perm.artifact = doc.key
doc_perm.put()
doc.permissions = doc_perm.key
if self.user.in_org():
doc.organization = self.user.organization
doc.parent_perms = [
pro.permissions,
pro.distilled_document.get().permissions
]
doc.put()
pro.documents.append(doc.key)
indexes = self.user.get_put_index()
doc.index(indexes)
pro.pw_modified_ts = datetime.datetime.now()
pro.put()
self.write_json_response(doc.to_dict(self.user))
action_data = {'document': doc.to_dict(self.user)}
trans = Transaction(action='doc_new',
user=self.user.key,
artifact=doc.key,
project=pro.key,
action_data=action_data)
trans.put()
self.get_channel_token()
channel_tokens = ChannelToken.get_by_project_key(
pro.key, self.user_channel_token)
channel_tokens = ChannelToken.remove_unauthorized_users(
channel_tokens, [doc])
for channel_token in channel_tokens:
trans.action_data['document'] = doc.to_dict(
channel_token.user.get())
message = {
'user': self.get_user_channel_data(),
'transaction': trans.to_dict(self.user)
}
ChannelToken.broadcast_message([channel_token], message)
def import_images(self):
concepts = []
deleted_concept = []
image_concept = []
for img_tag in self.image_tags:
try:
tag = img_tag['img']
src = HTMLImporter.normalize_image_url(self.url, tag['src'])
alt = tag.get('alt', None)
concept = img_tag['concept']
r = requests.get(src, stream=True)
concept.media_id = server.create_uuid()
filename = '/' + server.GCS_BUCKET_NAME + '/' + concept.media_id
image_data = StringIO(r.content).getvalue()
image = images.Image(image_data=image_data)
if 'image' not in r.headers['content-type']:
deleted_concept.append({
'concept':
concept.key.id(),
'parent_child_count':
concept.parent.get().num_of_children(self.user)
if concept.parent else None,
'parent':
concept.parent.id() if concept.parent else None
})
concept.remove_from_parent(concept.project)
concept.delete(self.user)
continue
if image.width < 100 or image.height < 100:
deleted_concept.append({
'concept':
concept.key.id(),
'parent_child_count':
concept.parent.get().num_of_children(self.user)
if concept.parent else None,
'parent':
concept.parent.id() if concept.parent else None
})
concept.remove_from_parent(concept.project)
concept.delete(self.user)
else:
f = gcs.open(filename,
mode='w',
content_type=r.headers['content-type'])
f.write(image_data)
f.close()
concept.media_blob = blobstore.create_gs_key('/gs' +
filename)
concept.media_ready = True
concept.media_mime_type = r.headers['content-type']
if alt and alt.rstrip() != '':
phr_perm = Permission(
permissions=Permission.init_perm_struct(
Phrasing.operations_list),
key=Permission.create_key(),
project=self.project.key)
phrasing = Phrasing(
key=Phrasing.create_key(),
text=alt,
concept=concept.key,
owner=[self.user.key],
permissions=phr_perm.key,
originating_document=self.document.key,
project=self.project.key)
phr_perm.artifact = phrasing.key
ndb.put_multi([phrasing, phr_perm])
concept.phrasings.append(phrasing.key)
concept.distilled_phrasing = phrasing.key
image_concept.append(concept.key.id())
concepts.append(concept)
concept.put()
except Exception as e:
# raise
type_, value_, traceback_ = sys.exc_info()
ex = traceback.format_exception(type_, value_, traceback_)
lr = tt_logging.construct_log(
msg_short='Error Importing Image',
msg=e.message + '\n\n' + str(ex),
log_type=tt_logging.USER,
artifact=self.project,
request_user=self.user)
log.error(lr['dict_msg']['msg'], extra=lr)
def create_new_project(self, title):
doc_perm = Permission(key=Permission.create_key(),
permissions=Permission.init_perm_struct(
Document.operations_list))
pro_perm = Permission(key=Permission.create_key(),
permissions=Permission.init_perm_struct(
Project.operations_list))
self.document = Document(key=Document.create_key(),
title='Outline',
owner=[self.user.key],
permissions=doc_perm.key,
subtitle='Outline',
author=self.user.username,
version='v0.1',
date=str(datetime.datetime.now().year),
copyright_text='',
description='')
self.project = Project(key=Project.create_key(),
title=title,
distilled_document=self.document.key,
permissions=pro_perm.key,
owner=[self.user.key])
self.document.project = self.project.key
self.document.parent_perms.append(pro_perm.key)
if self.user.in_org():
self.document.organization = self.user.organization
self.project.organization = self.user.organization
doc_perm.artifact = self.document.key
doc_perm.project = self.project.key
pro_perm.artifact = self.project.key
pro_perm.project = self.project.key
ndb.put_multi([doc_perm, pro_perm, self.document, self.project])
def restore_v1(self, project_json, request_user, zip_file):
doc_perm = Permission(key=Permission.create_key(),
permissions=Permission.init_perm_struct(
Document.operations_list))
pro_perm = Permission(key=Permission.create_key(),
permissions=Permission.init_perm_struct(
Project.operations_list))
document = Document(
key=Document.create_key(),
title=project_json['distilled_document']['title'],
author=project_json['distilled_document']['author'],
subtitle=project_json['distilled_document']['subtitle'],
date=project_json['distilled_document']['date'],
copyright_text=project_json['distilled_document']
['copyright_text'],
description=project_json['distilled_document']['description'],
owner=[request_user.key],
permissions=doc_perm.key)
project = Project(key=Project.create_key(),
title=project_json['title'],
distilled_document=document.key,
permissions=pro_perm.key,
owner=[request_user.key])
document.project = project.key
document.parent_perms.append(pro_perm.key)
if request_user.in_org():
document.organization = request_user.organization
project.organization = request_user.organization
doc_perm.artifact = document.key
doc_perm.project = project.key
pro_perm.artifact = project.key
pro_perm.project = project.key
ndb.put_multi([doc_perm, pro_perm, document, project])
# We do not want to re-use the old project, document, concept, ... ids so we will create a mapping
# from old to new :)
id_mapping = {
project_json['id']:
project.key.id(),
project_json['distilled_document']['id']:
project.distilled_document.id()
}
documents = {
project.distilled_document.id(): project.distilled_document.get()
}
for doc_json in project_json['documents']:
doc_json['project_id'] = project.key.id()
doc_perm = Permission(key=Permission.create_key(),
project=project.key,
permissions=Permission.init_perm_struct(
Document.operations_list))
doc_obj = Document(key=Document.create_key(),
title=doc_json['title'],
author=doc_json['author'],
subtitle=doc_json['subtitle'],
date=doc_json['date'],
copyright_text=doc_json['copyright_text'],
description=doc_json['description'],
owner=[request_user.key],
permissions=doc_perm.key,
project=project.key)
doc_perm.artifact = doc_obj.key
doc_obj.put()
doc_perm.put()
project.documents.append(doc_obj.key)
id_mapping[doc_json['id']] = doc_obj.key.id()
documents[doc_obj.key.id()] = doc_obj
project.put()
model_array = []
project.children = self.restore_concept_from_json_v1(
project, project_json['concepts'], id_mapping, request_user,
model_array, None, documents, zip_file)
ndb.put_multi(model_array)
Phrasing.index_multi(model_array, request_user)
project.put()
return project
def restore_concept_from_json_v2(self, project, concepts, id_mapping,
request_user, model_array, parent,
documents, zip_file):
concepts_key_array = []
for con_json in concepts:
con_json['distilled_phrasing_text'] = con_json[
'distilled_phrasing']['text']
phr_perm = Permission(permissions=Permission.init_perm_struct(
Phrasing.operations_list),
key=Permission.create_key(),
project=project.key)
con_perm = Permission(permissions=Permission.init_perm_struct(
Concept.operations_list),
key=Permission.create_key(),
project=project.key)
distilled_phrasing = Phrasing(
key=Phrasing.create_key(),
text=con_json['distilled_phrasing']['text'],
owner=[request_user.key],
permissions=phr_perm.key,
originating_document=project.distilled_document,
project=project.key)
concept = Concept(key=Concept.create_key(),
owner=[request_user.key],
parent=parent.key if parent else None,
project=project.key,
distilled_phrasing=distilled_phrasing.key,
phrasings=[distilled_phrasing.key],
permissions=con_perm.key)
if parent:
parent.children.append(concept.key)
concept.parent_perms = parent.parent_perms
concept.parent_perms.append(parent.permissions)
else:
project.children.append(concept.key)
concept.parent_perms.append(project.permissions)
phr_perm.artifact = distilled_phrasing.key
con_perm.artifact = concept.key
distilled_phrasing.concept = concept.key
if request_user.in_org():
distilled_phrasing.organization = request_user.organization
concept.organization = request_user.organization
model_array += [phr_perm, con_perm, distilled_phrasing, concept]
id_mapping[con_json['id']] = concept.key.id()
id_mapping[con_json['distilled_phrasing']
['id']] = distilled_phrasing.key.id()
if con_json['id'] == "c3598ccc0b5511e4b7839bfd6cb32178":
pass
for attr in con_json['attributes']:
try:
attribute = Attributes(
key=Attributes.create_key(),
project=project.key,
document=documents[id_mapping[attr['document']]].key,
attributes=attr['attributes'])
concept.attributes.append(attribute.key)
model_array.append(attribute)
except KeyError:
continue
for crawl in con_json['crawlcontexts']:
try:
crawlcontext = CrawlContext(
key=CrawlContext.create_key(),
project=project.key,
document=documents[id_mapping[crawl['document']]].key,
crawl=crawl['crawl'])
concept.crawlcontext.append(crawlcontext.key)
model_array.append(crawlcontext)
except KeyError:
continue
new_phrasing_dict = {}
for phrasing_dict in con_json['phrasings']:
if phrasing_dict['id'] != con_json['distilled_phrasing']['id']:
try:
phr_perm = Permission(
permissions=Permission.init_perm_struct(
Phrasing.operations_list),
key=Permission.create_key(),
project=project.key)
phrasing = Phrasing(
key=Phrasing.create_key(),
concept=concept.key,
text=phrasing_dict['text'],
owner=[request_user.key],
permissions=phr_perm.key,
originating_document=project.distilled_document,
project=project.key)
phr_perm.artifact = phrasing.key
new_phrasing_dict[phrasing.key.id()] = phrasing
concept.phrasings.append(phrasing.key)
id_mapping[phrasing_dict['id']] = phrasing.key.id()
model_array.append(phrasing)
model_array.append(phr_perm)
except KeyError:
continue
else:
new_phrasing_dict[
distilled_phrasing.key.id()] = distilled_phrasing
for sel_phr in con_json['selected_phrasings']:
try:
selected_phrasing = SelectedPhrasing(
key=SelectedPhrasing.create_key(),
project=project.key,
document=documents[id_mapping[
sel_phr['document']]].key,
phrasing=new_phrasing_dict[id_mapping[
sel_phr['phrsing']]].key)
concept.selected_phrasings.append(selected_phrasing.key)
model_array.append(selected_phrasing)
except KeyError:
continue
if 'is_media' in con_json.keys():
if con_json['is_media']:
if 'image' in con_json['content_type']:
concept.media_id = server.create_uuid()
image = zip_file.open('images/' + con_json['id'], 'r')
filename = '/' + server.GCS_BUCKET_NAME + '/' + concept.media_id
f = gcs.open(filename,
mode='w',
content_type=con_json['content_type'])
f.write(image.read())
f.close()
concept.media_blob = blobstore.create_gs_key('/gs' +
filename)
if con_json['children'] is not None:
self.restore_concept_from_json_v2(project,
con_json['children'],
id_mapping, request_user,
model_array, concept,
documents, zip_file)
concepts_key_array.append(concept.key)
return concepts_key_array
def put(self, project_id=None):
if self.json_request.get('title') is None:
raise HttpErrorException.bad_request('invalid project title')
if self.json_request.get('distilled_document') is None:
raise HttpErrorException.bad_request('invalid document')
distilled_document = self.json_request.get('distilled_document')
if not distilled_document.get('title'):
raise HttpErrorException.bad_request('invalid document title')
doc_perm = Permission(permissions=Permission.init_perm_struct(
Document.operations_list),
key=Permission.create_key())
doc = Document(
key=Document.create_key(),
title=distilled_document.get('title'),
subtitle=distilled_document.get('subtitle'),
author=distilled_document.get('author', self.user.full_name),
version=distilled_document.get('version', 'v0.1'),
date=distilled_document.get('date',
datetime.datetime.now().year),
copyright_text=distilled_document.get('copyright', ''),
description=distilled_document.get('description', ''),
owner=[self.user.key],
permissions=doc_perm.key,
)
doc_perm.artifact = doc.key
pro_perm = Permission(permissions=Permission.init_perm_struct(
Project.operations_list),
key=Permission.create_key())
pro = Project(
key=Project.create_key(),
title=self.json_request.get('title'),
distilled_document=doc.key,
permissions=pro_perm.key,
owner=[self.user.key],
)
pro_perm.artifact = pro.key
pro_perm.project = pro.key
doc_perm.project = pro.key
doc.project = pro.key
doc.parent_perms.append(pro_perm.key)
if self.user.in_org():
doc.organization = self.user.organization
pro.organization = self.user.organization
ndb.put_multi([doc_perm, doc, pro_perm, pro])
index = self.user.get_put_index()
doc.index(index)
pro.index(index)
self.write_json_response(pro.to_dict(self.user))