def post(self, concept_id=None):
if not concept_id and not Concept.valid_id(concept_id):
raise HttpErrorException.bad_request('invalid concept id')
concept = Concept.get_by_id(concept_id)
if not concept:
raise HttpErrorException.bad_request('invalid concept id')
if not self.json_request.get('document_id') and not Document.valid_id(
self.json_request.get('document_id')):
raise HttpErrorException.bad_request('invalid document id')
document = Document.get_by_id(self.json_request.get('document_id'))
if not document:
raise HttpErrorException.bad_request('invalid document id')
if not document.has_permission_set_crawlcontext(self.user):
raise HttpErrorException.forbidden()
if document.presentation_document is None:
raise HttpErrorException.bad_request('document has not summary')
presentation_document = document.presentation_document.get()
if document.presentation_document is None:
raise HttpErrorException.bad_request('document has not summary')
project = document.project.get()
if not isinstance(self.json_request.get('crawl'), bool):
raise HttpErrorException.bad_request('invalid crawl')
temp = self.json_request.get('temp', None)
if temp is None:
raise HttpErrorException.bad_request('no temp giving')
crawlcontexts = ndb.get_multi(concept.presentation_crawlcontext)
if temp:
for crawl in crawlcontexts:
if crawl.document == presentation_document.key:
concept.presentation_crawlcontext.remove(crawl.key)
concept.put()
crawl.key.delete()
break
else:
for crawl in crawlcontexts:
if crawl.document == presentation_document.key:
crawl.crawl = self.json_request.get('crawl')
crawl.put()
break
else:
crawl = PresentationCrawlContext(
key=PresentationCrawlContext.create_key(),
project=project.key,
document=presentation_document.key,
crawl=self.json_request.get('crawl'))
crawl.put()
concept.presentation_crawlcontext.append(crawl.key)
concept.put()
project.pw_modified_ts = datetime.datetime.now()
project.put()
def _create_analytic(self, start=None, org=None, usr=None):
concept_index = 0
if usr:
concepts = Concept.query(Concept.owner.IN([usr])).order(
Concept.created_ts).fetch(limit=10)
else:
concepts = Concept.query().order(-Concept.created_ts).fetch(
limit=10)
an_action_count = {}
for action in ANALYTIC_ACTIONS:
an_action_count[action] = random.randint(1, 3)
log.info('Generated Analytics:\n%s', an_action_count)
log.info('User: %s', usr)
log.info('Con len: %s', len(concepts))
ana_obj = []
log.info(len(concepts))
for action, count in an_action_count.iteritems():
for i in xrange(0, count):
ana = Analytics.new(
ts=start + timedelta(minutes=random.randint(0, 59),
seconds=random.randint(0, 59),
milliseconds=random.randint(0, 999)),
action=action,
)
if org:
ana.artifact_organization = org
if usr:
ana.artifact_owners = [usr]
if len(concepts) != 0:
con, concept_index = self._get_next(
concepts, concept_index)
ana.artifact = con.key
ana.project = con.project
ana_obj.append(ana)
ndb.put_multi(ana_obj)
return len(ana_obj)
def post(self):
query_dict = self.json_request.get('query')
if not query_dict:
raise HttpErrorException.bad_request('no query given')
if query_dict['return'] == 'project_ids':
self.write_json_response(
{'projects': Project.search_projects(query_dict, self.user)})
elif query_dict['return'] == 'concept_ids':
self.write_json_response(
{'concepts': Concept.search_concept(query_dict, self.user)})
def post(self, project, concept, action):
project = Project.get_by_id(project)
if not project:
raise HttpErrorException.bad_request('invalid project id')
concept = Concept.get_by_id(concept)
if not concept:
raise HttpErrorException.bad_request('invalid concept id')
if action.rstrip() == '' or not Analytics.is_valid_action(action):
raise HttpErrorException.bad_request('invalid action')
self.get_analytic_session()
concept.record_analytic(action,
self.analytic_session,
project=project.key)
def _delete_document(self):
pro = self.document.project.get()
pro.remove_document(self.document.key)
q = Concept.query()
q = q.filter(Concept.project == pro.key)
for concept in q.iter():
concept.remove_document(self.document.key)
q = Marker.query()
q = q.filter(Marker.document == self.document.key)
for m in q.iter():
m.key.delete()
q = PublishDocument.query()
q = q.filter(PublishDocument.document == self.document.key)
for m in q.iter():
m.key.delete()
q = PublishSummary.query()
q = q.filter(PublishSummary.document == self.document.key)
for m in q.iter():
m.key.delete()
q = PublishPresentation.query()
q = q.filter(PublishPresentation.document == self.document.key)
for m in q.iter():
m.key.delete()
q = PresentationDocument.query()
q = q.filter(PresentationDocument.document == self.document.key)
for m in q.iter():
m.key.delete()
q = SummaryDocument.query()
q = q.filter(SummaryDocument.document == self.document.key)
for m in q.iter():
m.key.delete()
self.document.permissions.delete()
self.document.key.delete()
indexes = self.user.get_indexes(create_new=False)
self.document.index_delete(indexes)
def create_new_concept(self, phrasing_text, parent=None, attribute=None):
phr_perm = Permission(permissions=Permission.init_perm_struct(
Phrasing.operations_list),
key=Permission.create_key(),
project=self.project.key)
con_perm = Permission(permissions=Permission.init_perm_struct(
Concept.operations_list),
key=Permission.create_key(),
project=self.project.key)
phrasing = Phrasing(key=Phrasing.create_key(),
text=phrasing_text,
owner=[self.user.key],
permissions=phr_perm.key,
originating_document=self.document.key,
project=self.project.key)
crawlcontext = CrawlContext(key=CrawlContext.create_key(),
project=self.project.key,
document=self.document.key,
crawl=True)
concept = Concept(key=Concept.create_key(),
owner=[self.user.key],
parent=parent.key if parent else None,
project=self.project.key,
distilled_phrasing=phrasing.key,
phrasings=[phrasing.key],
crawlcontext=[crawlcontext.key],
permissions=con_perm.key)
self.last_concept = concept
crawlcontext.concept = concept.key
if attribute:
concept.attributes = [attribute.key]
attribute.concept = concept.key
if parent:
parent.children.append(concept.key)
concept.parent_perms = parent.parent_perms
concept.parent_perms.append(parent.permissions)
else:
self.project.children.append(concept.key)
concept.parent_perms.append(self.project.permissions)
phr_perm.artifact = phrasing.key
con_perm.artifact = concept.key
phrasing.concept = concept.key
if self.user.in_org():
phrasing.organization = self.user.organization
concept.organization = self.user.organization
self.concept_array[concept.key.id()] = concept
self.crawlcontext_array.append(crawlcontext)
self.phrasing_array.append(phrasing)
self.prems_array += [phr_perm, con_perm]
return concept
def put(self, phrasing_id=None):
self.get_analytic_session()
phrasing_text = self.json_request.get('text')
if not phrasing_text:
raise HttpErrorException.bad_request('invalid phrasing text')
if not self.json_request.get('concept') and not Concept.valid_id(
self.json_request.get('concept')):
raise HttpErrorException.bad_request('invalid concept id')
concept = Concept.get_by_id(self.json_request.get('concept'))
if not concept:
raise HttpErrorException.bad_request('invalid concept id')
if not self.json_request.get('document') and not Document.valid_id(
self.json_request.get('document')):
raise HttpErrorException.bad_request('invalid document id')
document = Document.get_by_id(self.json_request.get('document'))
if not document:
raise HttpErrorException.bad_request('invalid document id')
sum_doc = None
if self.json_request.get('summary', False):
if not document.summary_document:
HttpErrorException.bad_request(
'no document does not have summary')
sum_doc = document.summary_document.get()
if not sum_doc:
HttpErrorException.bad_request(
'no document does not have summary')
pres_doc = None
if self.json_request.get('presentation', False):
if not document.presentation_document:
HttpErrorException.bad_request(
'no document does not have presentation')
pres_doc = document.presentation_document.get()
if not pres_doc:
HttpErrorException.bad_request(
'no document does not have presentation')
phr_perm = Permission(permissions=Permission.init_perm_struct(
Phrasing.operations_list),
key=Permission.create_key(),
project=document.project)
phrasing = Phrasing(key=Phrasing.create_key(),
text=phrasing_text,
concept=concept.key,
owner=[self.user.key],
permissions=phr_perm.key,
project=document.project)
phr_perm.artifact = phrasing.key
concept.phrasings.append(phrasing.key)
sel_phr = None
if (document.is_distilled_document()
and document.project == concept.project and sum_doc is None
and pres_doc is None):
concept.distilled_phrasing = phrasing.key
else:
if sum_doc is not None:
if concept.has_summary_selected_phrasing(document=sum_doc):
sel_phr = concept.get_summary_selected_phrasing(sum_doc)
sel_phr.phrasing = phrasing.key
else:
sel_phr = SummarySelectedPhrasing(
key=SummarySelectedPhrasing.create_key(),
project=concept.project,
document=sum_doc.key,
phrasing=phrasing.key)
concept.summary_selected_phrasings.append(sel_phr.key)
elif pres_doc is not None:
if concept.has_presentation_selected_phrasing(
document=pres_doc):
sel_phr = concept.get_presentation_selected_phrasing(
pres_doc)
sel_phr.phrasing = phrasing.key
else:
sel_phr = PresentationSelectedPhrasing(
key=PresentationSelectedPhrasing.create_key(),
project=concept.project,
document=pres_doc.key,
phrasing=phrasing.key)
concept.presentation_selected_phrasings.append(sel_phr.key)
else:
if concept.has_selected_phrasing(document=document):
sel_phr = concept.get_selected_phrasing(document=document)
sel_phr.phrasing = phrasing.key
else:
sel_phr = SelectedPhrasing(
key=SelectedPhrasing.create_key(),
document=document.key,
phrasing=phrasing.key,
project=document.project)
concept.selected_phrasings.append(sel_phr.key)
sel_phr.put()
phrasing.originating_document = document.key
project = phrasing.project.get()
project.pw_modified_ts = datetime.datetime.now()
ndb.put_multi([phr_perm, phrasing, concept, project])
index = self.user.get_put_index()
phrasing.index(index, concept=concept)
concept.record_analytic('con_phr_new', self.analytic_session)
self.write_json_response({
'phrasing':
phrasing.to_dict(self.user),
'selected_phrasing':
sel_phr.to_dict() if sel_phr else None
})
action_data = {
'concept': concept.key.id(),
'document': document.key.id(),
'phrasing': phrasing.to_dict(self.user)
}
trans = Transaction(action='phr_new',
user=self.user.key,
artifact=phrasing.key,
project=project.key,
action_data=action_data)
trans.put()
self.get_channel_token()
channel_tokens = ChannelToken.get_by_project_key(
project.key, self.user_channel_token)
channel_tokens = ChannelToken.remove_unauthorized_users(
channel_tokens, [concept, phrasing, document])
message = {
'user': self.get_user_channel_data(),
'transaction': trans.to_dict(self.user)
}
ChannelToken.broadcast_message(channel_tokens, message)
def post(self, concept_id=None):
if not concept_id and not Concept.valid_id(concept_id):
raise HttpErrorException.bad_request('invalid concept id')
concept = Concept.get_by_id(concept_id)
if not concept:
raise HttpErrorException.bad_request('invalid concept id')
if not self.json_request.get('document_id') and not Document.valid_id(
self.json_request.get('document_id')):
raise HttpErrorException.bad_request('invalid document id')
document = Document.get_by_id(self.json_request.get('document_id'))
if not document:
raise HttpErrorException.bad_request('invalid document id')
if not document.has_permission_set_crawlcontext(self.user):
raise HttpErrorException.forbidden()
project = document.project.get()
if not isinstance(self.json_request.get('crawl'), bool):
raise HttpErrorException.bad_request('invalid crawl')
crawlcontexts = ndb.get_multi(concept.crawlcontext)
for crawl in crawlcontexts:
if crawl.document == document.key:
crawl.crawl = self.json_request.get('crawl')
crawl.put()
crawlcontext = crawl
break
else:
crawl = CrawlContext(key=CrawlContext.create_key(),
project=project.key,
document=document.key,
crawl=self.json_request.get('crawl'))
crawl.put()
concept.crawlcontext.append(crawl.key)
concept.put()
crawlcontext = crawl
project.pw_modified_ts = datetime.datetime.now()
project.put()
self.get_analytic_session()
if crawl.crawl:
concept.record_analytic('con_cc_t', self.analytic_session)
else:
concept.record_analytic('con_cc_f', self.analytic_session)
action = 'con_hid'
if self.json_request.get('crawl'):
action = 'con_shw'
action_data = {
'concept': concept.key.id(),
'document': document.key.id()
}
trans = Transaction(action=action,
user=self.user.key,
artifact=crawlcontext.key,
project=project.key,
document=document,
action_data=action_data)
trans.put()
self.get_channel_token()
channel_tokens = ChannelToken.get_by_project_key(
project.key, self.user_channel_token)
channel_tokens = ChannelToken.remove_unauthorized_users(
channel_tokens, [concept, document])
message = {
'user': self.get_user_channel_data(),
'transaction': trans.to_dict(self.user)
}
ChannelToken.broadcast_message(channel_tokens, message)
def restore_concept_from_json_v2(self, project, concepts, id_mapping,
request_user, model_array, parent,
documents, zip_file):
concepts_key_array = []
for con_json in concepts:
con_json['distilled_phrasing_text'] = con_json[
'distilled_phrasing']['text']
phr_perm = Permission(permissions=Permission.init_perm_struct(
Phrasing.operations_list),
key=Permission.create_key(),
project=project.key)
con_perm = Permission(permissions=Permission.init_perm_struct(
Concept.operations_list),
key=Permission.create_key(),
project=project.key)
distilled_phrasing = Phrasing(
key=Phrasing.create_key(),
text=con_json['distilled_phrasing']['text'],
owner=[request_user.key],
permissions=phr_perm.key,
originating_document=project.distilled_document,
project=project.key)
concept = Concept(key=Concept.create_key(),
owner=[request_user.key],
parent=parent.key if parent else None,
project=project.key,
distilled_phrasing=distilled_phrasing.key,
phrasings=[distilled_phrasing.key],
permissions=con_perm.key)
if parent:
parent.children.append(concept.key)
concept.parent_perms = parent.parent_perms
concept.parent_perms.append(parent.permissions)
else:
project.children.append(concept.key)
concept.parent_perms.append(project.permissions)
phr_perm.artifact = distilled_phrasing.key
con_perm.artifact = concept.key
distilled_phrasing.concept = concept.key
if request_user.in_org():
distilled_phrasing.organization = request_user.organization
concept.organization = request_user.organization
model_array += [phr_perm, con_perm, distilled_phrasing, concept]
id_mapping[con_json['id']] = concept.key.id()
id_mapping[con_json['distilled_phrasing']
['id']] = distilled_phrasing.key.id()
if con_json['id'] == "c3598ccc0b5511e4b7839bfd6cb32178":
pass
for attr in con_json['attributes']:
try:
attribute = Attributes(
key=Attributes.create_key(),
project=project.key,
document=documents[id_mapping[attr['document']]].key,
attributes=attr['attributes'])
concept.attributes.append(attribute.key)
model_array.append(attribute)
except KeyError:
continue
for crawl in con_json['crawlcontexts']:
try:
crawlcontext = CrawlContext(
key=CrawlContext.create_key(),
project=project.key,
document=documents[id_mapping[crawl['document']]].key,
crawl=crawl['crawl'])
concept.crawlcontext.append(crawlcontext.key)
model_array.append(crawlcontext)
except KeyError:
continue
new_phrasing_dict = {}
for phrasing_dict in con_json['phrasings']:
if phrasing_dict['id'] != con_json['distilled_phrasing']['id']:
try:
phr_perm = Permission(
permissions=Permission.init_perm_struct(
Phrasing.operations_list),
key=Permission.create_key(),
project=project.key)
phrasing = Phrasing(
key=Phrasing.create_key(),
concept=concept.key,
text=phrasing_dict['text'],
owner=[request_user.key],
permissions=phr_perm.key,
originating_document=project.distilled_document,
project=project.key)
phr_perm.artifact = phrasing.key
new_phrasing_dict[phrasing.key.id()] = phrasing
concept.phrasings.append(phrasing.key)
id_mapping[phrasing_dict['id']] = phrasing.key.id()
model_array.append(phrasing)
model_array.append(phr_perm)
except KeyError:
continue
else:
new_phrasing_dict[
distilled_phrasing.key.id()] = distilled_phrasing
for sel_phr in con_json['selected_phrasings']:
try:
selected_phrasing = SelectedPhrasing(
key=SelectedPhrasing.create_key(),
project=project.key,
document=documents[id_mapping[
sel_phr['document']]].key,
phrasing=new_phrasing_dict[id_mapping[
sel_phr['phrsing']]].key)
concept.selected_phrasings.append(selected_phrasing.key)
model_array.append(selected_phrasing)
except KeyError:
continue
if 'is_media' in con_json.keys():
if con_json['is_media']:
if 'image' in con_json['content_type']:
concept.media_id = server.create_uuid()
image = zip_file.open('images/' + con_json['id'], 'r')
filename = '/' + server.GCS_BUCKET_NAME + '/' + concept.media_id
f = gcs.open(filename,
mode='w',
content_type=con_json['content_type'])
f.write(image.read())
f.close()
concept.media_blob = blobstore.create_gs_key('/gs' +
filename)
if con_json['children'] is not None:
self.restore_concept_from_json_v2(project,
con_json['children'],
id_mapping, request_user,
model_array, concept,
documents, zip_file)
concepts_key_array.append(concept.key)
return concepts_key_array
def post(self):
query_dict = self.json_request.get('query')
if not query_dict:
raise HttpErrorException.bad_request('no query given')
if not query_dict['pro']:
raise HttpErrorException.bad_request('no project id given')
project = Project.get_by_id(query_dict['pro'])
if not project:
raise HttpErrorException.bad_request('invalid project id given')
if not project.has_permission(self.user, 'read'):
raise HttpErrorException.forbidden()
index = project.get_indexes(create_new=False)
self.get_analytic_session()
query_string = 'pro=%s typ=(phr OR anno_reply) (reply=(%s) OR phr=(%s))' % (
project.key.id(), query_dict.get('string',
''), query_dict.get('string', ''))
if index is not None and len(index) > 0:
search_results = ttindex.ttsearch(index,
query_string,
limit=1000,
use_cursor=False,
user=self.user)
concepts = []
concept_ids = []
while len(concepts) < 20 and search_results is not None:
for sr in search_results:
if sr['fields']['con'] not in concept_ids:
concept_ids.append(sr['fields']['con'])
if sr['fields']['typ'] == 'phr':
concept = Concept.get_by_id(sr['fields']['con'])
phrasing = Phrasing.get_by_id(sr['id'])
if concept.has_permission_read(
self.user
) and phrasing.has_permission_read(self.user):
concepts.append({
'con_id': sr['fields']['con'],
'phr_text': sr['fields']['phr']
})
elif sr['fields']['typ'] == 'anno_reply':
concept = Concept.get_by_id(sr['fields']['con'])
document = Document.get_by_id(sr['fields']['doc'])
if concept.has_permission_read(self.user) and \
document.has_permission_read(self.user) and \
document.has_permission_annotation_read(self.user):
concepts.append({
'con_id':
sr['fields']['con'],
'phr_text':
sr['fields']['reply']
})
if len(concepts) >= 20:
break
else:
search_results = ttindex.ttsearch(index,
query_dict,
limit=1000,
user=self.user)
self.write_json_response({'concepts': concepts})
else:
self.write_json_response({'concepts': ''})
project.record_analytic('pro_search',
self.analytic_session,
meta_data={'sch_query': str(query_dict)})
def _do_migration():
log.info('Starting Datastore Migration')
logservice.flush()
projects = {}
concept_count = Concept.query().count()
concepts = Concept.query()
index = 1
logservice.flush()
for concept in concepts.iter():
if index % 50 == 0:
log.info('%s concepts finished of %s' % (index, concept_count))
logservice.flush()
project = projects.get(concept.project.id(), None)
if not project:
project = concept.project.get()
if not project:
if len(concept.phrasings) > 0:
perms = []
phrasings = ndb.get_multi(concept.phrasings)
for phrase in phrasings:
if phrase:
perms.append(phrase.permissions)
ndb.delete_multi(concept.phrasings)
ndb.delete_multi(perms)
if len(concept.selected_phrasings) > 0:
ndb.delete_multi(concept.selected_phrasings)
if len(concept.attributes) > 0:
ndb.delete_multi(concept.attributes)
if len(concept.crawlcontext) > 0:
ndb.delete_multi(concept.crawlcontext)
if len(concept.linkes) > 0:
links = ndb.get_multi(concept.linkes)
for link in links:
if link:
link.delete(None, False, True)
if concept.media_blob is not None:
concept.delete_media()
concept.distilled_phrasing.delete()
concept.permissions.delete()
concept.key.delete()
continue
projects[project.key.id()] = project
perm = project.permissions.get()
if not perm.project:
perm.project = project.key
perm.put()
for doc in project.documents:
d = doc.get()
perm = d.permissions.get()
if not perm.project:
perm.project = project.key
perm.put()
d = project.distilled_document.get()
perm = d.permissions.get()
if not perm.project:
perm.project = project.key
perm.put()
_fix_artifacts(concept, project)
index += 1
log.info('Finished Datastore Migration')
logservice.flush()