def put_index(self):
"""index the element at each put"""
empty_string = lambda x: x if x else ""
collaborators = " ".join(self.collaborators_ids)
organization = str(self.organization.id())
title_autocomplete = ','.join(tokenize_autocomplete(self.content))
my_document = search.Document(
doc_id=str(self.key.id()),
fields=[
search.TextField(name=u'type', value=u'Comment'),
search.TextField(name='organization',
value=empty_string(organization)),
search.TextField(name='entityKey',
value=empty_string(self.key.urlsafe())),
search.TextField(name='title',
value=empty_string(self.content)),
search.TextField(name='parent_id',
value=empty_string(self.parent_id)),
search.TextField(name='parent_kind',
value=empty_string(self.parent_kind)),
search.TextField(name='access',
value=empty_string(self.access)),
search.TextField(name='owner', value=empty_string(self.owner)),
search.TextField(name='collaborators', value=collaborators),
search.TextField(name='content',
value=empty_string(self.content)),
search.DateField(name='created_at', value=self.created_at),
search.DateField(name='updated_at', value=self.updated_at),
search.TextField(name='title_autocomplete',
value=empty_string(title_autocomplete)),
])
my_index = search.Index(name="GlobalIndex")
my_index.put(my_document)
def update_search_index(self, id, event):
event_doc = search.Document(
# As search documents cannot be updated and can only be replaced, for every create or update, we will replace the search document completely.
# Hence using the datastore id for the search id as well. Retrieving the entity also becomes easier this way.
doc_id=str(id),
# Store only the fields that are required to be searched/sorted upon. For retrievel, we will retrieve the id from search and use that to query the entity from the datastore.
fields=[
search.TextField(name='name', value=event.name),
search.TextField(name='caption', value=event.caption),
search.TextField(name='sport', value=event.sport),
search.TextField(name='locality',
value=event.address.locality),
search.TextField(name='locality_id',
value=event.address.locality_id),
search.TextField(name='city', value=event.address.city),
search.DateField(name='start_datetime',
value=event.start_datetime),
search.DateField(name='end_datetime',
value=event.end_datetime),
search.NumberField(name='status', value=event.status),
search.DateField(name='updated_on',
value=datetime.date(event.updated_on)),
search.GeoField(name='latlong',
value=search.GeoPoint(
event.address.latlong.lat,
event.address.latlong.lon))
])
try:
index = search.Index(name=EVENT)
index.put(event_doc)
except search.Error:
logger.exception('Storing event %s in search index failed' % id)
def _create_doc_event(cls, studio_class):
title = '%s: %s' % (studio_class.style, studio_class.teacher)
description = studio_class.teacher_link
# include twitter link to studio?
timestamp = min(int(time.mktime(studio_class.start_time.timetuple())),
2**31 - 1)
doc_event = search.Document(
doc_id=studio_class.key.string_id(),
fields=[
search.TextField(name='name', value=title),
search.TextField(name='studio',
value=studio_class.studio_name),
search.TextField(name='source_page',
value=studio_class.source_page),
search.TextField(name='description', value=description),
search.DateField(name='start_time',
value=studio_class.start_time),
search.DateField(name='end_time', value=studio_class.end_time),
search.NumberField(name='latitude',
value=studio_class.latitude),
search.NumberField(name='longitude',
value=studio_class.longitude),
search.TextField(name='categories',
value=' '.join(studio_class.auto_categories)),
search.TextField(name='sponsor', value=studio_class.sponsor),
#search.TextField(name='country', value=studio_class.country),
],
#language=XX, # We have no good language detection
rank=timestamp,
)
return doc_event
def _indexForumThread(forum_thread, new_forum_post = None):
"""Does the actual work of indexing the given forum thread. We expect to be called in a deferred handler."""
forum= forum_thread.forum
fields = [search.TextField(name="subject", value=forum_thread.subject),
search.DateField(name="posted", value=forum_thread.posted),
search.DateField(name="last_post", value=forum_thread.last_post),
search.AtomField(name="forum", value=forum.slug.replace(":", "_"))]
if forum.alliance:
fields.append(search.AtomField(name="alliance", value=forum.alliance.replace(":", "_")))
else:
fields.append(search.AtomField(name="alliance", value="NA"))
content = ""
for forum_post in model.forum.ForumPost.all().ancestor(forum_thread).order("posted"):
if new_forum_post and str(forum_post.key()) == str(new_forum_post.key()):
new_forum_post = None
content += "\r\n<hr />\r\n" + forum_post.content
if new_forum_post:
content = new_forum_post.content + content
fields.append(search.HtmlField(name="content", value=content))
doc = search.Document(
doc_id = str(forum_thread.key()),
fields = fields)
index = search.Index(name="forum")
index.put(doc)
def _create_doc_event(cls, db_event):
if not db_event.has_content():
return None
# TODO(lambert): find a way to index no-location events.
# As of now, the lat/long number fields cannot be None.
# In what radius/location should no-location events show up
# and how do we want to return them
# Perhaps a separate index that is combined at search-time?
if db_event.latitude is None:
logging.warning("Skipping event without latitude: %s", db_event.id)
return None
# If this event has been deleted from Facebook, let's skip re-indexing it here
if db_event.start_time is None:
logging.warning("Skipping event without start_time: %s",
db_event.id)
return None
if not isinstance(db_event.start_time,
datetime.datetime) and not isinstance(
db_event.start_time, datetime.date):
logging.error("DB Event %s start_time is not correct format: ",
db_event.id, db_event.start_time)
return None
timestamp = min(int(time.mktime(db_event.start_time.timetuple())),
2**31 - 1)
doc_event = search.Document(
doc_id=db_event.id,
fields=[
search.TextField(name='name', value=db_event.name),
search.TextField(name='description',
value=db_event.description),
search.NumberField(name='attendee_count',
value=db_event.attendee_count or 0),
search.DateField(name='start_time', value=db_event.start_time),
search.DateField(name='end_time',
value=dates.faked_end_time(
db_event.start_time, db_event.end_time)),
search.NumberField(name='latitude', value=db_event.latitude),
search.NumberField(name='longitude', value=db_event.longitude),
search.TextField(name='categories',
value=' '.join(db_event.auto_categories)),
search.TextField(name='country', value=db_event.country),
# Use NumberField instead of DateField since we care about hours/minutes/seconds,
# which are otherwise discarded with a DateField.
# We want this to know what events have been added in the last 24 hours,
# so we can promote them to users when we send out daily notifications.
search.NumberField(
name='creation_time',
value=int(time.mktime(db_event.creation_time.timetuple()))
if db_event.creation_time else 0),
],
# TODO: Re-enable this. We disabled it because:
# - some langauges are like 'zh-Hant', but this expects two-letter languages
# - some events don't have a 'language' field in their json_props....why/how?!?
# language=db_event.json_props['language'],
rank=timestamp,
)
return doc_event
def post(self):
keys = self.request.arguments()
myFields = []
for arg in keys:
if arg != "doctext" and arg != "_csrf_token":
value = self.request.get(arg)
logging.info("upqueuehandler arg=%s" % arg)
logging.info("value=%s" % value)
newField = search.TextField(name=arg, value=value)
myFields.append(newField)
try:
docdate = self.request.get('docdate')
dtSplt = docdate.split('-')
docyear = int(dtSplt[0])
docmonth = int(dtSplt[1])
docday = int(dtSplt[2])
myDocDate = search.DateField(name='datedate',
value=date(docyear, docmonth, docday))
myFields.append(myDocDate)
except:
b = "why am I here?"
myStamp = search.DateField(name='stamp', value=datetime.now().date())
myFields.append(myStamp)
doctext = self.request.get('doctext')
filename1 = self.request.get('link')
gcs_file = gcs.open(filename1)
try:
text = extractTextFromFile(gcs_file, filename1)
#text = "----> this is where the conversion occurs <----"
doctext = text + doctext
except:
a = "do something good"
gcs_file.close()
logging.info("len=")
logging.info(len(doctext))
if len(doctext) > 983040:
doctext = doctext[:983040]
mydoctext = search.TextField(name='doctext', value=doctext)
myFields.append(mydoctext)
""" make a seach document"""
award = search.Document(fields=myFields)
""" put that search document in to the index """
search.Index(name=_INDEX_NAME).put(award)
def CreateDocument(text, date_,num,sheet, row, col ):
return search.Document(
fields=[search.TextField(name='text', value=text),
search.DateField(name='date_', value=date_.date()),
search.NumberField(name='num', value=num),
search.TextField(name='sheet', value=sheet),
search.NumberField(name='row', value=row),
search.NumberField(name='col', value=col),
search.DateField(name='date', value=datetime.now().date())])
def ToDocument(self):
return search.Document(
doc_id = self.GetId(),
fields = [
search.DateField(name='when', value=self.when),
search.DateField(name='last_modified', value=self.last_modified),
search.TextField(name='title', value=self.title),
search.TextField(name='author', value=self.author),
search.TextField(name='code', value=self.code),
])
def put_index(self, data=None):
""" index the element at each"""
empty_string = lambda x: x if x else ""
collaborators = " ".join(self.collaborators_ids)
organization = str(self.organization.id())
title_autocomplete = ','.join(tokenize_autocomplete(
self.name + ' ' + empty_string(self.account_name) + ' ' + empty_string(self.contact_name)))
if data:
search_key = ['infos', 'cases', 'tags', 'collaborators']
for key in search_key:
if key not in data.keys():
data[key] = ""
my_document = search.Document(
doc_id=str(data['id']),
fields=[
search.TextField(name=u'type', value=u'Case'),
search.TextField(name='organization', value=empty_string(organization)),
search.TextField(name='access', value=empty_string(self.access)),
search.TextField(name='owner', value=empty_string(self.owner)),
search.TextField(name='collaborators', value=data['collaborators']),
search.TextField(name='title', value=empty_string(self.name)),
search.TextField(name='account_name', value=empty_string(self.account_name)),
search.TextField(name='contact_name', value=empty_string(self.contact_name)),
search.TextField(name='status', value=empty_string(self.status)),
search.NumberField(name='priority', value=int(self.priority or 1)),
search.DateField(name='created_at', value=self.created_at),
search.DateField(name='updated_at', value=self.updated_at),
search.TextField(name='infos', value=data['infos']),
search.TextField(name='tags', value=data['tags']),
search.TextField(name='cases', value=data['cases']),
search.TextField(name='title_autocomplete', value=empty_string(title_autocomplete)),
search.TextField(name='type_case', value=empty_string(self.type_case))
])
else:
my_document = search.Document(
doc_id=str(self.key.id()),
fields=[
search.TextField(name=u'type', value=u'Case'),
search.TextField(name='organization', value=empty_string(organization)),
search.TextField(name='access', value=empty_string(self.access)),
search.TextField(name='owner', value=empty_string(self.owner)),
search.TextField(name='collaborators', value=collaborators),
search.TextField(name='title', value=empty_string(self.name)),
search.TextField(name='account_name', value=empty_string(self.account_name)),
search.TextField(name='contact_name', value=empty_string(self.contact_name)),
search.TextField(name='status', value=empty_string(self.status)),
search.NumberField(name='priority', value=int(self.priority)),
search.DateField(name='created_at', value=self.created_at),
search.DateField(name='updated_at', value=self.updated_at),
search.TextField(name='title_autocomplete', value=empty_string(title_autocomplete)),
search.TextField(name='type_case', value=empty_string(self.type_case))
])
my_index = search.Index(name="GlobalIndex")
my_index.put(my_document)
def put_index(self, data=None):
""" index the element at each"""
empty_string = lambda x: x if x else ""
collaborators = " ".join(self.collaborators_ids)
organization = str(self.organization.id())
if data:
search_key = ['infos', 'events', 'tags']
for key in search_key:
if key not in data.keys():
data[key] = ""
my_document = search.Document(
doc_id=str(data['id']),
fields=[
search.TextField(name=u'type', value=u'Event'),
search.TextField(name='organization', value=empty_string(organization)),
search.TextField(name='access', value=empty_string(self.access)),
search.TextField(name='owner', value=empty_string(self.owner)),
search.TextField(name='collaborators', value=collaborators),
search.TextField(name='where', value=empty_string(self.where)),
search.TextField(name='description', value=empty_string(self.description)),
search.TextField(name='title', value=empty_string(self.title)),
search.DateField(name='created_at', value=self.created_at),
search.DateField(name='updated_at', value=self.updated_at),
search.TextField(name='infos', value=data['infos']),
search.TextField(name='tags', value=data['tags']),
search.TextField(name='events', value=data['events']),
search.NumberField(name='comments', value=self.comments),
search.TextField(name='about_kind', value=empty_string(self.about_kind)),
search.TextField(name='about_item', value=empty_string(self.about_item)),
])
else:
my_document = search.Document(
doc_id=str(self.key.id()),
fields=[
search.TextField(name=u'type', value=u'Event'),
search.TextField(name='organization', value=empty_string(organization)),
search.TextField(name='access', value=empty_string(self.access)),
search.TextField(name='owner', value=empty_string(self.owner)),
search.TextField(name='collaborators', value=collaborators),
search.TextField(name='where', value=empty_string(self.where)),
search.TextField(name='description', value=empty_string(self.description)),
search.TextField(name='title', value=empty_string(self.title)),
search.DateField(name='created_at', value=self.created_at),
search.DateField(name='updated_at', value=self.updated_at),
search.NumberField(name='comments', value=self.comments),
search.TextField(name='about_kind', value=empty_string(self.about_kind)),
search.TextField(name='about_item', value=empty_string(self.about_item)),
])
my_index = search.Index(name="GlobalIndex")
my_index.put(my_document)
def searchablize_tag_or_stream(item, index_name, response):
index = search.Index(name=index_name, namespace=search_index_namespace)
if item is None:
return
toks = item.name.split()
try:
full_str = ""
for tok in toks:
for i in range(len(tok) + 1):
for j in range(i):
substr = tok[j:i]
add_strs = [substr]
for s in add_strs:
doc = search.Document(fields=[
search.AtomField(name='id',
value=str(item.key.id())),
search.TextField(name='name', value=item.name),
search.TextField(name='string', value=s),
search.DateField(
name='date_added',
value=datetime.datetime.now().date())
])
# Index the document.
index.put(doc)
full_str += " " + tok
except search.PutError, e:
result = e.results[0]
response['errResult'] = str(result)
def create_investment_agreement_document(investment, iyo_username):
# type: (InvestmentAgreement) -> search.Document
investment_id_str = str(investment.id)
fields = [
search.AtomField(name='id', value=investment_id_str),
search.AtomField(name='reference', value=investment.reference),
search.NumberField(name='status', value=investment.status),
search.TextField(name='username', value=iyo_username),
search.DateField(name='creation_time',
value=datetime.utcfromtimestamp(
investment.creation_time)),
search.TextField(name='name', value=investment.name),
search.TextField(name='address',
value=investment.address
and investment.address.replace('\n', '')),
search.TextField(name='currency', value=investment.currency),
]
if investment.amount:
fields.append(
search.TextField(name='amount',
value=_stringify_float(investment.amount)))
if investment.token_count:
fields.append(
search.TextField(name='token_count',
value=_stringify_float(
investment.token_count_float)))
return search.Document(doc_id=investment_id_str, fields=fields)
def put_one_document(self, msg):
doc_id = '{channel_id}_{user}_{ts}'.format(channel_id=msg.channel_id,
user=msg.user,
ts=int(msg.ts))
doc = search.Document(doc_id=doc_id,
fields=[
search.TextField(name='text',
value=msg.text),
search.AtomField(name='user_name',
value=msg.get_user_name()),
search.AtomField(name='channel_id',
value=msg.channel_id),
search.AtomField(name='msg_key',
value=str(msg.key.id())),
search.DateField(name='ts',
value=msg.get_datetime()),
])
# Index the document.
try:
self.index.put(doc)
except search.PutError, e:
result = e.results[0]
if result.code == search.OperationResult.TRANSIENT_ERROR:
# possibly retry indexing result.object_id
return self.put_one_document(msg)
def get_field(self,key,value):
try:
if(self.mapping[key]):
if(self.mapping[key]=='TEXTFIELD'):
return search.TextField(name=key,value=value);
elif(self.mapping[key]=='ATOMFIELD'):
return search.AtomField(name=key,value=value);
elif(self.mapping[key]=='NUMBERFIELD'):
return search.NumberField(name=key,value=value);
elif(self.mapping[key]=='HTMLFIELD'):
return search.HtmlField(name=key,value=value);
elif(self.mapping[key]=='DATEFIELD'):
timestamp=value;
timestamp = timestamp if timestamp>0 else -timestamp;
value=datetime.fromtimestamp(timestamp/1000.0);
return search.DateField(name=key,value=value);
elif(self.mapping[key]=='GEOFIELD'):
return search.GeoField(name=key,value=value);
else:
return None;
else:
return None;
except KeyError,keyError:
print(keyError);
return None;
def after_insert(self, fields, id):
if DEBUG:
print 'after insert', fields, id
fieldlist = []
for f in self.fieldnames:
#fieldlist.append(name=fieldname, fields[fieldname]),
#Proposed mapping of fieldtypes to search types is as follows:
#string - textfield
#test - html field
#datetime & datetime to date fields
#list:string woudl be added later and would need to iterate through
if self.table[f].type == 'string':
fieldvalue = search.TextField(name=f, value=fields[f])
fieldlist.append(fieldvalue)
elif self.table[f].type == 'list:string':
for listvalue in fields[f]:
if listvalue:
fieldvalue = search.TextField(name=f, value=listvalue)
fieldlist.append(fieldvalue)
elif self.table[f].type == 'date' or self.table[
f].type == 'datetime':
fieldvalue = search.DateField(name=f, value=fields[f])
fieldlist.append(fieldvalue)
else: #should be text
fieldvalue = search.HtmlField(name=f, value=fields[f])
fieldlist.append(fieldvalue)
strid = self.table._tablename + '.' + str(id)
my_document = search.Document(doc_id=strid, fields=fieldlist)
add_result = self.index.put(my_document)
if DEBUG:
pass
return True
def index_artifact(index_, id_, fields):
f = []
for i in xrange(0, len(fields), 3):
if fields[i] == ATOMFIELD:
f.append(search.AtomField(name=fields[i + 1], value=fields[i + 2]))
elif fields[i] == TEXTFIELD:
f.append(search.TextField(name=fields[i + 1], value=fields[i + 2]))
elif fields[i] == HTMLFIELD:
f.append(search.HtmlField(name=fields[i + 1], value=fields[i + 2]))
elif fields[i] == NUMBERFIELD:
f.append(
search.NumberField(name=fields[i + 1], value=fields[i + 2]))
elif fields[i] == DATEFIELD:
f.append(search.DateField(name=fields[i + 1], value=fields[i + 2]))
elif fields[i] == GEOPOINTFIELD:
f.append(search.GeoField(name=fields[i + 1], value=fields[i + 2]))
doc = search.Document(doc_id=id_, fields=f)
retry_count = 0
while True:
try:
index_.put(doc)
break
except search.Error as e:
if retry_count < 3:
log.warning(
'Error put doc into index, could be out of space. Creating new index'
)
index_ = search.Index(index_.name[:-4] +
str(int(index_.name[-4:])).zfill(4),
namespace=index_.namespace)
retry_count += 1
else:
raise e
def create_document_for_paste(paste):
config = [
('author', search.TextField),
('description', search.TextField),
]
fields = [f(name=n, value=getattr(paste, n)) for n, f in config]
# The search API refuses to handle timezone-aware datetimes.
created = paste.created.replace(tzinfo=None)
fields.append(search.DateField(name='created', value=created))
# Then we need to get the paste's content.
for pasty_file in paste.files:
name_field = search.TextField(name='filename', value=pasty_file.filename)
type_field = search.TextField(name='content_type', value=pasty_file.content_type)
with pasty_file.open('r') as fh:
value = fh.read()
content_field = search.TextField(name='content', value=value)
fields.extend([name_field, type_field, content_field])
# The default rank is just when the doc was inserted. We use the created
# date as rank, which will automatically sort results by paste created.
rank = datetime_to_timestamp(paste.created)
doc_id = unicode(paste.key.id())
doc = search.Document(doc_id=doc_id, rank=rank, fields=fields)
return doc
def post(self):
headline = self.request.get("headline")
content = self.request.get("content")
author = self.request.get("author")
sideheadline = self.request.get("sideheadline")
featured = int(self.request.get("featured"))
tags = (self.request.get("tags")).split(',')
if self.request.get('picture'):
piclink = self.request.get('picture')
tempvar="upload/c_scale,h_900,q_auto:good,w_1600"
picture =piclink.replace('upload',tempvar)
else:
picture = "/images/default.jpg"
a = article(headline = headline,tags = tags,content =content,author = author,picture = picture,sideheadline = sideheadline,featured = featured)
key = a.put()
article_id = key.id()
fields = [search.TextField(name = "headline", value = headline),
search.TextField(name = "sideheadline", value = sideheadline),
search.TextField(name = "tags", value = ",".join(tags)),
search.HtmlField(name = "content", value = content),
search.TextField(name = "author", value = author),
search.DateField(name = "dateCreated", value = datetime.datetime.now().date())]
doc = search.Document(doc_id = str(article_id), fields = fields)
memcache.delete(key='homepage')
try:
add_result = search.Index(name="Articles").put(doc)
#wait for a small duration so that memcache is cleared before it can be reused
time.sleep(0.1)
self.redirect('/')
except search.Error:
self.request.out.write("adding to index failed")
def batch_create_index_and_document():
'''
As indicated by the documentation (https://cloud.google.com/appengine/docs/standard/python/search/best_practices#batch_indexput_and_indexdelete_calls), it's possible to batch insert
documents up to 200 at a time. This is implemented here.
'''
data_gen = batch_document_data_generator()
documents = []
index = search.Index(name="Books")
for counter, data in enumerate(data_gen):
if counter % 200 == 0:
index.put(documents)
documents = []
time.sleep(2)
date_fmt = datetime.strptime(data['date'], '%Y-%M-%d')
fields = [
search.TextField(name='Author', value=data['author']),
search.DateField(name='Date', value=date_fmt),
search.TextField(name='Text', value=data['sentence']),
search.NumberField(name='Price', value=data['price'])
]
document = search.Document(doc_id=fake.ean(length=13), fields=fields)
documents.append(document)
def create_index_and_document():
'''
Called from the '/gendata' handler, it creates documents and puts them into the "Book" index.
It fills every document identity with random data from the "faker" library.
'''
data_gen = document_data_generator()
for author, text, date, num in data_gen:
date_fmt = datetime.strptime(date, '%Y-%M-%d')
index = search.Index(name="Books")
fields = [
search.TextField(name='Author', value=author),
search.DateField(name='Date', value=date_fmt),
search.TextField(name='Text', value=text),
search.NumberField(name='Price', value=num)
]
document = search.Document(doc_id=fake.ean(length=13), fields=fields)
index.put(document)
time.sleep(0.05)
def _post_put_hook(self, future):
doc = search.Document(doc_id=self.key.urlsafe(),
fields=[search.TextField(name='key', value=unicode(self._key.urlsafe())),
search.DateField(name='created_at', value=self.created_at),
search.DateField(name='updated_at', value=self.updated_at),
search.TextField(name='title', value=self.title),
search.TextField(name='artist_name',value=unicode(self.artist_name)),
search.TextField(name='artist_id', value=unicode(self.artist_id.urlsafe())),
search.TextField(name='source_file', value=self.source_file),
search.TextField(name='explicit', value=unicode(self.explicit)),
search.TextField(name='is_active', value=unicode(self.is_active)),
search.NumberField(name='average_plays', value=self.average_plays),
search.NumberField(name='weekly_plays', value=self.weekly_plays),
search.NumberField(name='total_plays', value=self.total_plays),
search.TextField(name='substrings', value=tokenize(unicode(self.title) + ' ' + unicode(self.artist_name)))])
search.Index(name=self._class_name()).put(doc)
def create_index(entity=None):
index = search.Index(name=_INDEX_NAME)
if entity:
teachers = [entity]
else:
teachers = teachme_db.teacher.query()
for i in teachers:
fields = [
search.TextField(name="Nombre", value=i.name + " " + i.lname),
search.TextField(name="Ciudad", value=i.ciudad),
search.TextField(name="Pais", value=i.pais),
search.TextField(name="Area", value=id2str_areas(i.areas)),
search.TextField(name="Tags", value=list2string(i.tags)),
search.TextField(name="key", value=i.key.urlsafe()),
search.DateField(name="Updated",
value=datetime.datetime.now().date()),
]
d = search.Document(doc_id=i.key.urlsafe(), fields=fields)
try:
index.put(d)
# print 'Indice creado'
except:
print 'No se pudo crear el indice'
def create_node_order_document(order, iyo_username):
order_id_str = '%s' % order.id
fields = [
search.AtomField(name='id', value=order_id_str),
search.AtomField(name='socket', value=order.socket),
search.NumberField(name='so', value=order.odoo_sale_order_id or -1),
search.NumberField(name='status', value=order.status),
search.DateField(name='order_time',
value=datetime.utcfromtimestamp(order.order_time)),
search.TextField(name='username', value=iyo_username),
]
if order.shipping_info:
fields.extend([
search.TextField(name='shipping_name',
value=order.shipping_info.name),
search.TextField(name='shipping_email',
value=order.shipping_info.email),
search.TextField(name='shipping_phone',
value=order.shipping_info.phone),
search.TextField(name='shipping_address',
value=order.shipping_info.address)
])
if order.billing_info:
fields.extend([
search.TextField(name='billing_name',
value=order.billing_info.name),
search.TextField(name='billing_email',
value=order.billing_info.email),
search.TextField(name='billing_phone',
value=order.billing_info.phone),
search.TextField(name='billing_address',
value=order.billing_info.address)
])
return search.Document(order_id_str, fields)
def create_post(self, title, content):
"""
Create a post.
:param title: post title
:param content: post content
:param author: author's email
:return: True / False
"""
# check input value
if title is None: return False
if content is None: return False
title = title.strip()
content = content.strip()
if len(title) == 0: return False
if len(content) == 0: return False
if g.user:
author = g.user['email']
else:
author = ''
# insert to text search
index = search.Index(name=config.text_search_name)
document = search.Document(fields=[search.TextField(name='title', value=title),
search.TextField(name='content', value=content),
search.TextField(name='author', value=author),
search.DateField(name='create_time', value=datetime.datetime.now())])
index.put(document)
return True
def job_doc(self,counter):
latitude = self.request.get('lat')
longitude = self.request.get('lon')
job_id = self.request.get('job_id')
address = self.request.get('address')
description = self.request.get('description')
skill = self.request.get('skill')
job_date = self.request.get('job_date')
job_document = search.Document(
# Setting the doc_id is optional. If omitted, the search service will create an identifier.
doc_id = str(counter),
fields=[
search.TextField(name='job_id', value=str(job_id)),
search.TextField(name='address', value=str(address)),
search.TextField(name='description', value=str(description)),
search.TextField(name='skill', value=str(skill)),
search.DateField(name='job_date', value=datetime.datetime.fromtimestamp(float(long(str(job_date))))),
search.GeoField(name='job_location', value=search.GeoPoint(float(latitude),float(longitude)))
])
documents_list.extend([job_document])
job = AJob(id = job_id, job_id = job_id, assigned = False, address = address,
description = description, skill = skill, job_date = datetime.datetime.fromtimestamp(float(long(str(job_date)))), date_posted = datetime.datetime.today())
job.put()
self.response.write(len(documents_list))
def CreateDocumentManifestDetail(manifest,booking_number,sfx, container_number,commodity,disch_port,temp,code,vents, vessel_name, voyage, port):
author = ""
"""Creates a search.Document from content written by the author."""
if author:
nickname = author.nickname().split('@')[0]
else:
nickname = 'anonymous'
# Let the search service supply the document id.
return search.Document(
fields=[search.TextField(name='author', value=nickname),
search.TextField(name="manifest", value=manifest),
search.TextField(name="booking_number", value=booking_number),
search.TextField(name="sfx", value=sfx),
search.TextField(name="container_number", value=container_number),
search.TextField(name="commodity", value=commodity),
search.TextField(name="disch_port", value=disch_port),
search.TextField(name="temp", value=temp),
search.TextField(name="code", value=code),
search.TextField(name="vents", value=vents),
search.TextField(name="vessel_name", value=vessel_name),
search.TextField(name="voyage", value=voyage),
search.TextField(name="port", value=port),
search.DateField(name="updated", value=datetime.now().date())])
def update_search_index(self, id, playground):
playground_doc = search.Document(
# As search documents cannot be updated and can only be replaced, for every create or update, we will replace the search document completely.
# Hence using the datastore id for the search id as well. Retrieving the entity also becomes easier this way.
doc_id = str(id),
# Be vey cautious on what fields are indexed, as that impacts the cost and search performance.
# Store only the fields that are required to be searched upon. For retrievel, we will retrieve the id from search and use that to query the entity from the datastore.
fields=[
search.TextField(name='name', value=playground.name),
search.TextField(name='caption', value=playground.caption),
search.TextField(name='sport', value=playground.sport),
search.TextField(name='locality', value=playground.address.locality),
search.TextField(name='locality_id', value=playground.address.locality_id),
search.TextField(name='city', value=playground.address.city),
search.NumberField(name='status', value=playground.status),
search.DateField(name='updated_on', value=datetime.date(playground.updated_on)),
search.GeoField(name='latlong', value=search.GeoPoint(playground.address.latlong.lat, playground.address.latlong.lon))
])
try:
index = search.Index(name=PLAYGROUND)
index.put(playground_doc)
logger.debug("Successfully stored playground in search index %s " % id)
except search.Error:
logger.exception('Storing playground %s in search index failed' % id)
def _buildCoreProductFields(cls, pid, name, description, category,
category_name, price):
"""Construct a 'core' document field list for the fields common to all
Products. The various categories (as defined in the file 'categories.py'),
may add additional specialized fields; these will be appended to this
core list. (see _buildProductFields)."""
fields = [
search.TextField(name=cls.PID, value=pid),
# The 'updated' field is always set to the current date.
search.DateField(name=cls.UPDATED,
value=datetime.datetime.now().date()),
search.TextField(name=cls.PRODUCT_NAME, value=name),
# strip the markup from the description value, which can
# potentially come from user input. We do this so that
# we don't need to sanitize the description in the
# templates, showing off the Search API's ability to mark up query
# terms in generated snippets. This is done only for
# demonstration purposes; in an actual app,
# it would be preferrable to use a library like Beautiful Soup
# instead.
# We'll let the templating library escape all other rendered
# values for us, so this is the only field we do this for.
search.TextField(name=cls.DESCRIPTION,
value=re.sub(r'<[^>]*?>', '', description)),
search.AtomField(name=cls.CATEGORY, value=category),
search.NumberField(name=cls.AVG_RATING, value=0.0),
search.NumberField(name=cls.PRICE, value=price)
]
return fields
def _make_fields(key, value):
"""Returns the fields corresponding to the key value pair according to the
type of value.
Args:
key: str. The name of the field.
value: *. The field value.
Returns:
list(*). A list of fields.
Raises:
ValueError. The type of field value is not list, str, Number or
datetime.
"""
if isinstance(value, list):
_validate_list(key, value)
return [_make_fields(key, v)[0] for v in value]
if isinstance(value, python_utils.BASESTRING):
return [gae_search.TextField(name=key, value=value)]
if isinstance(value, numbers.Number):
return [gae_search.NumberField(name=key, value=value)]
if isinstance(value, (datetime.datetime, datetime.date)):
return [gae_search.DateField(name=key, value=value)]
raise ValueError(
'Value for document field %s should be a (unicode) string, numeric '
'type, datetime.date, datetime.datetime or list of such types, got %s'
% (key, type(value)))
def put_index(self, data=None):
""" index the element at each"""
empty_string = lambda x: x if x else ""
organization = str(self.organization.id())
title_autocomplete = ','.join(tokenize_autocomplete(self.name))
if data:
search_key = ['infos', 'pipelines', 'tags', 'collaborators']
for key in search_key:
if key not in data.keys():
data[key] = ""
my_document = search.Document(
doc_id=str(data['id']),
fields=[
search.TextField(name=u'type', value=u'Pipeline'),
search.TextField(name='organization',
value=empty_string(organization)),
search.TextField(name='access',
value=empty_string(self.access)),
search.TextField(name='owner',
value=empty_string(self.owner)),
search.TextField(name='title',
value=empty_string(self.name)),
search.DateField(name='created_at', value=self.created_at),
search.DateField(name='updated_at', value=self.updated_at),
search.TextField(name='title_autocomplete',
value=empty_string(title_autocomplete)),
])
else:
my_document = search.Document(
doc_id=str(self.key.id()),
fields=[
search.TextField(name=u'type', value=u'Pipeline'),
search.TextField(name='organization',
value=empty_string(organization)),
search.TextField(name='access',
value=empty_string(self.access)),
search.TextField(name='owner',
value=empty_string(self.owner)),
search.TextField(name='title',
value=empty_string(self.name)),
search.DateField(name='created_at', value=self.created_at),
search.DateField(name='updated_at', value=self.updated_at),
search.TextField(name='title_autocomplete',
value=empty_string(title_autocomplete)),
])
my_index = search.Index(name="GlobalIndex")
my_index.put(my_document)