def person_doc(self,counter, body):
request_body = json.loads(self.request.body)
lat_val = request_body['lat']
latitude = float(str(lat_val))
lon_val = request_body['lon']
longitude = float(str(lon_val))
car_breakdown_area = request_body['car_breakdown_area']
safe_parking_area = request_body['safe_parking_area']
water_level_area = request_body['water_level_area']
# if len(geopoints) == 0:
# poi = latitude,longitude;
# geopoints.extend(poi)
# else:
# for eachpoi in geopoints:
# distance = geocalc(latitude,longitude,eachpoi[0],eachpoi[1])
# if distance > 1:
# geopoints.extend(poi)
my_document = search.Document(
# Setting the doc_id is optional. If omitted, the search service will create an identifier.
doc_id = str(counter),
fields=[
search.TextField(name='supplier', value='person'),
search.NumberField(name='wiper_speed', value=-100.0),
search.NumberField(name='speed', value=-100.0),
search.TextField(name='car_break_down', value=str(car_breakdown_area)),
search.TextField(name='car_parked', value=str(safe_parking_area)),
search.NumberField(name='water_level_area', value=int(water_level_area)),
search.GeoField(name='wlocation', value=search.GeoPoint(latitude,longitude))
])
documents_list.extend([my_document])
self.response.write(len(documents_list))
def car_doc(self,counter):
#body = self.request.body
#json_body = json.loads(body)
latitude = self.request.get('lat')
longitude = self.request.get('lon')
car_speed = self.request.get('car_speed')
wiper_speed = self.request.get('wiper_speed')
car_parked = self.request.get('car_parked')
water_level_area = self.request.get('water_level_area')
car_breakdown_area = self.request.get('car_breakdown_area')
#self.response.write(float(car_speed)*2)
#self.response.write(float(water_level_area)*2)
my_document = search.Document(
# Setting the doc_id is optional. If omitted, the search service will create an identifier.
doc_id = str(counter),
fields=[
search.TextField(name='supplier', value='car'),
search.NumberField(name='speed', value=float(car_speed)),
search.TextField(name='car_break_down', value=str(car_breakdown_area)),
search.NumberField(name='wiper_speed', value=float(wiper_speed)),
search.TextField(name='car_parked', value=str(car_parked)),
search.NumberField(name='water_level_area', value=int(water_level_area)),
search.GeoField(name='wlocation', value=search.GeoPoint(float(latitude),float(longitude)))
])
documents_list.extend([my_document])
s = '%s, %s' % (len(documents_list), counter)
self.response.write(s)
def _create_doc_event(cls, studio_class):
title = '%s: %s' % (studio_class.style, studio_class.teacher)
description = studio_class.teacher_link
# include twitter link to studio?
timestamp = min(int(time.mktime(studio_class.start_time.timetuple())),
2**31 - 1)
doc_event = search.Document(
doc_id=studio_class.key.string_id(),
fields=[
search.TextField(name='name', value=title),
search.TextField(name='studio',
value=studio_class.studio_name),
search.TextField(name='source_page',
value=studio_class.source_page),
search.TextField(name='description', value=description),
search.DateField(name='start_time',
value=studio_class.start_time),
search.DateField(name='end_time', value=studio_class.end_time),
search.NumberField(name='latitude',
value=studio_class.latitude),
search.NumberField(name='longitude',
value=studio_class.longitude),
search.TextField(name='categories',
value=' '.join(studio_class.auto_categories)),
search.TextField(name='sponsor', value=studio_class.sponsor),
#search.TextField(name='country', value=studio_class.country),
],
#language=XX, # We have no good language detection
rank=timestamp,
)
return doc_event
def CreateDocument(text,sheet, row, col):
return search.Document(
fields=[search.TextField(name='text', value=text),
search.TextField(name='sheet', value=sheet),
search.NumberField(name='row', value=row),
search.NumberField(name='col', value=col),
search.DateField(name='date', value=datetime.now().date())])
def add_to_number_index(author,
number_id,
number,
units,
description,
labels,
source,
year,
month,
day,
contained_in_series=""):
return search.Index(name=_INDEX_NAME).put(
search.Document(doc_id=number_id,
fields=[
search.TextField(name='author', value=author),
search.NumberField(name='number', value=number),
search.TextField(name='units', value=units),
search.TextField(name='description',
value=description),
search.TextField(name='labels', value=labels),
search.TextField(name='source', value=source),
search.NumberField(name='year_of_number',
value=year),
search.NumberField(name='month_of_number',
value=month),
search.NumberField(name='day_of_number',
value=day),
search.TextField(name='contained_in_series',
value=contained_in_series)
]))[0].id
def create_node_order_document(order, iyo_username):
order_id_str = '%s' % order.id
fields = [
search.AtomField(name='id', value=order_id_str),
search.AtomField(name='socket', value=order.socket),
search.NumberField(name='so', value=order.odoo_sale_order_id or -1),
search.NumberField(name='status', value=order.status),
search.DateField(name='order_time',
value=datetime.utcfromtimestamp(order.order_time)),
search.TextField(name='username', value=iyo_username),
]
if order.shipping_info:
fields.extend([
search.TextField(name='shipping_name',
value=order.shipping_info.name),
search.TextField(name='shipping_email',
value=order.shipping_info.email),
search.TextField(name='shipping_phone',
value=order.shipping_info.phone),
search.TextField(name='shipping_address',
value=order.shipping_info.address)
])
if order.billing_info:
fields.extend([
search.TextField(name='billing_name',
value=order.billing_info.name),
search.TextField(name='billing_email',
value=order.billing_info.email),
search.TextField(name='billing_phone',
value=order.billing_info.phone),
search.TextField(name='billing_address',
value=order.billing_info.address)
])
return search.Document(order_id_str, fields)
def _buildCoreProductFields(cls, pid, name, description, category,
category_name, price):
"""Construct a 'core' document field list for the fields common to all
Products. The various categories (as defined in the file 'categories.py'),
may add additional specialized fields; these will be appended to this
core list. (see _buildProductFields)."""
fields = [
search.TextField(name=cls.PID, value=pid),
# The 'updated' field is always set to the current date.
search.DateField(name=cls.UPDATED,
value=datetime.datetime.now().date()),
search.TextField(name=cls.PRODUCT_NAME, value=name),
# strip the markup from the description value, which can
# potentially come from user input. We do this so that
# we don't need to sanitize the description in the
# templates, showing off the Search API's ability to mark up query
# terms in generated snippets. This is done only for
# demonstration purposes; in an actual app,
# it would be preferrable to use a library like Beautiful Soup
# instead.
# We'll let the templating library escape all other rendered
# values for us, so this is the only field we do this for.
search.TextField(name=cls.DESCRIPTION,
value=re.sub(r'<[^>]*?>', '', description)),
search.AtomField(name=cls.CATEGORY, value=category),
search.NumberField(name=cls.AVG_RATING, value=0.0),
search.NumberField(name=cls.PRICE, value=price)
]
return fields
def _build_event_date(i, event, ed, venue, start, end, is_hours=False):
"""
Helper to create a specific date - yeilds one search doc
"""
category = ed.category
if is_hours:
category = CATEGORY.HOURS
fields = []
doc_id = '%s-%s' % (event.slug, i)
fields.append(search.TextField(name='name', value=event.name))
fields.append(search.AtomField(name='slug', value=event.slug))
fields.append(search.AtomField(name='event_keystr', value=str(event.key.urlsafe())))
# Populate bits specific to the event date
fields.append(search.NumberField(name='start', value=unix_time(timezone('UTC').localize(start))))
fields.append(search.NumberField(name='end', value=unix_time(timezone('UTC').localize(end))))
fields.append(search.AtomField(name='category', value=category))
# Attach Venue/Geo Information
fields.append(search.AtomField(name='venue_slug', value=ed.venue_slug))
venue_geo = None
if venue.geo:
geoPt = venue.geo
if isinstance(geoPt, list):
geoPt = geoPt[0]
venue_geo = search.GeoPoint(geoPt.lat, geoPt.lon)
fields.append(search.GeoField(name='venue_geo', value=venue_geo))
return search.Document(doc_id=doc_id, fields=fields)
def _create_doc_event(cls, db_event):
if not db_event.has_content():
return None
# TODO(lambert): find a way to index no-location events.
# As of now, the lat/long number fields cannot be None.
# In what radius/location should no-location events show up
# and how do we want to return them
# Perhaps a separate index that is combined at search-time?
if db_event.latitude is None:
logging.warning("Skipping event without latitude: %s", db_event.id)
return None
# If this event has been deleted from Facebook, let's skip re-indexing it here
if db_event.start_time is None:
logging.warning("Skipping event without start_time: %s",
db_event.id)
return None
if not isinstance(db_event.start_time,
datetime.datetime) and not isinstance(
db_event.start_time, datetime.date):
logging.error("DB Event %s start_time is not correct format: ",
db_event.id, db_event.start_time)
return None
timestamp = min(int(time.mktime(db_event.start_time.timetuple())),
2**31 - 1)
doc_event = search.Document(
doc_id=db_event.id,
fields=[
search.TextField(name='name', value=db_event.name),
search.TextField(name='description',
value=db_event.description),
search.NumberField(name='attendee_count',
value=db_event.attendee_count or 0),
search.DateField(name='start_time', value=db_event.start_time),
search.DateField(name='end_time',
value=dates.faked_end_time(
db_event.start_time, db_event.end_time)),
search.NumberField(name='latitude', value=db_event.latitude),
search.NumberField(name='longitude', value=db_event.longitude),
search.TextField(name='categories',
value=' '.join(db_event.auto_categories)),
search.TextField(name='country', value=db_event.country),
# Use NumberField instead of DateField since we care about hours/minutes/seconds,
# which are otherwise discarded with a DateField.
# We want this to know what events have been added in the last 24 hours,
# so we can promote them to users when we send out daily notifications.
search.NumberField(
name='creation_time',
value=int(time.mktime(db_event.creation_time.timetuple()))
if db_event.creation_time else 0),
],
# TODO: Re-enable this. We disabled it because:
# - some langauges are like 'zh-Hant', but this expects two-letter languages
# - some events don't have a 'language' field in their json_props....why/how?!?
# language=db_event.json_props['language'],
rank=timestamp,
)
return doc_event
def add_log_lines(log_lines):
data = []
for log_line in log_lines:
fields = [
search.TextField(name='line', value=log_line.line),
search.TextField(name='server_key',
value=log_line.server_key.urlsafe())
]
if log_line.timestamp is not None:
fields.append(
search.DateField(name='timestamp',
value=log_line.timestamp.date()
if log_line.timestamp else None))
fields.append(
search.TextField(
name='timestamp_string',
value=log_line.timestamp.strftime('%Y-%m-%d %H:%M:%S')))
fields.append(
search.NumberField(name='timestamp_sse',
value=log_line.timestamp_sse))
if log_line.log_level:
fields.append(
search.TextField(name='log_level', value=log_line.log_level))
if log_line.username:
fields.append(
search.TextField(name='username', value=log_line.username))
if log_line.location:
fields += [
search.NumberField(
name='location_x',
value=log_line.location.x if log_line.location else None),
search.NumberField(
name='location_y',
value=log_line.location.y if log_line.location else None),
search.NumberField(
name='location_z',
value=log_line.location.z if log_line.location else None)
]
if log_line.chat:
fields.append(search.TextField(name='chat', value=log_line.chat))
if log_line.death_message:
fields.append(
search.TextField(name='death_message',
value=log_line.death_message))
if log_line.achievement:
fields.append(
search.TextField(name='achievement',
value=log_line.achievement))
if log_line.tags:
fields.append(
search.TextField(
name='tags',
value=' '.join(log_line.tags) if log_line.tags else ''))
data.append((log_line.key, fields))
return add_to_index(log_line_index, data)
def put_index(self, data=None):
""" index the element at each"""
empty_string = lambda x: x if x else ""
collaborators = " ".join(self.collaborators_ids)
organization = str(self.organization.id())
title_autocomplete = ','.join(tokenize_autocomplete(
self.name + ' ' + empty_string(self.account_name) + ' ' + empty_string(self.contact_name)))
if data:
search_key = ['infos', 'cases', 'tags', 'collaborators']
for key in search_key:
if key not in data.keys():
data[key] = ""
my_document = search.Document(
doc_id=str(data['id']),
fields=[
search.TextField(name=u'type', value=u'Case'),
search.TextField(name='organization', value=empty_string(organization)),
search.TextField(name='access', value=empty_string(self.access)),
search.TextField(name='owner', value=empty_string(self.owner)),
search.TextField(name='collaborators', value=data['collaborators']),
search.TextField(name='title', value=empty_string(self.name)),
search.TextField(name='account_name', value=empty_string(self.account_name)),
search.TextField(name='contact_name', value=empty_string(self.contact_name)),
search.TextField(name='status', value=empty_string(self.status)),
search.NumberField(name='priority', value=int(self.priority or 1)),
search.DateField(name='created_at', value=self.created_at),
search.DateField(name='updated_at', value=self.updated_at),
search.TextField(name='infos', value=data['infos']),
search.TextField(name='tags', value=data['tags']),
search.TextField(name='cases', value=data['cases']),
search.TextField(name='title_autocomplete', value=empty_string(title_autocomplete)),
search.TextField(name='type_case', value=empty_string(self.type_case))
])
else:
my_document = search.Document(
doc_id=str(self.key.id()),
fields=[
search.TextField(name=u'type', value=u'Case'),
search.TextField(name='organization', value=empty_string(organization)),
search.TextField(name='access', value=empty_string(self.access)),
search.TextField(name='owner', value=empty_string(self.owner)),
search.TextField(name='collaborators', value=collaborators),
search.TextField(name='title', value=empty_string(self.name)),
search.TextField(name='account_name', value=empty_string(self.account_name)),
search.TextField(name='contact_name', value=empty_string(self.contact_name)),
search.TextField(name='status', value=empty_string(self.status)),
search.NumberField(name='priority', value=int(self.priority)),
search.DateField(name='created_at', value=self.created_at),
search.DateField(name='updated_at', value=self.updated_at),
search.TextField(name='title_autocomplete', value=empty_string(title_autocomplete)),
search.TextField(name='type_case', value=empty_string(self.type_case))
])
my_index = search.Index(name="GlobalIndex")
my_index.put(my_document)
def _create_doc_event(cls, source):
fb_info = source.fb_info
if not fb_info:
return None
# Only index fan pages for now:
# - Profiles are not public or indexable.
# - Groups don't have a location. (but they do!?)
# - Fan Pages are both.
if source.graph_type != thing_db.GRAPH_TYPE_FANPAGE:
return None
# TODO(lambert): find a way to index no-location sources.
# As of now, the lat/long number fields cannot be None.
# In what radius/location should no-location sources show up
# and how do we want to return them
# Perhaps a separate index that is combined at search-time?
if fb_info.get('location', None) is None:
return None
if not source.latitude:
return None
country = fb_info['location'].get('country', '').upper()
if country in iso3166.countries_by_name:
country_code = iso3166.countries_by_name[country].alpha2
else:
country_code = None
doc_event = search.Document(
doc_id=source.graph_id,
fields=[
search.TextField(name='name', value=source.name),
search.TextField(name='description',
value=fb_info.get('general_info', '')),
search.NumberField(name='like_count', value=fb_info['likes']),
search.TextField(name='category', value=fb_info['category']),
search.TextField(
name='category_list',
value=', '.join(
str(x['name'])
for x in fb_info.get('category_list', []))),
search.NumberField(name='latitude', value=source.latitude),
search.NumberField(name='longitude', value=source.longitude),
#search.TextField(name='categories', value=' '.join(source.auto_categories)),
search.TextField(name='country', value=country_code),
search.NumberField(name='num_real_events',
value=source.num_real_events or 0),
],
#language=XX, # We have no good language detection
rank=source.num_real_events or 0,
)
return doc_event
def put_index(self, data=None):
""" index the element at each"""
empty_string = lambda x: x if x else ""
collaborators = " ".join(self.collaborators_ids)
organization = str(self.organization.id())
if data:
search_key = ['infos', 'events', 'tags']
for key in search_key:
if key not in data.keys():
data[key] = ""
my_document = search.Document(
doc_id=str(data['id']),
fields=[
search.TextField(name=u'type', value=u'Event'),
search.TextField(name='organization', value=empty_string(organization)),
search.TextField(name='access', value=empty_string(self.access)),
search.TextField(name='owner', value=empty_string(self.owner)),
search.TextField(name='collaborators', value=collaborators),
search.TextField(name='where', value=empty_string(self.where)),
search.TextField(name='description', value=empty_string(self.description)),
search.TextField(name='title', value=empty_string(self.title)),
search.DateField(name='created_at', value=self.created_at),
search.DateField(name='updated_at', value=self.updated_at),
search.TextField(name='infos', value=data['infos']),
search.TextField(name='tags', value=data['tags']),
search.TextField(name='events', value=data['events']),
search.NumberField(name='comments', value=self.comments),
search.TextField(name='about_kind', value=empty_string(self.about_kind)),
search.TextField(name='about_item', value=empty_string(self.about_item)),
])
else:
my_document = search.Document(
doc_id=str(self.key.id()),
fields=[
search.TextField(name=u'type', value=u'Event'),
search.TextField(name='organization', value=empty_string(organization)),
search.TextField(name='access', value=empty_string(self.access)),
search.TextField(name='owner', value=empty_string(self.owner)),
search.TextField(name='collaborators', value=collaborators),
search.TextField(name='where', value=empty_string(self.where)),
search.TextField(name='description', value=empty_string(self.description)),
search.TextField(name='title', value=empty_string(self.title)),
search.DateField(name='created_at', value=self.created_at),
search.DateField(name='updated_at', value=self.updated_at),
search.NumberField(name='comments', value=self.comments),
search.TextField(name='about_kind', value=empty_string(self.about_kind)),
search.TextField(name='about_item', value=empty_string(self.about_item)),
])
my_index = search.Index(name="GlobalIndex")
my_index.put(my_document)
def __init__(self, *args, **kwargs):
super(Map, self).__init__(fields=[
search.GeoField(name="location",
value=search.GeoPoint(float(kwargs['latitude']),
float(kwargs['longitude']))),
search.TextField(name="building_name",
value=kwargs['building_name']),
search.TextField(name="level_name", value=kwargs['level_name']),
search.NumberField(name="floor_number",
value=int(kwargs['floor_number'])),
search.NumberField(name="scale", value=float(kwargs['scale'])),
search.TextField(name="path", value=kwargs['path'])
])
def update_search_index(self, id, playground):
playground_doc = search.Document(
# As search documents cannot be updated and can only be replaced, for every create or update, we will replace the search document completely.
# Hence using the datastore id for the search id as well. Retrieving the entity also becomes easier this way.
doc_id = str(id),
# Be vey cautious on what fields are indexed, as that impacts the cost and search performance.
# Store only the fields that are required to be searched upon. For retrievel, we will retrieve the id from search and use that to query the entity from the datastore.
fields=[
search.TextField(name='name', value=playground.name),
search.TextField(name='caption', value=playground.caption),
search.TextField(name='sport', value=playground.sport),
search.TextField(name='locality', value=playground.address.locality),
search.TextField(name='locality_id', value=playground.address.locality_id),
search.TextField(name='city', value=playground.address.city),
search.NumberField(name='status', value=playground.status),
search.DateField(name='updated_on', value=datetime.date(playground.updated_on)),
search.GeoField(name='latlong', value=search.GeoPoint(playground.address.latlong.lat, playground.address.latlong.lon))
])
try:
index = search.Index(name=PLAYGROUND)
index.put(playground_doc)
logger.debug("Successfully stored playground in search index %s " % id)
except search.Error:
logger.exception('Storing playground %s in search index failed' % id)
def get_field(self,key,value):
try:
if(self.mapping[key]):
if(self.mapping[key]=='TEXTFIELD'):
return search.TextField(name=key,value=value);
elif(self.mapping[key]=='ATOMFIELD'):
return search.AtomField(name=key,value=value);
elif(self.mapping[key]=='NUMBERFIELD'):
return search.NumberField(name=key,value=value);
elif(self.mapping[key]=='HTMLFIELD'):
return search.HtmlField(name=key,value=value);
elif(self.mapping[key]=='DATEFIELD'):
timestamp=value;
timestamp = timestamp if timestamp>0 else -timestamp;
value=datetime.fromtimestamp(timestamp/1000.0);
return search.DateField(name=key,value=value);
elif(self.mapping[key]=='GEOFIELD'):
return search.GeoField(name=key,value=value);
else:
return None;
else:
return None;
except KeyError,keyError:
print(keyError);
return None;
def _make_fields(key, value):
"""Returns the fields corresponding to the key value pair according to the
type of value.
Args:
key: str. The name of the field.
value: *. The field value.
Returns:
list(*). A list of fields.
Raises:
ValueError. The type of field value is not list, str, Number or
datetime.
"""
if isinstance(value, list):
_validate_list(key, value)
return [_make_fields(key, v)[0] for v in value]
if isinstance(value, python_utils.BASESTRING):
return [gae_search.TextField(name=key, value=value)]
if isinstance(value, numbers.Number):
return [gae_search.NumberField(name=key, value=value)]
if isinstance(value, (datetime.datetime, datetime.date)):
return [gae_search.DateField(name=key, value=value)]
raise ValueError(
'Value for document field %s should be a (unicode) string, numeric '
'type, datetime.date, datetime.datetime or list of such types, got %s'
% (key, type(value)))
def index():
html = get_file()
products = json.loads(html)
index = search.Index('products')
#Should really batch here
current_docs = []
count = 0
index_count = 1
print(str(len(products)))
for product in products:
fields = [
search.NumberField(name='sku', value=product['sku']),
search.TextField(name='name', value=product['name']),
search.TextField(name='description', value=product['description']),
search.TextField(name='url', value=product['url']),
search.TextField(name='image', value=product['image'])
]
doc = search.Document(doc_id=str(product['sku']), fields=fields)
current_docs.append(doc)
count = count + 1
if count > 199:
print("Index " + str(index_count) + "," + str(count))
index_count = index_count + 1
index.put(current_docs)
current_docs = []
count = 0
time.sleep(0.5) # Without safety quota lift need to slow this down
print("Final Index!!")
index.put(current_docs)
return "Complete"
def index_artifact(index_, id_, fields):
f = []
for i in xrange(0, len(fields), 3):
if fields[i] == ATOMFIELD:
f.append(search.AtomField(name=fields[i + 1], value=fields[i + 2]))
elif fields[i] == TEXTFIELD:
f.append(search.TextField(name=fields[i + 1], value=fields[i + 2]))
elif fields[i] == HTMLFIELD:
f.append(search.HtmlField(name=fields[i + 1], value=fields[i + 2]))
elif fields[i] == NUMBERFIELD:
f.append(
search.NumberField(name=fields[i + 1], value=fields[i + 2]))
elif fields[i] == DATEFIELD:
f.append(search.DateField(name=fields[i + 1], value=fields[i + 2]))
elif fields[i] == GEOPOINTFIELD:
f.append(search.GeoField(name=fields[i + 1], value=fields[i + 2]))
doc = search.Document(doc_id=id_, fields=f)
retry_count = 0
while True:
try:
index_.put(doc)
break
except search.Error as e:
if retry_count < 3:
log.warning(
'Error put doc into index, could be out of space. Creating new index'
)
index_ = search.Index(index_.name[:-4] +
str(int(index_.name[-4:])).zfill(4),
namespace=index_.namespace)
retry_count += 1
else:
raise e
def create_index_and_document():
'''
Called from the '/gendata' handler, it creates documents and puts them into the "Book" index.
It fills every document identity with random data from the "faker" library.
'''
data_gen = document_data_generator()
for author, text, date, num in data_gen:
date_fmt = datetime.strptime(date, '%Y-%M-%d')
index = search.Index(name="Books")
fields = [
search.TextField(name='Author', value=author),
search.DateField(name='Date', value=date_fmt),
search.TextField(name='Text', value=text),
search.NumberField(name='Price', value=num)
]
document = search.Document(doc_id=fake.ean(length=13), fields=fields)
index.put(document)
time.sleep(0.05)
def batch_create_index_and_document():
'''
As indicated by the documentation (https://cloud.google.com/appengine/docs/standard/python/search/best_practices#batch_indexput_and_indexdelete_calls), it's possible to batch insert
documents up to 200 at a time. This is implemented here.
'''
data_gen = batch_document_data_generator()
documents = []
index = search.Index(name="Books")
for counter, data in enumerate(data_gen):
if counter % 200 == 0:
index.put(documents)
documents = []
time.sleep(2)
date_fmt = datetime.strptime(data['date'], '%Y-%M-%d')
fields = [
search.TextField(name='Author', value=data['author']),
search.DateField(name='Date', value=date_fmt),
search.TextField(name='Text', value=data['sentence']),
search.NumberField(name='Price', value=data['price'])
]
document = search.Document(doc_id=fake.ean(length=13), fields=fields)
documents.append(document)
def _post_put_hook(self, future):
doc = search.Document(doc_id=self.key.urlsafe(),
fields=[search.TextField(name='key', value=unicode(self._key.urlsafe())),
search.DateField(name='created_at', value=self.created_at),
search.DateField(name='updated_at', value=self.updated_at),
search.TextField(name='title', value=self.title),
search.TextField(name='artist_name',value=unicode(self.artist_name)),
search.TextField(name='artist_id', value=unicode(self.artist_id.urlsafe())),
search.TextField(name='source_file', value=self.source_file),
search.TextField(name='explicit', value=unicode(self.explicit)),
search.TextField(name='is_active', value=unicode(self.is_active)),
search.NumberField(name='average_plays', value=self.average_plays),
search.NumberField(name='weekly_plays', value=self.weekly_plays),
search.NumberField(name='total_plays', value=self.total_plays),
search.TextField(name='substrings', value=tokenize(unicode(self.title) + ' ' + unicode(self.artist_name)))])
search.Index(name=self._class_name()).put(doc)
def _indexIssue(issue, updates):
"""Does the actual work of indexing the given issue. We expect to be called in a deferred handler."""
fields = [
search.TextField(name="summary", value=issue.summary),
search.TextField(name="description", value=issue.description),
search.AtomField(name="id", value=str(issue.key().id_or_name())),
search.AtomField(name="type", value=issue.type),
search.NumberField(name="priority", value=issue.priority),
search.AtomField(name="state", value=issue.state),
search.AtomField(name="resolution", value=issue.resolution)
]
if not updates:
issue, updates = getIssue(issue.key().id())
comments = ""
for update in updates:
if update.comment:
if comments:
comments += "\r\n<hr />\r\n"
comments += update.comment
fields.append(search.HtmlField(name="comments", value=comments))
doc = search.Document(doc_id=str(issue.key()), fields=fields)
index = search.Index(name="issue")
index.put(doc)
def add_new_donation(cls,
title,
quantity,
city,
country,
state,
comments,
photo_key,
photo_url,
user_key=None):
user_id = str(user_key.id())
donation_key = cls(title=title,
quantity=quantity,
city=city,
country=country,
state=state,
comments=comments,
photo_key=photo_key,
photo_url=photo_url,
user=user_key).put()
index = search.Index('donation')
doc = search.Document(
doc_id=str(donation_key.id()),
fields=[
search.TextField(name='title', value=title),
search.TextField(name='city', value=city),
search.TextField(name='country', value=country),
search.TextField(name='state', value=state),
search.TextField(name='comments', value=comments),
search.NumberField(name='quantity', value=quantity),
search.TextField(name='photo_url', value=photo_url),
search.TextField(name='user_id', value=user_id)
],
)
index.put(doc)
def create_investment_agreement_document(investment, iyo_username):
# type: (InvestmentAgreement) -> search.Document
investment_id_str = str(investment.id)
fields = [
search.AtomField(name='id', value=investment_id_str),
search.AtomField(name='reference', value=investment.reference),
search.NumberField(name='status', value=investment.status),
search.TextField(name='username', value=iyo_username),
search.DateField(name='creation_time',
value=datetime.utcfromtimestamp(
investment.creation_time)),
search.TextField(name='name', value=investment.name),
search.TextField(name='address',
value=investment.address
and investment.address.replace('\n', '')),
search.TextField(name='currency', value=investment.currency),
]
if investment.amount:
fields.append(
search.TextField(name='amount',
value=_stringify_float(investment.amount)))
if investment.token_count:
fields.append(
search.TextField(name='token_count',
value=_stringify_float(
investment.token_count_float)))
return search.Document(doc_id=investment_id_str, fields=fields)
def _buildProductFields(cls,
pid=None,
category=None,
name=None,
user_id=None,
description=None,
category_name=None,
image_url=None,
price=None,
ppacc=None,
**params):
"""Build all the additional non-core fields for a document of the given
product type (category), using the given params dict, and the
already-constructed list of 'core' fields. All such additional
category-specific fields are treated as required.
"""
fields = cls._buildCoreProductFields(pid, name, user_id, description,
category, category_name,
image_url, price, ppacc)
# get the specification of additional (non-'core') fields for this category
pdict = categories.product_dict.get(category_name)
if pdict:
# for all fields
for k, field_type in pdict.iteritems():
# see if there is a value in the given params for that field.
# if there is, get the field type, create the field, and append to the
# document field list.
if k in params:
v = params[k]
if field_type == search.NumberField:
try:
val = float(v)
fields.append(search.NumberField(name=k,
value=val))
except ValueError:
error_message = (
'bad value %s for field %s of type %s' %
(k, v, field_type))
logging.error(error_message)
raise errors.OperationFailedError(error_message)
elif field_type == search.TextField:
fields.append(search.TextField(name=k, value=str(v)))
else:
# you may want to add handling of other field types for generality.
# Not needed for our current sample data.
logging.warn('not processed: %s, %s, of type %s', k, v,
field_type)
else:
error_message = (
'value not given for field "%s" of field type "%s"' %
(k, field_type))
logging.warn(error_message)
raise errors.OperationFailedError(error_message)
else:
# else, did not have an entry in the params dict for the given field.
logging.warn(
'product field information not found for category name %s',
params['category_name'])
return fields
def update_search_index(self, id, event):
event_doc = search.Document(
# As search documents cannot be updated and can only be replaced, for every create or update, we will replace the search document completely.
# Hence using the datastore id for the search id as well. Retrieving the entity also becomes easier this way.
doc_id=str(id),
# Store only the fields that are required to be searched/sorted upon. For retrievel, we will retrieve the id from search and use that to query the entity from the datastore.
fields=[
search.TextField(name='name', value=event.name),
search.TextField(name='caption', value=event.caption),
search.TextField(name='sport', value=event.sport),
search.TextField(name='locality',
value=event.address.locality),
search.TextField(name='locality_id',
value=event.address.locality_id),
search.TextField(name='city', value=event.address.city),
search.DateField(name='start_datetime',
value=event.start_datetime),
search.DateField(name='end_datetime',
value=event.end_datetime),
search.NumberField(name='status', value=event.status),
search.DateField(name='updated_on',
value=datetime.date(event.updated_on)),
search.GeoField(name='latlong',
value=search.GeoPoint(
event.address.latlong.lat,
event.address.latlong.lon))
])
try:
index = search.Index(name=EVENT)
index.put(event_doc)
except search.Error:
logger.exception('Storing event %s in search index failed' % id)
def getSearchDocumentFields(self, valuesCache, name, prefix=""):
if isinstance(valuesCache.get(name), int) or isinstance(
valuesCache.get(name), float):
return [
search.NumberField(name=prefix + name, value=valuesCache[name])
]
return []
def create_player(player):
# logging.debug(player)
player = search.Document(doc_id=str(player.key.id()),
fields=[
search.TextField(name='name',
value=player.name),
search.NumberField(name='age',
value=player.age),
search.TextField(name='national',
value=player.national),
search.TextField(name='position',
value=player.position),
search.NumberField(name='salary',
value=player.salary)
])
return player
def _buildCoreVideoFields(cls, unique_id, title, category, subtitle,
duration_min, speakers, description,
published_date, views, slides_link, tags, image,
video_id, session_id):
"""Construct a 'core' document field list for the fields common to all
Videos. The various categories (as defined in the file 'categories.py'),
may add additional specialized fields; these will be appended to this
core list. (see _buildVideoFields)."""
fields = [
search.TextField(name=cls.UNIQUEID, value=unique_id),
# The 'updated' field is always set to the current date.
search.DateField(name=cls.UPDATED,
value=datetime.datetime.now().date()),
search.TextField(name=cls.TITLE, value=title),
search.AtomField(name=cls.CATEGORY, value=category),
search.TextField(name=cls.SUBTITLE, value=subtitle),
search.NumberField(name=cls.DURATION_MIN, value=int(duration_min)),
search.TextField(name=cls.SPEAKERS, value=ScrubSpeakers(speakers)),
# strip the markup from the description value, which can
# potentially come from user input. We do this so that
# we don't need to sanitize the description in the
# templates, showing off the Search API's ability to mark up query
# terms in generated snippets. This is done only for
# demonstration purposes; in an actual app,
# it would be preferrable to use a library like Beautiful Soup
# instead.
# We'll let the templating library escape all other rendered
# values for us, so this is the only field we do this for.
search.TextField(name=cls.DESCRIPTION,
value=ScrubDescription(description)),
search.NumberField(name=cls.VIEWS, value=int(views)),
search.TextField(name=cls.SLIDES_LINK, value=slides_link),
search.TextField(name=cls.TAGS, value=tags),
search.TextField(name=cls.IMAGE, value=image),
search.TextField(name=cls.VID, value=video_id),
search.TextField(name=cls.SESSIONID, value=session_id),
search.NumberField(name=cls.AVG_RATING, value=0.0),
]
# Some fields can sometimes be empty.
scrubbed_publish_date = utils.dateFromDateString(published_date)
if scrubbed_publish_date:
fields.append(
search.DateField(name=cls.PUBLISHED_DATE,
value=scrubbed_publish_date))
return fields
