def bulk_dereference_posts(posts):
"""
Bulk dereference author and image. This is to increase performance for rss and rest collections
"""
# Step 1: Collect all the resources we need and prep the map
entity_map = {}
for post in posts:
# Default the dereferenced prop placeholders
setattr(post, AUTHOR_PROP, None)
setattr(post, PRIMARY_IMAGE_PROP, None)
setattr(post, CATEGORY_PROP, None)
# Collect properties we want to bulk dereference
if post.author_resource_id:
entity_map[get_key_from_resource_id(
post.author_resource_id)] = None
if post.primary_image_resource_id:
entity_map[get_key_from_resource_id(
post.primary_image_resource_id)] = None
if post.category_resource_id:
entity_map[get_key_from_resource_id(
post.category_resource_id)] = None
# Fetch all of the entities we want to deref
entities = ndb.get_multi(entity_map.keys())
# Repopulate the map NOTE: This adds keys to map using resource_id rather than key
for entity in entities:
entity_map[get_resource_id_from_key(entity.key)] = entity
# Step 3: Iterate over posts and link up the dereferenced props
for post in posts:
if post.author_resource_id:
e = entity_map.get(post.author_resource_id, None)
setattr(post, AUTHOR_PROP, e)
if post.primary_image_resource_id:
e = entity_map.get(post.primary_image_resource_id, None)
setattr(post, PRIMARY_IMAGE_PROP, e)
if post.category_resource_id:
e = entity_map.get(post.category_resource_id, None)
setattr(post, CATEGORY_PROP, e)
return posts
def get_post_category_by_resource_id(resource_id):
"""
Given a resource id, fetch the category entity
TODO: Ensure that it is of kind BlogCategory
"""
key = get_key_from_resource_id(resource_id)
return key.get()
def from_resource(self, obj, field):
"""
Resolve a REST resource from an entity
"""
resource_id = super(ResourceField,
self).from_resource(obj, self.resource_id_prop)
if not resource_id:
return None
# Resolve Entity
resource_entity = None
if hasattr(obj, self.key):
resource_entity = getattr(obj, self.key, None)
else:
logging.error('Reference prop `%s` was not bulk dereferenced.' %
self.key)
try:
resource_key = get_key_from_resource_id(resource_id)
resource_entity = resource_key.get()
except ValueError:
logging.error(
'Failed to convert resource id %s to a key for prop %s.' %
(resource_id, self.key))
logging.error(obj)
return None
return Resource(resource_entity, self.resource_rules).to_dict()
def get(self):
# Serialize the params for cache key
cache_key = 'super_featured-resourcesx'
cached_events = ubercache.cache_get(cache_key)
if cached_events:
results = cached_events
else:
gs = get_global_settings()
resource_ids = json.loads(gs.FEATURED_HEADER_RESOURCES)
results = []
for resource_id in resource_ids:
key = get_key_from_resource_id(resource_id)
entity = key.get()
if entity:
resource = None
kind = entity.key.kind()
if kind == 'Event':
resource = create_event_resource(entity)
if kind == 'BlogPost':
resource = create_blogpost_resource(entity)
if resource:
results.append(resource)
ubercache.cache_set(cache_key, results, category='events')
# Finally...
self.serve_success(results)
def get_post_by_resource_id(resource_id):
"""
Given a resource id, fetch the post entity
#TODO: Error handling?
TODO: Ensure that it is of kind BlogCategory
"""
key = get_key_from_resource_id(resource_id)
return key.get()
def test_get_triple(self):
"""
Additional Test Case to ensure that we can do more than 3 pairs
"""
result = utils.get_key_from_resource_id(
'UGFyZW50Hh8xMjMeQ2hpbGQeHzQ1Nh5HcmFuZB4fNzg5')
self.assertEqual(result,
ndb.Key('Parent', 123, 'Child', 456, 'Grand', 789))
def test_base_case(self):
"""
Test general conversion including int ids
"""
result = utils.get_key_from_resource_id(
'UGFyZW50Hh8xMjMeQ2hpbGQeZWwtbmlcdTIwOTlv')
self.assertEqual(result, ndb.Key('Parent', 123, 'Child',
'el-ni\u2099o'))
def _get(self, resource_id):
f_key = get_key_from_resource_id(resource_id)
# TODO: Abstract this a bit more out into a rest-like service...
f = f_key.get()
if not f:
raise Exception('File Not Found')
self.serve_success(Resource(f, self.get_rules()).to_dict())
def _put(self, resource_id):
"""
Edit a User
"""
key = get_key_from_resource_id(resource_id)
e = key.get()
e = auth_api.edit_user(e, self.cleaned_data)
self.serve_success(Resource(e, REST_RULES).to_dict())
def get_event_entity_or_404(self, resource_id):
key = get_key_from_resource_id(resource_id)
if not key:
raise Exception(
'404 - TODO: Throw legit 404') # or Resource Not Found
e = key.get()
if not e:
raise Exception(
'404 - TODO: Throw legit 404') # or Resource Not Found
return e
def _get(self, slug):
# TODO: Abstract this a bit more out into a rest-like service...
resource_id = slug
key = get_key_from_resource_id(resource_id) #ndb.Key(urlsafe=slug)
e = key.get()
#e = venues_api.get_venue_by_slug(slug)
if not e:
self.serve_404('Gallery Not Found')
return False
resource = create_resource_from_entity(e)
self.serve_success(resource)
def from_resource(self, obj, field):
"""
"""
resource_id = super(FileField,
self).from_resource(obj, self.resource_id_prop)
if not resource_id:
return None
try:
resource_key = get_key_from_resource_id(resource_id)
except ValueError:
logging.error('Invalid Resource Id %s' % resource_id)
return None
return Resource(resource_key.get(), REST_RESOURCE_RULES).to_dict()
def _post(self, resource_id):
f_key = get_key_from_resource_id(resource_id)
f = f_key.get()
if not f:
raise Exception('File Not Found')
do_put = False # Only do put if something changed
for prop, val in self.cleaned_data.items():
if getattr(f, prop) == val:
continue
do_put = True
setattr(f, prop, val)
if do_put:
f.put()
self.serve_success(Resource(f, self.get_rules()).to_dict())
def _put(self, slug):
"""
Edit a resource
TODO: None of the data is validated right now...
"""
"""
# Expected payload
{
"name": "Super Cool Gallery",
"address": "123 Whatever St",
"address2": "",
"city": "Minneapolis",
"state": "MN",
"country": "USA",
"geo": null,
"website": "http://supercool.com",
"phone": "612-555-5555",
"email": "*****@*****.**",
"category": "gallery"
}
"""
resource_id = slug
key = get_key_from_resource_id(resource_id) #ndb.Key(urlsafe=slug)
venue = key.get()
#venue = venues_api.get_venue_by_slug(slug)
if not venue:
self.serve_404('Gallery Not Found')
return False
if not self.cleaned_data['slug'] == key.id():
raise Exception(
'Unable to edit slugs on venues at this time. Please change slug to: %s'
% key.id())
venue = venues_api.edit_venue(venue, self.cleaned_data)
result = create_resource_from_entity(venue)
self.serve_success(result)
def _get(self, resource_id):
# Get a Public User Profile
key = get_key_from_resource_id(resource_id)
e = key.get()
self.serve_success(Resource(e, REST_RULES).to_dict())
def post(self):
"""
Callback for a successful upload... keep this lightweight
"""
fs = Filesystem(BUCKET_NAME)
has_files = fs.get_uploads(self.request,
'the_file',
populate_post=True)
if has_files:
file_info = has_files[0]
original_filename = file_info.filename
content_type = file_info.content_type
size = file_info.size
gs_object_name = file_info.gs_object_name # We could urlfetch this, but file not public
blob_key = blobstore.create_gs_key(gs_object_name)
data = fs.read(gs_object_name.replace('/gs', ''))
# What we want to do now is create a copy of the file with our own info
dest_filename = 'juniper/%s' % original_filename
# Prep the file object
file_obj = self.create_image(fs, data, dest_filename)
file_obj_key = file_obj.key
resource_id = get_resource_id_from_key(file_obj_key)
# Finally delete the tmp file
#data = fs.delete(gs_object_name.replace('/gs', ''))
# "Return" a rest resource of sorts
payload = {
'status': 200,
'messages': [],
'results': Resource(file_obj, REST_RESOURCE_RULES).to_dict()
}
self.response.set_status(200)
self.response.headers['Content-Type'] = 'application/json'
self.response.write(json.dumps(payload))
# Handle Attachment to Resource
# Do this in a deferred task?
attach_to_resource_id = self.request.get('attach_to_resource',
None)
# TODO: This should be done in a txn - especially when there are multiple uploads
if attach_to_resource_id:
attachment_resource_key = get_key_from_resource_id(
attach_to_resource_id)
attachment_resource = attachment_resource_key.get()
if not attachment_resource:
raise Exception(
'Resource with key %s not found. File was uploaded...'
% attach_to_resource_id)
if not attachment_resource.attachment_resources:
attachment_resource.attachment_resources = []
# Update attachments
attachment_resource.attachment_resources.append(resource_id)
target_property = self.request.get('target_property', None)
if target_property:
setattr(attachment_resource, target_property, resource_id)
attachment_resource.put()
return
