def delete_blob(key):
try:
BlobInfo.get(key).delete()
except:
return 'failed'
else:
return 'succeeded'
def update_background(self, full, thumb):
station = self.station
old_full_blob_key = None
old_thumb_blob_key = None
m1 = re.match(r"/picture/([^/]+)?/view", station.full)
m2 = re.match(r"/picture/([^/]+)?/view", station.thumb)
if m1 and m2:
logging.info("Background is a blob")
old_full_blob_key = m1.group(1)
old_thumb_blob_key = m2.group(1)
else:
logging.info("Background is a static file")
station.full = full
station.thumb = thumb
station.put()
logging.info("Station updated in datastore")
memcache.set(self._memcache_station_id, station)
logging.info("Station updated in memcache")
# Update in runtime
self._station = station
if old_full_blob_key and old_thumb_blob_key:
old_full = BlobInfo.get(old_full_blob_key)
old_full.delete()
logging.info("Old full size background removed from blobstore")
old_thumb = BlobInfo.get(old_thumb_blob_key)
old_thumb.delete()
logging.info("Old thumbnail removed from blobstore")
def image_manager(page):
"""
GET --> The main image manager page
POST --> Delete requested file(s)
:param page: The requested page
"""
if request.method == 'POST':
img_ref_key = request.get_json()
# Delete the img from ndb
for img_ref in img_ref_key['objects']:
img_inst = ndb.Key(ImageReference, int(img_ref))
img = img_inst.get()
blob_key = img.blob
# Delete img and blob
img_inst.delete()
BlobInfo.get(blob_key).delete()
logging.info("Delete image: {}".format(img_ref))
return "true"
offset = (page-1)*IMAGES_PER_PAGE
images = ImageReference.query().order(-ImageReference.date)
pagination = Pagination(page, IMAGES_PER_PAGE, images.count())
query = images.fetch(IMAGES_PER_PAGE, offset=offset)
return render_template('image-manager/admin-manager-images.html',
keys=query,
pagination=pagination)
def start_batch(key, bid):
try:
reader = BlobReader(BlobKey(key))
except:
return 'failed to find key: please re-upload.'
newest_date = branch_newest_date(bid)
dic = {}
for line in reader:
line = line.strip()
try:
game_date = valid(line)
if game_date < newest_date:
game_date = newest_date
except:
continue
if util.tomorrow(util.today()) < game_date:
continue
if game_date not in dic:
dic[game_date] = []
dic[game_date].append(','.join(line.split(',')[:8]))
for key_date in sorted(dic.keys()):
qs = QueueScore(
bid=bid, game_date=key_date, body=(
'\n'.join(reversed(dic[key_date]))
))
qs.put()
BlobInfo.get(key).delete()
return 'upload succeeded!'
def action_edit(self):
if self.section.handler.request.get('submit'):
self.SITE_HEADER = self.section.handler.request.get('SITE_HEADER')
self.SITE_SUB_HEADER = self.section.handler.request.get('SITE_SUB_HEADER')
self.DEFAULT_THEME = self.section.handler.request.get('DEFAULT_THEME')
self.GOOGLE_ANALYTICS_UA = self.section.handler.request.get('GOOGLE_ANALYTICS_UA')
self.ROBOTS_TXT = self.section.handler.request.get('ROBOTS_TXT')
if self.section.handler.request.get('FAVICON_ICO'):
if self.FAVICON_ICO:
BlobInfo.get(self.FAVICON_ICO).delete()
data = self.section.handler.request.get('FAVICON_ICO')
handle = files.blobstore.create(mime_type='image/x-icon', _blobinfo_uploaded_filename='favicon.ico')
with files.open(handle, 'a') as f: f.write(data)
files.finalize(handle)
self.FAVICON_ICO = files.blobstore.get_blob_key(handle)
self.ENABLE_THEME_PREVIEW = self.section.handler.request.get('ENABLE_THEME_PREVIEW') != ''
self.DEBUG_MODE = self.section.handler.request.get('DEBUG_MODE') != ''
cache.delete(CACHE_KEY)
self.update()
raise Exception('Redirect', self.section.action_redirect_path)
f = form(self.section, self.section.full_path)
f.add_control(control(self.section, 'text', 'SITE_HEADER', self.SITE_HEADER, 'Site header', 50))
f.add_control(control(self.section, 'text', 'SITE_SUB_HEADER', self.SITE_SUB_HEADER, 'Site sub-header', 50))
combined_themes = get_local_theme_namespaces() + get_custom_theme_namespaces()
f.add_control(selectcontrol(self.section, 'DEFAULT_THEME', combined_themes, self.DEFAULT_THEME if self.DEFAULT_THEME else DEFAULT_LOCAL_THEME_TEMPLATE, 'Default theme'))
f.add_control(control(self.section, 'text', 'GOOGLE_ANALYTICS_UA', self.GOOGLE_ANALYTICS_UA, 'Google analytics UA'))
f.add_control(control(self.section, 'file', 'FAVICON_ICO', label='favicon.ico'))
f.add_control(textareacontrol(self.section, 'ROBOTS_TXT', self.ROBOTS_TXT, 'robots.txt', 90, 5))
f.add_control(checkboxcontrol(self.section, 'ENABLE_THEME_PREVIEW', self.ENABLE_THEME_PREVIEW, 'Enable theme preview'))
f.add_control(checkboxcontrol(self.section, 'DEBUG_MODE', self.DEBUG_MODE, 'Debug mode'))
f.add_control(control(self.section, 'submit', 'submit', 'Submit'))
return '<h2>Edit configuration</h2>%s' % unicode(f)
def delete(self, **kwargs):
if self.imageid:
b = BlobInfo.get(self.imageid.split('.')[0])
if b:
b.delete()
if self.apkkey:
b = BlobInfo.get(self.apkkey)
if b:
b.delete()
for imguri in self.imagelist:
b = BlobInfo.get(imguri.split('.')[0])
if b:
b.delete()
super(Plugin, self).delete(**kwargs)
pluginCount = PluginCount.get_or_insert('plugin_count')
pluginCount.num -= 1
pluginCount.put()
memcache.delete('allplugincount')
memcache.delete('appnamelist')
memcache.delete('pluginid%s' % self.key().id())
memcache.delete('user_applist_%s' % (self.username))
l = []
for i in range(0, pluginCount.num % 30):
l.append('applist__%s' % i)
l.append('applist__%s' % len(l))
memcache.delete_multi(l)
def get(self):
query_params = {'blob_prop':str(urllib.unquote(self.request.get('blob_key')))}
file = object_query(Blob, query_params).get()
blob = BlobInfo(file.blob_key)
blob.delete()
file.key.delete()
time.sleep(TIME_SLEEP)
self.redirect('/files/view_list')
def delete_all_images():
k = ndb.Key("Firm", "frl")
for img in Image.query(ancestor=k).iter():
print img.key.id()
for bk in [img.small_blob_key, img.large_blob_key]:
if BlobInfo.get(bk):
BlobInfo.get(bk).delete()
img.key.delete()
def test_remove_image_with_only_original_blob(self):
content_type = 'image/jpeg'
blob_key = self.make_blob(content_type, 'dummy')
self.assertTrue(BlobInfo.get(blob_key),
'Should be able to load BlobInfo for key.')
image = ae_image.core.Image(blob_key, content_type)
image.remove()
self.assertFalse(BlobInfo.get(blob_key),
'Should no longer be able to load BlobInfo for key.')
def serve_file(request, blob_key_or_info, as_download=False, content_type=None, filename=None, offset=None, size=None):
"""
Serves a file from the blobstore, reads most of the data from the blobinfo by default but you can override stuff
by passing kwargs.
You can also pass a Google Cloud Storage filename as `blob_key_or_info` to use Blobstore API to serve the file:
https://cloud.google.com/appengine/docs/python/blobstore/#Python_Using_the_Blobstore_API_with_Google_Cloud_Storage
"""
if isinstance(blob_key_or_info, BlobKey):
info = BlobInfo.get(blob_key_or_info)
blob_key = blob_key_or_info
elif isinstance(blob_key_or_info, basestring):
info = BlobInfo.get(BlobKey(blob_key_or_info))
blob_key = BlobKey(blob_key_or_info)
elif isinstance(blob_key_or_info, BlobInfo):
info = blob_key_or_info
blob_key = info.key()
else:
raise ValueError("Invalid type %s" % blob_key_or_info.__class__)
if info == None:
# Lack of blobstore_info means this is a Google Cloud Storage file
if has_cloudstorage:
info = cloudstorage.stat(blob_key_or_info)
info.size = info.st_size
blob_key = create_gs_key('/gs{0}'.format(blob_key_or_info))
else:
raise ImportError("To serve a Cloud Storage file you need to install cloudstorage")
response = HttpResponse(content_type=content_type or info.content_type)
response[BLOB_KEY_HEADER] = str(blob_key)
response['Accept-Ranges'] = 'bytes'
http_range = request.META.get('HTTP_RANGE')
if offset or size:
# Looks a little bonkers, but basically create the HTTP range string, we cast to int first to make sure
# nothing funky gets into the headers
http_range = "{}-{}".format(
str(int(offset)) if offset else "",
str(int(offset or 0) + size) if size else ""
)
if http_range is not None:
response[BLOB_RANGE_HEADER] = http_range
if as_download:
response['Content-Disposition'] = smart_str(
u'attachment; filename="%s"' % (filename or info.filename)
)
elif filename:
raise ValueError("You can't specify a filename without also specifying as_download")
if info.size is not None:
response['Content-Length'] = info.size
return response
def upload_post_processing():
file = request.files.data.filename
# validate file is image format
if mimetypes.guess_type(file)[0].split('/')[0] != 'image':
# delete non-image file types
BlobInfo.gql("WHERE filename = :fname", fname=file).get().delete()
return template('upload_error.html')
response.set_cookie('img', file, path='/')
redirect('/upload_success')
def __init__(self, **kwargs):
gs_object_name = kwargs.pop('gs_object_name', None)
blob_key = kwargs.pop('blob_key', None)
if gs_object_name:
self.blobstore_info = CloudStorageInfo(gs_object_name)
elif blob_key:
self.blobstore_info = BlobInfo(blob_key)
else:
raise ValueError('A gs_object_name or blob_key is required.')
super(AppEngineUploadedFile, self).__init__(self.blobstore_info.open(), **kwargs)
def on_remove(self):
for i in range(len(self.theme_namespaces)):
# This can be done more efficiently via GQL
theme = self.get_theme(self.theme_namespaces[i])
cache.delete(CACHE_KEY_PREPEND + self.theme_namespaces[i])
for key in theme.image_keys:
cache.delete(CACHE_KEY_PREPEND + str(key))
BlobInfo.get(key).delete()
theme.key.delete()
del self.theme_keys[i]
del self.theme_namespaces[i]
self.update()
def post(self):
key = self.request.get("key")
value = self.request.get("value")
filename = urllib.unquote(key)
# Clean up current file
query = DataFile.all().filter("f_key", filename)
for data_file in query:
data_file.delete()
# Create a file
writable_file_name = files.blobstore.create(mime_type="application/octect-stream")
with files.open(writable_file_name, "a") as f:
f.write(value)
files.finalize(writable_file_name)
blob_key = files.blobstore.get_blob_key(writable_file_name)
data_file = DataFile(f_key=filename, f_value=blob_key)
data_file.put()
if memcache.get(key="turn") == "on":
if BlobInfo.get(blob_key).size <= 100000:
memcache.set(key=filename, value=value, time=3600)
write_boolean(self, True)
def test_append_from_blob_info(self):
blob_key = self.make_blob('image/jpeg', 'dummy')
collection = ae_image.core.Collection(
[ae_image.core.Style('big', 500)])
collection.append_from_blob_info(BlobInfo.get(blob_key))
self.assertTrue(
collection.get_url('big', blob_key), 'Expect URL back.')
def GarbageCollectBlobs(self):
keys_to_blobs = {}
for blob in BlobInfo.all():
keys_to_blobs[blob.key()] = blob
for responder in Responder.all():
image_blob = responder.image_data
if image_blob:
key = image_blob.key()
if key in keys_to_blobs:
del keys_to_blobs[key]
for product in Product.all():
image_blob = product.image_data
if image_blob:
key = image_blob.key()
if key in keys_to_blobs:
del keys_to_blobs[key]
for key, blob_info in keys_to_blobs.iteritems():
logging.info('deleting %s' % key)
blob_info.delete()
if keys_to_blobs:
return 'Deleted blobs: \n%s' % '\n'.join(str(k) for k in keys_to_blobs)
else:
return 'No blobs to delete'
def post(self):
upload_files = self.get_uploads('file')
blob_info = upload_files[0]
# Resize the image
image = images.Image(blob_key=blob_info.key())
image.resize(width=WOOF_FEED_ITEM_IMAGE_MAX_WIDTH, height=WOOF_FEED_ITEM_IMAGE_MAX_HEIGHT)
thumbnail = image.execute_transforms(output_encoding=images.JPEG)
# Save Resized Image back to blobstore
file_name = files.blobstore.create(mime_type='image/jpeg')
with files.open(file_name, 'a') as f:
f.write(thumbnail)
files.finalize(file_name)
# Remove the original image
blobstore.delete(blob_info.key())
blob_key = files.blobstore.get_blob_key(file_name)
# New FeedImage
feed_image = FeedImage()
feed_image.data = BlobInfo.get(blob_key)
feed_image.width = image.width
feed_image.height = image.height
feed_image.put()
# Create new FeedItem
feed_item = FeedItem()
feed_item.text = self.request.get("text")
feed_item.image = feed_image
feed_item.put()
self.redirect('/')
def test_remove_with_single_image(self):
blob_key = self.make_blob('image/jpeg', 'dummy')
self.assertTrue(BlobInfo.get(blob_key),
'Should be able to load BlobInfo for key.')
collection = ae_image.core.Collection(
[ae_image.core.Style('big', 500)])
collection.append_from_blob_info(BlobInfo.get(blob_key))
self.assertTrue(
collection.get_url('big', blob_key), 'Expect URL back.')
collection.remove(blob_key)
self.assertFalse(BlobInfo.get(blob_key),
'Should no longer be able to load BlobInfo for key.')
self.assertRaises(ae_image.core.UnknownImage, collection.get_url,
'big', blob_key)
self.assertRaises(ae_image.core.UnknownImage, collection.remove,
blob_key)
def action_get(self):
if not self.section.path_params or len(self.section.path_params) != 3:
raise Exception('NotFound')
theme = self.get_theme(self.section.path_params[0])
resource = self.section.path_params[1]
filename = self.section.path_params[2]
if resource == 'css':
filenames, contents = theme.css_filenames, theme.css_contents
content_type = 'text/css'
elif resource == 'js':
filenames, contents = theme.js_filenames, theme.js_contents
content_type = 'text/javascript'
elif resource == 'image':
data = None
try:
key = theme.image_keys[theme.image_filenames.index(filename)]
data = cache.get(CACHE_KEY_PREPEND + str(key))
if not data:
data = BlobInfo.get(key)
cache.set(CACHE_KEY_PREPEND + str(key), data)
finally:
if not data:
raise Exception('NotFound')
raise Exception('SendFileBlob', data.open().read(), data.content_type)
else:
raise Exception('NotFound')
try:
index = filenames.index(filename)
except:
raise Exception('NotFound')
else:
raise Exception('SendFileBlob', str(contents[index]), content_type)
def returnAllJSON(self):
results = []
fils = BlobInfo.all()
for record in fils:
blob_key = str(record.key())
result = {}
result['key'] = blob_key
result['name'] = record.filename
result['type'] = record.content_type
result['size'] = record.size
result['deleteType'] = 'DELETE'
result['deleteUrl'] = self.request.host_url +'/?key=' + urllib.quote(blob_key, '')
if (IMAGE_TYPES.match(result['type'])):
try:
result['url'] = images.get_serving_url(blob_key, size=1024, crop=True, secure_url=None)
result['thumbnailUrl'] = images.get_serving_url(blob_key, size=240, crop=True, secure_url=None)
result['thumbnailUrl2'] = images.get_serving_url(blob_key, size=120, crop=True, secure_url=None)
except: # Could not get an image serving url
pass
if not 'url' in result:
result['url'] = self.request.host_url +\
'/' + blob_key + '/' + urllib.quote(
result['name'].encode('utf-8'), '')
results.append(result)
#result = {'files': results}
s = json.dumps(results, separators=(',', ':'))
redirect = self.request.get('redirect')
if redirect:
return self.redirect(str(
redirect.replace('%s', urllib.quote(s, ''), 1)
))
if 'application/json' in self.request.headers.get('Accept'):
self.response.headers['Content-Type'] = 'application/json'
self.response.write(s)
class AppEngineUploadedFile(UploadedFile):
"""
A file uploaded via App Engine's upload mechanism.
"""
def __init__(self, **kwargs):
gs_object_name = kwargs.pop('gs_object_name', None)
blob_key = kwargs.pop('blob_key', None)
if gs_object_name:
self.blobstore_info = CloudStorageInfo(gs_object_name)
elif blob_key:
self.blobstore_info = BlobInfo(blob_key)
else:
raise ValueError('A gs_object_name or blob_key is required.')
super(AppEngineUploadedFile, self).__init__(self.blobstore_info.open(), **kwargs)
def open(self, mode=None):
pass
def chunks(self, chunk_size=1024 * 128):
self.file.seek(0)
while True:
content = self.read(chunk_size)
if not content:
break
yield content
def multiple_chunks(self, chunk_size=1024 * 128):
return True
def _get_info(self, name):
if name.startswith('/gs/'):
assert cloudstorage, 'cloudstorage module is not available.'
return CloudStorageInfo(name)
else:
key = BlobKey(name.split('/', 1)[0])
return BlobInfo.get(key)
def post(self):
try:
if not db.WRITE_CAPABILITY.is_enabled():
raise utils.CapabilityUnavailable('Datastore unavailable')
if 'user_uuid' not in self.context['request_args'] or self.context['request_args']['user_uuid'] == '':
raise Exception('No user_uuid provided')
if 'serving_url' not in self.context['request_args'] or self.context['request_args']['serving_url'] == '':
raise Exception('No serving_url provided')
if 'short_url' not in self.context['request_args'] or self.context['request_args']['short_url'] == '':
raise Exception('No short_url provided')
if 'blob_key_name' not in self.context['request_args'] or self.context['request_args']['blob_key_name'] == '':
raise Exception('No blob_key_name provided')
blob_key = BlobKey(self.context['request_args']['blob_key_name'])
blob_info = BlobInfo.get(blob_key)
user_image = photo_module.Photo(
user_uuid=self.context['request_args']['user_uuid'],
serving_url=self.context['request_args']['serving_url'],
short_url=self.context['request_args']['short_url'],
blob=blob_info
)
user_image.save()
except modules_base.CapabilityUnavailable, cu:
logging.exception(cu)
self.response.set_status(503)
def deleteOldBlobs():
blobs = BlobInfo.all().fetch(500)
for blob in blobs:
if blob.filename.find(SMARTHISTORY_URL) != -1:
age = datetime.now() - blob.creation
if age.days * 86400 + age.seconds >= SMARTHISTORY_IMAGE_CACHE_EXPIRATION:
blob.delete()
def output(self, filename, file_out):
content = file_out.read()
base, ext = os.path.splitext(filename)
if ext == ".css":
mimetype = "text/css"
elif ext == ".js":
mimetype = "text/javascript"
else:
mimetype = "application/octet-stream"
already_exists = False
for info in BlobInfo.all().filter('content_type = ', mimetype):
if info.filename == filename:
already_exists = True
continue
#Clear out old blobs
if info.filename.split(".")[0] == filename.split(".")[0]:
logging.debug("Deleting: %s", info.filename)
info.delete()
if not already_exists:
logging.info("Creating: %s", filename)
result = files.blobstore.create(mime_type=mimetype, _blobinfo_uploaded_filename=filename)
with files.open(result, "a") as f:
f.write(content)
files.finalize(result)
blob_key = files.blobstore.get_blob_key(result)
while not blob_key:
blob_key = files.blobstore.get_blob_key(result)
def do_clean(cursor=None):
bq = BlobInfo.all()
if cursor:
bq.with_cursor(cursor)
blob = bq.get()
if not blob:
return
key = str(blob.key())
thq = Thread.all(keys_only=True)
thq.filter("images", key)
th = thq.get()
if th:
logging.info("thread: %r" % th)
else:
logging.info("no thread for image %r" % key)
blob.delete()
deferred.defer(do_clean, bq.cursor(), _countdown=30)
def action_delete(self):
if not self.section.path_params or len(self.section.path_params) != 1:
raise Exception('NotFound')
theme = self.get_theme(self.section.path_params[0])
if self.section.handler.request.get('submit'):
for key in theme.image_keys:
cache.delete(CACHE_KEY_PREPEND + str(key))
BlobInfo.get(key).delete()
self.theme_keys.remove(theme.key)
self.theme_namespaces.remove(theme.namespace)
theme.key.delete()
self.update()
cache.flush_all() # Flush all cached resources for this theme which is important for sections where it is active
raise Exception('Redirect', self.section.action_redirect_path)
f = form(self.section, self.section.full_path)
f.add_control(control(self.section, 'submit', 'submit', 'Confirm'))
return '<div class="status warning">Are you sure you wish to delete theme "%s" and all associated resources?</div>%s' % (theme.namespace, unicode(f))
def presentation_download(request, key, name):
blob = BlobInfo.get(key)
if not blob:
return Response(status=304)
headers = Headers()
headers['Content-Type'] = blob.content_type.encode('utf-8')
headers['Last-Modified'] = blob.creation.strftime(HTTP_DATE_FMT)
return Response(blob.open().read(), headers=headers)
def get(self, resource): blob_key = str(urllib.unquote(resource)) blob_info = BlobInfo.get(blob_key) if not blob_info: self.error(404) else: self.send_blob(blob_info)
def post(self):
try:
recipeID = self.request.get('recipe_id')
print("rID: " + recipeID)
recipe = ndb.Key(urlsafe=recipeID)
recipe_key = recipe.get()
# Image Elements
recipe_key.img = []
deletable = []
# Get Deletable Elements
try:
removedImages = self.request.get('removed_images')
images = removedImages.split(',')
for image in images:
if image != '':
print('Deletable: ' + image)
deletable.append(BlobInfo.get(image).key())
except Exception, e:
print e
# Avatar Model
try:
avi = self.get_uploads('avatar_file')
recipe_key.avatar = avi[0].key()
try:
oldAvi = self.request.get('existing_avatar')
recipe_key.img.append(BlobInfo.get(oldAvi).key())
except Exception, e:
print(e)
except Exception, e:
print('No new Avatar Image Uploaded')
# Existing Avatar Model
try:
aviKey = self.request.get_all('existing_avatar')
recipe_key.avatar = BlobInfo.get(aviKey[0]).key()
except Exception, e:
# Pull old avatar that was deleted out of the trash
if recipe_key.avatar in deletable:
deletable.remove(recipe_key.avatar)
print('No Existing Image Chosen as Avatar. Keeping Same Avatar')
def serve_file(request, file, save_as, content_type, **kwargs):
if isinstance(file, BlobKey):
blobkey = file
elif hasattr(file, 'file') and hasattr(file.file, 'blobstore_info'):
blobkey = file.file.blobstore_info.key()
elif hasattr(file, 'blobstore_info'):
blobkey = file.blobstore_info.key()
else:
raise ValueError("The provided file can't be served via the "
"Google App Engine Blobstore.")
response = HttpResponse(content_type=content_type)
response[BLOB_KEY_HEADER] = str(blobkey)
response['Accept-Ranges'] = 'bytes'
http_range = request.META.get('HTTP_RANGE')
if http_range is not None:
response[BLOB_RANGE_HEADER] = http_range
if save_as:
response['Content-Disposition'] = smart_str(
u'attachment; filename="%s"' % save_as)
info = BlobInfo.get(blobkey)
if info.size is not None:
response['Content-Length'] = info.size
return response
def _get_blobinfo(self, name):
return BlobInfo.get(self._get_key(name))
def download_file(blob_key):
blob = BlobInfo.get(blob_key)
response.headers['X-AppEngine-BlobKey'] = blob_key.decode('utf-8')
response.headers['Content-Disposition'] = "attachment; filename=%s" % blob.filename
response.headers['Content-Type'] = blob.content_type
return response
try:
aviKey = self.request.get_all('existing_avatar')
recipe_key.avatar = BlobInfo.get(aviKey[0]).key()
except Exception, e:
# Pull old avatar that was deleted out of the trash
if recipe_key.avatar in deletable:
deletable.remove(recipe_key.avatar)
print(
'No Existing Image Chosen as Avatar. Keeping Same Avatar'
)
# Existing Images Model
try:
urlKeys = self.request.get_all('existing_img')
for keye in urlKeys:
recipe_key.img.append(BlobInfo.get(keye).key())
except Exception, e:
print e
# New Images
try:
sm_images = self.get_uploads('sm_image')
for image in sm_images:
recipe_key.img.append(image.key())
except Exception, e:
print e
# Process Deletable
try:
for delete in deletable:
BlobInfo.get(delete).delete()
def getphotogallery(self, dish_id):
dish = Dish.get_by_id(int(dish_id))
docinfo = BlobInfo.get(dish.photogallery)
return docinfo
def get_blobstore_uploaded_file(blobstore_key):
""" Convenience method for returning a BlobstoreUploadedFile object for a given blobstore_key
"""
return BlobstoreUploadedFile(BlobInfo(blobstore_key), charset="utf-8")
def getaudio(self, dish_id):
dish = Dish.get_by_id(int(dish_id))
docinfo = BlobInfo.get(dish.audio)
return docinfo
def on_remove(self):
for key in self.file_keys:
cache.delete(CACHE_KEY_PREPEND + str(key))
BlobInfo.get(key).delete()
self.update()
def file_info(self, name):
key = BlobKey(name.split('/', 1)[0])
return BlobInfo(key)
def titi(value):
return BlobInfo.get(str(value)).key() #blobstore.get(str(value))
def delete_blob(self):
if self.blob_key:
blob_info = BlobInfo.get(self.blob_key)
if blob_info:
blob_info.delete()
def serve_file(request,
blob_key_or_info,
as_download=False,
content_type=None,
filename=None,
offset=None,
size=None):
"""
Serves a file from the blobstore, reads most of the data from the blobinfo by default but you can override stuff
by passing kwargs.
You can also pass a Google Cloud Storage filename as `blob_key_or_info` to use Blobstore API to serve the file:
https://cloud.google.com/appengine/docs/python/blobstore/#Python_Using_the_Blobstore_API_with_Google_Cloud_Storage
"""
if isinstance(blob_key_or_info, BlobKey):
info = BlobInfo.get(blob_key_or_info)
blob_key = blob_key_or_info
elif isinstance(blob_key_or_info, basestring):
info = BlobInfo.get(BlobKey(blob_key_or_info))
blob_key = BlobKey(blob_key_or_info)
elif isinstance(blob_key_or_info, BlobInfo):
info = blob_key_or_info
blob_key = info.key()
else:
raise ValueError("Invalid type %s" % blob_key_or_info.__class__)
if info == None:
# Lack of blobstore_info means this is a Google Cloud Storage file
if has_cloudstorage:
cached_value = _get_from_cache(blob_key_or_info)
if cached_value:
blob_key, info = cached_value
else:
info = cloudstorage.stat(blob_key_or_info)
info.size = info.st_size
blob_key = create_gs_key('/gs{0}'.format(blob_key_or_info))
_add_to_cache(blob_key_or_info, blob_key, info)
else:
raise ImportError(
"To serve a Cloud Storage file you need to install cloudstorage"
)
response = HttpResponse(content_type=content_type or info.content_type)
response[BLOB_KEY_HEADER] = str(blob_key)
response['Accept-Ranges'] = 'bytes'
http_range = request.META.get('HTTP_RANGE')
if offset or size:
# Looks a little bonkers, but basically create the HTTP range string, we cast to int first to make sure
# nothing funky gets into the headers
http_range = "{}-{}".format(
str(int(offset)) if offset else "",
str(int(offset or 0) + size) if size else "")
if http_range is not None:
response[BLOB_RANGE_HEADER] = http_range
if as_download:
response['Content-Disposition'] = smart_str(
u'attachment; filename="%s"' % (filename or info.filename))
elif filename:
raise ValueError(
"You can't specify a filename without also specifying as_download")
if info.size is not None:
response['Content-Length'] = info.size
return response
def file_up_to_date(self, filename):
result = bool(BlobInfo.all().filter('filename =', filename).count())
return result
def post_get_blob_key(input_dict, entity_instance, bulkload_state):
entity_instance['file'] = BlobInfo.get(str(entity_instance['file'])).key()
return entity_instance
def getvideo(self, dish_id):
dish = Dish.get_by_id(int(dish_id))
docinfo = BlobInfo.get(dish.video)
return docinfo
def pull_from_dropbox():
client = get_dropbox_client()
if client is None:
return
path_prefix = get_config('dropbox_path')
cursor = get_config('dropbox_delta_cursor')
last_sync = get_config('dropbox_last_sync') or datetime.datetime(
2000, 1, 1)
first = cursor is None
if first:
set_config('dropbox_sync_progress', (0, 1))
entries = []
while 1:
result = client.delta(cursor, path_prefix=path_prefix.rstrip('/'))
entries.extend(
(path, metadata) for path, metadata in result['entries'])
cursor = result['cursor']
set_config('dropbox_delta_cursor', cursor)
if not result['has_more']:
break
for i, (path, metadata) in enumerate(entries):
repo_key = path[len(path_prefix):].split('/')
cache_key = make_cache_key(repo_key)
list_cache_key = make_cache_key(repo_key[:-1])
if not repo_key or any(not part for part in repo_key):
continue
db_key = make_db_key(repo_key)
if metadata:
rev = metadata['rev']
modified_at = parse_rfc2822(metadata['modified'])
last_sync = max(modified_at, last_sync)
if metadata['is_dir']:
blob_info = None
cache_value = 'D'
else:
filename = create(mime_type='text/xml')
cache_value = None
cache_buffer = ['F']
dst_size = 0
with fopen(filename, 'ab') as dst:
for offset in xrange(0, metadata['bytes'],
INCOMING_BYTES_LIMIT):
src = client.get_file(path,
rev=rev,
start=offset,
length=offset)
while 1:
chunk = src.read(10240)
if chunk:
dst_size += len(chunk)
dst.write(chunk)
if dst_size < CACHE_BYTES_LIMIT:
cache_buffer.append(chunk)
else:
break
if dst_size < CACHE_BYTES_LIMIT:
cache_value = ''.join(cache_buffer)
del cache_buffer
finalize(filename)
blob_key = get_blob_key(filename)
blob_info = BlobInfo.get(blob_key)
def txn():
delete(cache_key, namespace='slot')
delete(list_cache_key, namespace='list')
slot = Slot.get(db_key)
if slot is None:
slot = Slot(depth=len(repo_key),
key=db_key,
blob=blob_info,
rev=rev,
updated_at=modified_at,
synced_at=modified_at)
else:
if slot.blob is not None:
slot.blob.delete()
slot.blob = blob_info
slot.rev = rev
slot.updated_at = modified_at
slot.synced_at = modified_at
slot.put()
if cache_value is not None:
put(cache_key, cache_value, namespace='slot')
delete(list_cache_key, namespace='list')
run_in_transaction_options(create_transaction_options(xg=True),
txn)
else:
slot = Slot.get(db_key)
if slot is not None:
slot.delete()
delete(cache_key, namespace='slot')
delete(list_cache_key, namespace='list')
if first:
set_config('dropbox_sync_progress', (i + 1, len(entries)))
set_config('dropbox_last_sync', last_sync)
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
import os, glob
import cgi
import cgitb
cgitb.enable()
#from config import *
from google.appengine.ext.blobstore import BlobInfo
blobs = BlobInfo.all()
vars = {
'checkboxes': "",
'transplantws': "/transplantdata",
'survivaldatasource': "/sampledata",
'genedatasource': "/genedata",
"jsdir": "/js",
"loadergif": "/images/loader.gif"
}
files = []
tr = ""
lastfile = ""
namedic = {}
for blob in blobs.run():