def test_iteration(self):
fs = motor.MotorGridFS(self.db)
_id = yield fs.put(b'foo')
g = motor.MotorGridOut(self.db.fs, _id)
# Iteration is prohibited.
self.assertRaises(TypeError, iter, g)
def main2():
import sys
sys.path.insert(0, '/Users/amodig/tornado-test/Keyword-APP/src/handlers')
from profile_handlers import ArticleDataHandler
# Connect to MongoDB
MONGO_SERVER_ADDRESS = 'localhost'
MONGO_SERVER_PORT = 27017
client = motor.MotorClient(MONGO_SERVER_ADDRESS, MONGO_SERVER_PORT)
# Choose correct database
db = client['app']
fs = motor.MotorGridFS(db, collection=u'fs')
cursor = fs.find({}, timeout=False)
while (yield cursor.fetch_next):
grid_out = cursor.next_object()
content = yield grid_out.read()
content_type = grid_out.content_type
title = grid_out.title
filename = grid_out.filename
key = grid_out._id
abstract = ArticleDataHandler.extract_abstract(content, content_type)
print "--------------------------------------"
print "Title:"
print title
print "Filename:"
print filename
print "Key:"
print key
print "--------------------------------------"
print "Abstract:"
print abstract
print "finished"
def test_alt_collection(self):
db = self.db
alt = motor.MotorGridFS(db, 'alt')
oid = yield alt.put(b"hello world")
gridout = yield alt.get(oid)
self.assertEqual(b"hello world", (yield gridout.read()))
self.assertEqual(1, (yield self.db.alt.files.count()))
self.assertEqual(1, (yield self.db.alt.chunks.count()))
yield alt.delete(oid)
with self.assertRaises(NoFile):
yield alt.get(oid)
self.assertEqual(0, (yield self.db.alt.files.count()))
self.assertEqual(0, (yield self.db.alt.chunks.count()))
with self.assertRaises(NoFile):
yield alt.get("foo")
oid = yield alt.put(b"hello world", _id="foo")
self.assertEqual("foo", oid)
gridout = yield alt.get("foo")
self.assertEqual(b"hello world", (yield gridout.read()))
yield alt.put(b"", filename="mike")
yield alt.put(b"foo", filename="test")
yield alt.put(b"", filename="hello world")
self.assertEqual(set(["mike", "test", "hello world"]),
set((yield alt.list())))
def test_put_unacknowledged(self):
client = self.motor_client(w=0)
fs = motor.MotorGridFS(client.motor_test)
with self.assertRaises(ConfigurationError):
yield fs.put(b"hello")
client.close()
def post(self):
file = self.request.files["uploadedfile"][0]
fs = motor.MotorGridFS(db)
gridin = yield fs.new_file()
length = len(file.body)
md5_dig = hashlib.md5(file.body).hexdigest()
result = yield gridin.write(file.body)
# TODO: we can write another chunk- as many times we want-
# When support for streaming file comes in mainstram branch
# we may use that
yield gridin.set('content_type', file.get("content_type"))
yield gridin.set('filename', file.get("filename"))
yield gridin.close()
file_id = gridin._id
# TODO: dump the gridin object which contains all the interesting fields including md5
self.write(
json.dumps({
"md5": md5_dig,
"file_id": str(file_id),
"file_size": length
}))
self.finish()
def read_gridfs_owned_file(user, db, collection_name, file_id, properties):
owner_id = user.get("_id")
if len(owner_id) < 1:
raise tornado.web.HTTPError(401, "Unauthorized")
fs = motor.MotorGridFS(db)
try:
gridout = yield fs.get(ObjectId(file_id))
if not gridout:
raise tornado.web.HTTPError(404, "File not found")
owner = gridout.owner
if owner != owner_id:
raise tornado.web.HTTPError(401, "Unauthorized")
content_type = gridout.content_type
got_col_name = gridout.collection_name
if got_col_name != collection_name:
raise tornado.web.HTTPError(404, "File not found")
content = yield gridout.read()
data = {"body": content }
if "content_type" in properties:
data["content_type"] = content_type
raise gen.Return(data)
except gridfs.NoFile:
raise tornado.web.HTTPError(404, "File not found")
def read_gridfs_file(db, collection_name, file_id, properties, ignore_col_name=False):
fs = motor.MotorGridFS(db)
try:
gridout = yield fs.get(ObjectId(file_id))
if not gridout:
raise tornado.web.HTTPError(404, "File not found")
content_type = gridout.content_type
if not ignore_col_name:
got_col_name = gridout.collection_name
if got_col_name != collection_name:
raise tornado.web.HTTPError(404, "File not found")
content = yield gridout.read()
data = {"body": content }
if "content_type" in properties:
data["content_type"] = content_type
raise gen.Return(data)
except gridfs.NoFile:
raise tornado.web.HTTPError(404, "File not found")
def test_gridfs_secondary(self):
primary_host, primary_port = test.env.primary
primary = self.motor_client(primary_host, primary_port)
if test.env.auth:
yield primary.admin.authenticate(test.db_user, test.db_password)
secondary_host, secondary_port = test.env.secondaries[0]
secondary = self.motor_client(secondary_host,
secondary_port,
read_preference=ReadPreference.SECONDARY)
if test.env.auth:
yield secondary.admin.authenticate(db_user, db_password)
yield primary.motor_test.drop_collection("fs.files")
yield primary.motor_test.drop_collection("fs.chunks")
# Should detect it's connected to secondary and not attempt to
# create index
fs = motor.MotorGridFS(secondary.motor_test)
# This won't detect secondary, raises error
with self.assertRaises(AutoReconnect):
yield fs.put(b'foo')
def get_files_db(self, user):
"""Finds all files by selected user.
:return a list of file dictionaries (can be empty).
"""
files = []
fs = motor.MotorGridFS(self.application.db, collection=u'fs')
cursor = fs.find({"user": user}, timeout=False)
while (yield cursor.fetch_next):
file = {}
grid_out = cursor.next_object()
# content = yield grid_out.read()
try:
file['title'] = grid_out.title
except AttributeError:
file['title'] = "[No title]"
file['name'] = grid_out.filename
file['key'] = grid_out._id
file['size'] = grid_out.length
file['type'] = grid_out.content_type
# grid returns a datetime.datetime
upload_date = grid_out.upload_date.replace(
tzinfo=pytz.utc).astimezone(self.timezone)
file['uploadDate'] = upload_date.strftime('%Y-%m-%d %H:%M')
file['deleteType'] = 'DELETE'
file['deleteUrl'] = self.get_delete_url(key=file['key'])
file['url'] = self.get_download_url(key=file['key'])
if self.access_control_allow_credentials:
file['deleteWithCredentials'] = True
files.append(file)
raise gen.Return(files)
def test_put_unacknowledged(self):
client = self.motor_client(w=0)
fs = yield motor.MotorGridFS(client.pymongo_test).open()
with assert_raises(ConfigurationError):
yield fs.put(b("hello"))
client.close()
def test_alt_collection(self):
db = self.cx.pymongo_test
alt = yield motor.MotorGridFS(db, 'alt').open()
oid = yield alt.put(b("hello world"))
gridout = yield alt.get(oid)
self.assertEqual(b("hello world"), (yield gridout.read()))
self.assertEqual(1, (yield db.alt.files.count()))
self.assertEqual(1, (yield db.alt.chunks.count()))
yield alt.delete(oid)
with assert_raises(NoFile):
yield alt.get(oid)
self.assertEqual(0, (yield db.alt.files.count()))
self.assertEqual(0, (yield db.alt.chunks.count()))
with assert_raises(NoFile):
yield alt.get("foo")
oid = yield alt.put(b("hello world"), _id="foo")
self.assertEqual("foo", oid)
gridout = yield alt.get("foo")
self.assertEqual(b("hello world"), (yield gridout.read()))
yield alt.put(b(""), filename="mike")
yield alt.put(b("foo"), filename="test")
yield alt.put(b(""), filename="hello world")
self.assertEqual(set(["mike", "test", "hello world"]),
set((yield alt.list())))
def test_put_filelike(self):
db = self.cx.pymongo_test
fs = yield motor.MotorGridFS(db).open()
oid = yield fs.put(StringIO(b("hello world")), chunk_size=1)
self.assertEqual(11, (yield db.fs.chunks.count()))
gridout = yield fs.get(oid)
self.assertEqual(b("hello world"), (yield gridout.read()))
def get(self, path, include_body=True):
fs = yield motor.MotorGridFS(self.database, self.root_collection).open()
try:
gridout = yield self.get_gridfs_file(fs, path)
except gridfs.NoFile:
raise tornado.web.HTTPError(404)
# If-Modified-Since header is only good to the second.
modified = gridout.upload_date.replace(microsecond=0)
self.set_header("Last-Modified", modified)
# MD5 is calculated on the MongoDB server when GridFS file is created
self.set_header("Etag", '"%s"' % gridout.md5)
mime_type = gridout.content_type
# If content type is not defined, try to check it with mimetypes
if mime_type is None:
mime_type, encoding = mimetypes.guess_type(path)
# Starting from here, largely a copy of StaticFileHandler
if mime_type:
self.set_header("Content-Type", mime_type)
cache_time = self.get_cache_time(path, modified, mime_type)
if cache_time > 0:
self.set_header("Expires", datetime.datetime.utcnow() +
datetime.timedelta(seconds=cache_time))
self.set_header("Cache-Control", "max-age=" + str(cache_time))
else:
self.set_header("Cache-Control", "public")
self.set_extra_headers(path, gridout)
# Check the If-Modified-Since, and don't send the result if the
# content has not been modified
ims_value = self.request.headers.get("If-Modified-Since")
if ims_value is not None:
date_tuple = email.utils.parsedate(ims_value)
if_since = datetime.datetime.fromtimestamp(time.mktime(date_tuple))
if if_since >= modified:
self.set_status(304)
return
# Same for Etag
etag = self.request.headers.get("If-None-Match")
if etag is not None and etag.strip('"') == gridout.md5:
self.set_status(304)
return
self.set_header("Content-Length", gridout.length)
if include_body:
yield gridout.stream_to_handler(self)
# Needed until fix for Tornado bug 751 is released, see
# https://github.com/facebook/tornado/issues/751 and
# https://github.com/facebook/tornado/commit/5491685
self.finish()
def test_gridfs_callback(self):
db = self.cx.pymongo_test
fs = motor.MotorGridFS(db)
yield self.check_optional_callback(fs.open)
fs = yield motor.MotorGridFS(db).open()
yield self.check_optional_callback(fs.new_file)
yield self.check_optional_callback(partial(fs.put, b('a')))
yield fs.put(b('foo'), _id=1, filename='f')
yield self.check_optional_callback(fs.get, 1)
yield self.check_optional_callback(fs.get_version, 'f')
yield self.check_optional_callback(fs.get_last_version, 'f')
yield self.check_optional_callback(partial(fs.delete, 1))
yield self.check_optional_callback(fs.list)
yield self.check_optional_callback(fs.exists)
def generate_abstract(self, file_id):
"""Get single abstract"""
fs = motor.MotorGridFS(self.application.db, collection=u'fs')
grid_out = yield fs.get(file_id)
content = yield grid_out.read()
content_type = grid_out.content_type
abstract = self.extract_abstract(content, content_type)
raise gen.Return(abstract)
def open_gridfs():
for collection in ('avatars', ):
def set_fs(fs, error):
if error: raise error
global fss
fss[collection] = GridFSWrapper(fs, collection)
motor.MotorGridFS(get_db(), collection=collection).open(set_fs)
def generate_abstracts(self, file_keys):
"""Abstract generator"""
fs = motor.MotorGridFS(self.application.db, collection=u'fs')
for file_id in file_keys:
grid_out = yield fs.get(file_id)
content = yield grid_out.read()
content_type = grid_out.content_type
abstract = self.extract_abstract(content, content_type)
yield file_id, abstract
def test_stream_to_handler(self):
fs = motor.MotorGridFS(self.db)
for content_length in (0, 1, 100, 100 * 1000):
_id = yield fs.put(b'a' * content_length)
gridout = yield fs.get(_id)
handler = MockRequestHandler()
yield gridout.stream_to_handler(handler)
self.assertEqual(content_length, handler.n_written)
yield fs.delete(_id)
def test_gridfs_replica_set(self):
rsc = yield self.motor_rsc(
w=self.w, wtimeout=5000,
read_preference=ReadPreference.SECONDARY)
fs = yield motor.MotorGridFS(rsc.pymongo_test).open()
oid = yield fs.put(b('foo'))
gridout = yield fs.get(oid)
content = yield gridout.read()
self.assertEqual(b('foo'), content)
def test_gridfs_replica_set(self):
rsc = self.motor_rsc(w=test.env.w,
wtimeout=5000,
read_preference=ReadPreference.SECONDARY)
fs = motor.MotorGridFS(rsc.motor_test)
oid = yield fs.put(b'foo')
gridout = yield fs.get(oid)
content = yield gridout.read()
self.assertEqual(b'foo', content)
def get_grid_out(self, file_id=None, filename=None):
"""Get GridOut object from MongoDB's GridFS file system"""
fs = motor.MotorGridFS(self.application.db, collection=u'fs')
fid = self.fid(file_id, filename)
try:
grid_out = yield fs.get(fid)
except NoFile:
raise web.HTTPError(404)
if filename:
assert grid_out.filename is filename, "file names does not match!"
raise gen.Return(grid_out)
def store_image(self, name, content, content_type):
"""Put an image in GridFS, and return the URL."""
fs = motor.MotorGridFS(self.settings['db'])
# This is the tail end of the URL, like 2012/06/foo.png.
now = datetime.datetime.utcnow()
fullname = media_link(now.year, now.month, name)
gridin = yield fs.new_file(filename=fullname,
content_type=content_type)
yield gridin.write(content)
yield gridin.close()
raise gen.Return(self.application.reverse_url('media', fullname))
def test_list(self):
db = self.cx.pymongo_test
fs = yield motor.MotorGridFS(db).open()
self.assertEqual([], (yield fs.list()))
yield fs.put(b("hello world"))
self.assertEqual([], (yield fs.list()))
yield fs.put(b(""), filename="mike")
yield fs.put(b("foo"), filename="test")
yield fs.put(b(""), filename="hello world")
self.assertEqual(set(["mike", "test", "hello world"]),
set((yield fs.list())))
def write_file(self, file_content, user, key, result):
"""Write file to MongoDB's GridFS system."""
fs = motor.MotorGridFS(self.application.db, collection=u'fs')
# default: file_id is the ObjectId of the resulting file
try:
file_id = yield fs.put(file_content,
_id=key,
user=user,
filename=result['name'],
content_type=result['type'],
title=result['title'])
except KeyError:
raise web.HTTPError(500)
assert file_id is key, "file_id is not key (%r): %r" % (key, file_id)
def __init__(self, server, database, store=None):
self.mongo = pymongo.Connection(server)
self.motor = motor.MotorClient(server)
self.mongo.document_class = bson.SON
self.motor.document_class = bson.SON
self.db = self.mongo[database]
self.mdb = self.motor[database]
if store:
self.fs = gridfs.GridFS(self.mongo[store])
self.mfs = motor.MotorGridFS(self.motor[store])
else:
self.fs = None
self.mfs = None
def delete(self):
"""Delete a file in the database"""
key = self.get_argument('key', default=None) or ''
fs = motor.MotorGridFS(self.application.db, collection=u'fs')
# get file name
grid_out = yield fs.get(key)
filename = grid_out.filename
# delete a file with the key
yield fs.delete(key)
# make JSON response
s = json.dumps({'files': {filename: True}}, separators=(',', ':'))
if 'application/json' in self.request.headers.get_list('Accept'):
self.request.set_headers['Content-Type'] = 'application/json'
print "DELETE writing: %s" % s
self.write(s)
def get(self):
file_id = self.get_argument("file_id", None)
if file_id:
fs = motor.MotorGridFS(db)
gridout = yield fs.get(ObjectId(file_id))
content = yield gridout.read()
self.write(content)
self.set_header("Content-Type", gridout.content_type)
else:
self.write('''
<form enctype="multipart/form-data" method="POST">
File tu upload: <input name="uploadedfile" type="file" /><br />
<input type="submit" value="Upload File" />
</form>''')
self.finish()
def get_info(self, file_id):
"""Get single article's info"""
fs = motor.MotorGridFS(self.application.db, collection=u'fs')
grid_out = yield fs.get(file_id)
title = grid_out.title
try:
abstract = grid_out.abstract
except AttributeError:
abstract = yield self.generate_abstract(file_id)
coll = self.application.db[u'fs.files']
yield coll.update({'_id': file_id},
{'$set': {
'abstract': abstract
}})
print "Abstract extracted automatically"
info = {'file_id': file_id, 'title': title, 'abstract': abstract}
raise gen.Return(info)
def test_gridfs_secondary(self):
primary_host, primary_port = self.primary
primary_client = self.motor_client(primary_host, primary_port)
secondary_host, secondary_port = self.secondaries[0]
secondary_client = self.motor_client(
secondary_host, secondary_port,
read_preference=ReadPreference.SECONDARY)
yield primary_client.pymongo_test.drop_collection("fs.files")
yield primary_client.pymongo_test.drop_collection("fs.chunks")
# Should detect it's connected to secondary and not attempt to
# create index
fs = yield motor.MotorGridFS(secondary_client.pymongo_test).open()
# This won't detect secondary, raises error
with assert_raises(AutoReconnect):
yield fs.put(b('foo'))
def test_stream_to_handler(self):
class MockRequestHandler(object):
def __init__(self):
self.n_written = 0
def write(self, data):
self.n_written += len(data)
def flush(self):
pass
fs = motor.MotorGridFS(self.db)
for content_length in (0, 1, 100, 100 * 1000):
_id = yield fs.put(b'a' * content_length)
gridout = yield fs.get(_id)
handler = MockRequestHandler()
yield gridout.stream_to_handler(handler)
self.assertEqual(content_length, handler.n_written)
yield fs.delete(_id)