def drop(user, password):
"""
Drop every database and re-add net user account (CAREFUL BEFORE RUNNING)
"""
connection = Connection(get_config('database', 'master_host'),
int(get_config('database', 'port')))
db = database.Database(connection, 'admin')
db.authenticate(user, password)
for log_type in get_log_types():
try:
db = database.Database(connection, log_type)
print "dropping " + log_type
connection.drop_database(log_type)
# re-add net user account
db.add_user(get_config('database', 'user'),
get_config('database', 'password'))
except Exception as e:
print str(e)
continue
def main_config():
if (not session.get('logged_in')):
return redirect(url_for('login'))
access = check_access('main_config')
if (request.method == 'POST' and access != 'rw'):
abort(403)
filenames = {'pubkey': 'rsa_1024_pub.pem', 'privkey': 'rsa_1024_priv.pem'}
certs = None
if (request.method == 'POST'):
changes = to_dict(request.form)
if (changes['action'] == 'configurations'):
del (changes['action'])
mongodb.db.configs.update_one({'name': changes['name']},
{'$set': changes})
elif (changes['action'] == 'genkeys'):
certs = genkeypair()
gridfsdb = database.Database(
MongoClient(host=GRIDFS_HOST, port=GRIDFS_PORT), 'certs')
fs = GridFS(gridfsdb)
for key in ['privkey', 'pubkey']:
oldfile = fs.find_one({'filename': filenames[key]})
if (oldfile is not None):
fs.delete(oldfile._id)
fs.put(certs[key].copy(),
content_type="text/plain",
filename=filenames[key])
result = mongodb.db.configs.find({}, {'_id': 0})
gridfsdb = database.Database(
MongoClient(host=GRIDFS_HOST, port=GRIDFS_PORT), 'images')
fs = GridFS(gridfsdb)
avatar = fs.exists(filename='avatar.png')
background = fs.exists(filename='background.png')
logo = fs.exists(filename='logo.png')
imgresult = {'avatar': avatar, 'background': background, 'logo': logo}
if (certs is None):
gridfsdb = database.Database(
MongoClient(host=GRIDFS_HOST, port=GRIDFS_PORT), 'certs')
fs = GridFS(gridfsdb)
if (fs.exists(filename=filenames['pubkey'])):
file = fs.get_last_version(filenames['pubkey'])
pubkey = file.read()
certs = {'pubkey': pubkey}
languages = copy_cursor(
mongodb.db.languages.find({}, sort=([('name', 1), ('variant', 1)])))
return render_template('main_config.html',
access=access,
images=imgresult,
configs=result,
certs=certs,
languages=languages)
def show_collections(self, host, port, dbname):
try:
c = MongoConnection(host=host, port=int(port), pool_size=1)
db = database.Database(c, dbname)
collections = db.collection_names()
except Exception, err:
return -1
def set_obj_media(objid, isocode, mediatype, file):
if (file.filename[-4:] not in ['.mp3', '.mp4']):
return False
if (mediatype not in ['audios', 'videos']):
return False
gridfsdb = database.Database(
MongoClient(host=GRIDFS_HOST, port=GRIDFS_PORT),
mediatypes[mediatype]['db'])
fs = GridFS(gridfsdb)
mongodb.db.objects.update_one(
{
'id': objid,
'translations.isocode': isocode
}, {'$set': {
'translations.$.' + mediatype: True
}})
oldfile = fs.find_one(
{'filename': isocode + '-' + objid + mediatypes[mediatype]['ext']})
if (oldfile is not None):
fs.delete(oldfile._id)
oid = fs.put(file,
content_type=file.content_type,
filename=isocode + '-' + objid + mediatypes[mediatype]['ext'])
return True
def drop_collection(self, host, port, dbname, collection_name):
try:
c = MongoConnection(host=host, port=int(port), pool_size=1)
db = database.Database(c, dbname)
db.drop_collection(collection_name)
except Exception, err:
return -1
def del_files_of_object(objid, types=['all']):
allowed_types = ['images', 'thumbs', 'videos', 'audios']
if (types == ['all']):
types = allowed_types
for filetype in types:
if filetype in allowed_types:
dbname = mediatypes[filetype]['db']
ext = mediatypes[filetype]['ext']
gridfsdb = database.Database(
MongoClient(host=GRIDFS_HOST, port=GRIDFS_PORT), dbname)
fs = GridFS(gridfsdb)
if (filetype in ['audios', 'videos']):
languages = mongodb.db.languages.find({}, {
'_id': 0,
'code': 1,
'locale': 1
})
for language in languages:
isocode = ','.join([language['code'], language['locale']])
fileid = fs.find_one({'filename': isocode + objid + ext})
if (fileid is not None):
fs.delete(fileid._id)
else:
fileid = fs.find_one({'filename': objid + ext})
if (fileid is not None):
fs.delete(fileid._id)
else:
return False
def query(self, host, port, dbname, query):
try:
c = MongoConnection(host=host, port=int(port), pool_size=1)
db = database.Database(c, dbname)
if not query.startswith('db.'):
return -2
res = eval(query)
if isinstance(res, (GeneratorType, ListType, Cursor)):
entries = []
try:
for r in res:
tmp = {}
for k, v in r.items():
if isinstance(v, datetime):
tmp[k] = strftime('%Y-%m-%d %X',
datetime.timetuple(v))
else:
tmp[k] = v
entries.append(tmp)
return {'count': len(entries), 'entries': entries}
except:
return str(res)
else:
return str(res)
except Exception, err:
return -1
def setupcode():
if (not session.get('logged_in')):
return redirect(url_for('login'))
gridfsdb = database.Database(
MongoClient(host=GRIDFS_HOST, port=GRIDFS_PORT), 'certs')
fs = GridFS(gridfsdb)
filename = 'rsa_1024_pub.pem'
if (fs.exists(filename=filename)):
file = fs.get_last_version(filename)
pubkey = file.read().replace('\n', '').replace(
'-----BEGIN PUBLIC KEY-----',
'').replace('-----END PUBLIC KEY-----', '')
server_address = mongodb.db.configs.find_one({}, {
'_id': 0,
'server_address': 1
})['server_address']
qrdata = pyqrcode.create(server_address + '|' + pubkey, mode='binary')
output = StringIO()
qrdata.svg(output, scale=6)
contents = output.getvalue()
output.close()
response = virtualrest.response_class(contents,
direct_passthrough=True,
mimetype='image/svg+xml')
response.headers.set('Content-Length', len(contents))
return response
def get_mongod():
"""
返回mongodb链接
:return:
"""
DB = database.Database(
MongoClient(host=host, port=port, tz_aware=False, connect=True),
dbname)
return DB
def __getattr__(self, name):
"""Get a database by name.
Raises :class:`~pymongo.errors.InvalidName` if an invalid
database name is used.
:Parameters:
- `name`: the name of the database to get
"""
return database.Database(self, name)
def __init__(self):
if 0:
from pymongo import database
self._conn = pymongo.Connection()
self._db = database.Database()
self._conn = None
self._db = None
self.cls_infos = None
self.url = None
self.dbkw = None
def connect_collection(db_name, collection_name, local=False):
""" Connect to DB and get collection, return collection object
for small scope use """
if local:
connection = Connection('localhost', 27017)
db_connect = database.Database(connection, db_name)
return collection.Collection(db_connect, collection_name)
host = get_config('database', 'host')
port = int(get_config('database', 'port'))
user = get_config('database', 'user')
password = get_config('database', 'password')
connection = Connection(host, port)
db_connect = database.Database(connection, db_name)
db_connect.authenticate(user, password)
return collection.Collection(db_connect, collection_name)
def __init__(self, name=''):
if 0:
from pymongo import database
self._conn = pymongo.Connection()
self._db = database.Database()
self._conn = None
self._db = None
self._name = name
self.username = None
self.pwd = None
self._pool = None
def __init__(self, mongo_client, db, collection='fs'):
"""
Create new manager
@param mongo_client: Mongo client
@param db: name of MongoDB database
@param collection: name of gridfs collection
"""
self.client = mongo_client
self.db = database.Database(mongo_client, db)
self.gfs = gridfs.GridFS(self.db, collection)
self.files_collection = self.db[collection + '.files']
def connect_db(db_name, local=False, master=False, user=None, password=None):
""" Connect to db, returns db object """
if local:
connection = Connection('localhost', 27017)
return database.Database(connection, db_name)
host = get_config('database', 'host')
if master:
host = get_config('database', 'master_host')
port = int(get_config('database', 'port'))
if not user:
user = get_config('database', 'user')
if not password:
password = get_config('database', 'password')
connection = Connection(host, port)
db_connect = database.Database(connection, db_name)
db_connect.authenticate(user, password)
return db_connect
def __init__(self):
if 0:
from pymongo import database
self._conn = pymongo.MongoClient()
self._db = database.Database()
self._conn = None
self._db = None
self.cls_infos = None
self.url = None
self.dbkw = None
self.tables = {}
def create_obj_img_intodb(imagefile, objid):
sizes = {'images': (320, 240), 'thumbs': (64, 64)}
for filetype in sizes.keys():
output = StringIO()
im = Image.open(imagefile)
im.thumbnail(sizes[filetype], Image.ANTIALIAS)
im.save(output, format="PNG")
contents = output.getvalue()
output.close()
gridfsdb = database.Database(
MongoClient(host=GRIDFS_HOST, port=GRIDFS_PORT), filetype)
fs = GridFS(gridfsdb)
del_files_of_object(objid, types=[filetype])
fs.put(contents,
content_type=imagefile.content_type,
filename=objid + '.png')
del (contents)
def get_database(self,
name,
codec_options=None,
read_preference=None,
write_concern=None,
read_concern=None):
"""Get a :class:`~pymongo.database.Database` with the given name and
options.
Useful for creating a :class:`~pymongo.database.Database` with
different codec options, read preference, and/or write concern from
this :class:`MongoClient`.
>>> client.read_preference
Primary()
>>> db1 = client.test
>>> db1.read_preference
Primary()
>>> from pymongo import ReadPreference
>>> db2 = client.get_database(
... 'test', read_preference=ReadPreference.SECONDARY)
>>> db2.read_preference
Secondary(tag_sets=None)
:Parameters:
- `name`: The name of the database - a string.
- `codec_options` (optional): An instance of
:class:`~bson.codec_options.CodecOptions`. If ``None`` (the
default) the :attr:`codec_options` of this :class:`MongoClient` is
used.
- `read_preference` (optional): The read preference to use. If
``None`` (the default) the :attr:`read_preference` of this
:class:`MongoClient` is used. See :mod:`~pymongo.read_preferences`
for options.
- `write_concern` (optional): An instance of
:class:`~pymongo.write_concern.WriteConcern`. If ``None`` (the
default) the :attr:`write_concern` of this :class:`MongoClient` is
used.
- `read_concern` (optional): An instance of
:class:`~pymongo.read_concern.ReadConcern`. If ``None`` (the
default) the :attr:`read_concern` of this :class:`MongoClient` is
used.
"""
return database.Database(self, name, codec_options, read_preference,
write_concern, read_concern)
def do_verify():
dev_uuid = request.args.get('uuid')
salt = request.args.get('salt')
if ((dev_uuid and salt) is None):
abort(403)
griddb = database.Database(MongoClient(host=GRIDFS_HOST, port=GRIDFS_PORT),'certs')
fs = GridFS(griddb)
find_result = mongodb.db.trusts.find_one({'uuid': dev_uuid},{'_id':0,'secret':1})
if (find_result is None):
abort(403)
else:
hash = hashlib.md5('%s:%s:%s' % (dev_uuid,find_result['secret'],salt)).hexdigest()
mongodb.db.trusts.update_one(
{'uuid': dev_uuid},
{'$set': { 'timestamp': int(time()) } },
upsert=False
)
session['logged_in'] = True
return output_json({'Result' : hash}, 200, def_headers)
def browse_collection(self, host, port, dbname, collection_name, page,
step):
try:
c = MongoConnection(host=host, port=int(port), pool_size=1)
db = database.Database(c, dbname)
cl = MongoCollection.Collection(db, collection_name)
res = cl.find(skip=(int(page) - 1) * int(step), limit=int(step))
entries = []
for r in res:
tmp = {}
for k, v in r.items():
if isinstance(v, datetime):
tmp[k] = strftime('%Y-%m-%d %X', datetime.timetuple(v))
else:
tmp[k] = v
entries.append(tmp)
return {'count': cl.count(), 'entries': entries}
except Exception, err:
return -1
def del_obj_media(objid, isocode, mediatype):
if (mediatype not in ['audios', 'videos']):
return False
gridfsdb = database.Database(
MongoClient(host=GRIDFS_HOST, port=GRIDFS_PORT),
mediatypes[mediatype]['db'])
fs = GridFS(gridfsdb)
mongodb.db.objects.update_one(
{
'id': objid,
'translations.isocode': isocode
}, {'$set': {
'translations.$.audio': False
}})
oldfile = fs.find_one(
{'filename': isocode + '-' + objid + mediatypes[mediatype]['ext']})
if (oldfile is not None):
fs.delete(oldfile._id)
return True
def get_file(rtype, filename):
if (not session.get('logged_in')):
return redirect(url_for('login'))
if (rtype not in ['audios', 'videos', 'flags', 'images', 'thumbs']):
abort(404)
gridfsdb = database.Database(
MongoClient(host=GRIDFS_HOST, port=GRIDFS_PORT), rtype)
fs = GridFS(gridfsdb)
if (fs.exists(filename=filename)):
file = fs.get_last_version(filename)
mime = mimetypes.guess_type(filename)[0]
response = virtualrest.response_class(file,
direct_passthrough=True,
mimetype=mime)
response.headers.set('Content-Length', file.length)
response.headers.set('Cache-Control',
'no-cache, no-store, must-revalidate')
response.headers.set('Pragma', 'no-cache')
response.headers.set('Expires', '0')
return response
abort(404)
def set_setup():
dev_uuid = request.args.get('uuid')
code = request.args.get('code')
salt = request.args.get('salt')
if ((dev_uuid and code and salt) is None):
abort(403)
griddb = database.Database(MongoClient(host=GRIDFS_HOST, port=GRIDFS_PORT),'certs')
fs = GridFS(griddb)
if (fs.find_one({'filename': 'rsa_1024_priv.pem'})):
key_string = fs.get_last_version('rsa_1024_priv.pem').read()
priv = RSA.load_key_string(key_string)
try:
ctxt = b64decode(code.replace(' ','+'))
decrypted_text = priv.private_decrypt(ctxt, RSA.pkcs1_padding)
hash = hashlib.md5('%s:%s:%s' % (dev_uuid,decrypted_text,salt)).hexdigest()
except TypeError as e:
return output_json({'Result' : 'Error: %s' % e}, 200, def_headers)
find_result = mongodb.db.trusts.find_one({'uuid': dev_uuid},{'_id':0,'timestamp':1})
if (find_result is not None):
# Timeout after 600 seconds (10 minutes)
# After timeout, users are able to setup trustee again
if (int(find_result['timestamp']) + 600 <= time()):
update_result = mongodb.db.trusts.update_one(
{'uuid': dev_uuid},
{'$set': {'secret': decrypted_text, 'timestamp': int(time())} },
upsert=True
)
else:
return output_json({'Result' : 'Too early'}, 403, def_headers)
else:
update_result = mongodb.db.trusts.update_one(
{'uuid': dev_uuid},
{'$set': {'secret': decrypted_text, 'timestamp': int(time())} },
upsert=True
)
session['logged_in'] = True
return output_json({'Result' : hash}, 200, def_headers)
def upload_file(filename):
if (not session.get('logged_in')):
return redirect(url_for('login'))
check_access('main_config')
if (filename not in ['avatar', 'background', 'logo']):
abort(403)
sizes = {'avatar': (64, 64), 'background': (400, 300), 'logo': (320, 240)}
gridfsdb = database.Database(
MongoClient(host=GRIDFS_HOST, port=GRIDFS_PORT), 'images')
fs = GridFS(gridfsdb)
output = StringIO()
file = request.files['file']
im = Image.open(file)
if (filename == 'background'):
color_thief = ColorThief(file)
pallete = color_thief.get_palette(color_count=2)
result_color, parcial_color = '', ''
for t in pallete:
for c in t:
parcial_color += hex(c).lstrip('0x')
result_color += '#' + parcial_color + ','
parcial_color = ''
mongodb.db.configs.update({}, {'$set': {'bgcolor': result_color[:-1]}})
im.thumbnail(sizes[filename], Image.ANTIALIAS)
im.save(output, format="PNG")
contents = output.getvalue()
output.close()
oldfile = fs.find_one({'filename': filename + '.png'})
if (oldfile is not None):
fs.delete(oldfile._id)
oid = fs.put(contents,
content_type=file.content_type,
filename=filename + '.png')
del (contents)
return redirect(url_for('main_config'))
from pymongo import database
import datetime
import nltk
from nltk.collocations import *
from nltk.probability import FreqDist
from pytz import timezone
import pytz
city = 'berlin'
bigram_measures = nltk.collocations.BigramAssocMeasures()
trigram_measures = nltk.collocations.TrigramAssocMeasures()
#### MongoDB connect
connection = Connection('mongodb://*****:*****@localhost:20707')
db = database.Database(connection, 'datasift')
# db.authenticate('norrix_mongoadmin', 'EntIftub')
col = db[city]
time_dict = {
"berlin": "Europe/Berlin",
"potsdam": "Europe/Berlin",
"munchen": "Europe/Berlin",
"rosenheim": "Europe/Berlin",
"newyork": "US/Eastern",
"sanfrancisco": "US/Pacific",
"menlopark": "US/Pacific",
"curpertino": "US/Pacific"
}
fmt = '%Y-%m-%d %H:%M:%S %Z%z'
from pymongo import database, MongoClient
import json
from mbid.settings import BOT_NAME
connection = MongoClient()
db = database.Database(connection, BOT_NAME)
class MongoPipeline(object):
def process_item(self, item, spider):
setattr(self, 'spider', spider)
db[self.spider.name].insert(dict(item))
return item
__author__ = 'mamad'
import cv
import cv2
from pymongo import MongoClient
from pymongo import database
from bson.objectid import ObjectId
import gridfs
import numpy as np
client = MongoClient()
db = database.Database(client, 'peach-dev')
gfs = gridfs.GridFS(db, 'fs')
id1 = ObjectId('524b77b43bbf498614000001')
img1 = gfs.get(id1)
print img1.metadata
print img1.filename
print img1.upload_date
img_array = np.asarray(bytearray(img1.read()), dtype=np.uint8)
img_data_ndarray = cv2.imdecode(img_array, 0)
frame = cv.fromarray(img_data_ndarray)
cv.ShowImage('mongo image ', frame)
while 1:
# do forever
# handle events
k = cv.WaitKey(10)
if k == 0x1b: # ESC
print 'ESC pressed. Exiting ...'
break
def get_file(rtype,filename):
MAX_SIZE=2097152 # 2MB
if (not session.get('logged_in')):
abort(403)
if (rtype not in ['audios','videos','flags','images', 'thumbs']):
abort(404)
griddb = database.Database(MongoClient(host=GRIDFS_HOST, port=GRIDFS_PORT),rtype)
fs = GridFS(griddb)
if (fs.exists(filename=filename)):
file = fs.get_last_version(filename)
file_length = file.length
chunk_length = file_length
mime = guess_type(filename)[0]
range_header = False
if ('Range' in request.headers.keys()):
range_header = True
start, end = request.headers['Range'].split('=')[1].split(',')[0].split('-')
if (end == '' or int(end) > file_length):
end = file_length
if (start == ''):
start = file_length - int(end)
chunk_file = StringIO()
end = int(end)
start = int(start)
file.seek(start)
chunk_file.write(file.read(end))
chunk_file.seek(0)
chunk_length = end - start
file.close()
else:
file.seek(0)
def generate():
while True:
if (range_header):
chunk = chunk_file.read(MAX_SIZE)
else:
chunk = file.read(MAX_SIZE)
if not chunk:
break
yield chunk
if (chunk_length > MAX_SIZE):
if (range_header):
response = Response(stream_with_context(generate()), 206, mimetype = mime)
response.headers.set('Content-Range', 'bytes %d-%d/%d' % (start, (start + chunk_length) - 1, file_length))
else:
response = Response(stream_with_context(generate()), 200, mimetype = mime)
response.headers.set('Content-Length',file_length)
else:
if (range_header):
response = virtualrest.response_class(chunk_file, 206, direct_passthrough = True, mimetype = mime)
response.headers.set('Content-Range', 'bytes %d-%d/%d' % (start, (start + chunk_length) - 1, file_length))
else:
response = virtualrest.response_class(file, 200, direct_passthrough = True, mimetype = mime)
response.headers.set('Content-Length',file_length)
if (rtype in ['audio','video']):
response.headers.set('Cache-Control', 'no-cache, no-store, must-revalidate')
response.headers.set('Pragma','no-cache')
response.headers.set('Accept-Ranges','bytes')
return response
abort(404)
__author__ = 'sapphire'
from pymongo import MongoClient, collection, database
con = MongoClient("localhost", 27017)
db = database.Database(con, 'mydb')
def insert_bus_data(data):
db.att.insert_one(data)
def update_bus_data(no, data):
db.att.replace_one({'bus_id': no}, data, True)
# for i in range(39):
# update_bus_data(i+1,{'bus_id':i+1,'attack':1})
#db.att.remove()
#for i in range(39):
# insert_bus_data({'bus_id':i+1,'attack':0})
# for i in db.att.find():
# print i
__author__ = 'mamad'
from pymongo import MongoClient
from pymongo import database
import gridfs
client = MongoClient()
db = database.Database(client, 'face-test')
gfs = gridfs.GridFS(db, 'fs')
f = open('../tests/images/uwa-f2f.jpg', mode='rb')
image_id = gfs.put(f, metadata={'test': 'test'})
print image_id
img1 = gfs.get(image_id)
print img1.metadata
print img1.filename
print img1.upload_date
