def _StorableObject(obj):
# Mongo can handle integer types directly
if isinstance(obj, (int, long)):
return obj
elif isinstance(obj, str):
return binary.Binary(obj)
try:
return utils.SmartUnicode(obj)
except UnicodeError:
# We can store a binary object but regex dont apply to it:
return binary.Binary(obj)
def test_mongodb_origin_simple_with_BSONBinary(sdc_builder, sdc_executor, mongodb):
"""
Create 3 simple documents consists with BSON Binary data type in MongoDB and confirm that MongoDB origin reads them.
The pipeline looks like:
mongodb_origin >> trash
"""
ORIG_BINARY_DOCS = [
{'data': binary.Binary(b'Binary Data Flute')},
{'data': binary.Binary(b'Binary Data Oboe')},
{'data': binary.Binary(b'Binary Data Violin')}
]
pipeline_builder = sdc_builder.get_pipeline_builder()
pipeline_builder.add_error_stage('Discard')
mongodb_origin = pipeline_builder.add_stage('MongoDB', type='origin')
mongodb_origin.set_attributes(capped_collection=False,
database=get_random_string(ascii_letters, 5),
collection=get_random_string(ascii_letters, 10))
trash = pipeline_builder.add_stage('Trash')
mongodb_origin >> trash
pipeline = pipeline_builder.build().configure_for_environment(mongodb)
try:
# MongoDB and PyMongo add '_id' to the dictionary entries e.g. docs_in_database
# when used for inserting in collection. Hence the deep copy.
docs_in_database = copy.deepcopy(ORIG_BINARY_DOCS)
# Create documents in MongoDB using PyMongo.
# First a database is created. Then a collection is created inside that database.
# Then documents are created in that collection.
logger.info('Adding documents into %s collection using PyMongo...', mongodb_origin.collection)
mongodb_database = mongodb.engine[mongodb_origin.database]
mongodb_collection = mongodb_database[mongodb_origin.collection]
insert_list = [mongodb_collection.insert_one(doc) for doc in docs_in_database]
assert len(insert_list) == len(docs_in_database)
# Start pipeline and verify the documents using snaphot.
sdc_executor.add_pipeline(pipeline)
snapshot = sdc_executor.capture_snapshot(pipeline=pipeline, start_pipeline=True).snapshot
sdc_executor.stop_pipeline(pipeline)
rows_from_snapshot = [{'data': str(record.value2['data'])} for record in snapshot[mongodb_origin].output]
assert rows_from_snapshot == [{'data': str(record.get('data'))} for record in ORIG_BINARY_DOCS]
finally:
logger.info('Dropping %s database...', mongodb_origin.database)
mongodb.engine.drop_database(mongodb_origin.database)
def novel_edit(req, novel_id):
obj_id = bson.objectid.ObjectId(oid=novel_id)
obj = models.NovelModel.objects.get(id=obj_id)
if req.method == "GET":
return render(req, 'novel/novel_edit.html', {"obj": obj})
if req.method == "POST":
title = req.POST.get('title', '')
author = req.POST.get('author', '')
tags = req.POST.get('tags', '')
tag_list = []
if ' ' in tags:
tag_list = tags.split(' ')
else:
tag_list.append(tags)
file = req.FILES.get('img_content')
if file:
obj.update(
title=title,
author=author,
tags=tag_list,
img_content=binary.Binary(file.read()),
)
else:
obj.update(
title=title,
author=author,
tags=tag_list,
)
return redirect('/novel/novel_manager/')
pass
def novel_add(req):
'''
title
author
tags
:param req:
:return:
'''
if req.method == 'POST':
title = req.POST.get('title', '')
author = req.POST.get('author', '')
tags = req.POST.get('tags', '')
tag_list = []
if ' ' in tags:
tag_list = tags.split(' ')
else:
tag_list.append(tags)
file = req.FILES.get('img_content')
models.NovelModel.objects.create(title=title,
author=author,
tags=tag_list,
img_content=binary.Binary(
file.read()))
return redirect('/novel/novel_manager/')
pass
def close_spider(self, spider):
# assume map image is saved before crawling text
with open('airportmap.png', 'rb') as f:
data = f.read()
self.document['image'] = binary.Binary(data)
self.mcol.insert_one(self.document)
self.mclient.close()
os.remove('airportmap.png')
def save_img_bson(self,item):
client = pymongo.MongoClient("mongodb://localhost:27017/")
db = client['doubanTop']
img_collection = db['movie_bson']
img_binary = requests.get(item['m_img_url'],headers=self.headers,timeout=10).content
if not img_collection.find_one({'img_url':item['m_img_url']}):
item['img_binary'] = binary.Binary(img_binary)
img_collection.insert_one(item)
print('Save successfully!')
def mongoTest02():
client = MongoClient('localhost', 27017)
db = client.infos
image_collection = db.images
dic = {}
data = requests.get(dic['photo_url'], timeout=10).content
if not image_collection.find_one({'photo_url': dic['photo_url']}):
dic['imagecontent'] = binary.Binary(data)
image_collection.insert(dic)
def getdata(self, docid, url): #获取数据
rsp = self.s.get(url=url, verify=False)
data = rsp.content
#now=time.time()
#crawl_time = time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(now)) ##抓取时间
dc = {'docid': docid, 'thumb_url': url}
dc['data'] = binary.Binary(data)
self.my_set.insert(dc)
def post_image():
f = request.files['photo']
if f:
imgstr = cStringIO.StringIO(f.stream.read())
img = Image.open(imgstr)
img.convert('RGB')
img.thumbnail((300, 300))
imgbuf = cStringIO.StringIO()
img.save(imgbuf, format='JPEG')
gitshot = dict(img=binary.Binary(imgbuf.getvalue()))
return str(mongo.db.gitshots.insert(gitshot))
return 400
def encode_item(trait, value):
if value is None:
return value
elif hasattr(trait, 'reference'):
value = trait.reference(value)
elif 'savedict' in dir(value):
value = value.savedict
elif isinstance(trait, traitlets.Container):
value = [Document.encode_item(trait._trait,elem) for elem in value]
elif not isinstance(value, SAME_TYPES):
value = binary.Binary(pickle.dumps(value))
return value
def add_pubkey(self, user, blob, name="unnamed", fingerprint=""):
"""Add a new public key for a user"""
if self.user_has_pubkey(user, blob):
return False
self.database.pubkeys.insert({
'userid': user.userid,
'blob': binary.Binary(blob),
'name': name,
'fingerprint': fingerprint
})
return True
def test_sort_key_fetcher():
doc1 = {'a': ['a', 'b', 'c', binary.Binary('AAAA', 0)]}
doc2 = {'a': [{'b': 1, 'c': 1}, {'b': 2, 'c': 2}]}
sort1 = [('a', True)]
sort2 = [('a.c', True), ('a.b', False)]
# query1 = {'a.b': {'$gt':2}}
query1 = {'a': {'$type': 2}}
fetcher = SortKeyFetcher(query1, doc1, ModelOptions(''))
result = tuple([fetcher.get_sort_value(k, dir) for (k, dir) in sort1])
print result
def saveFileToDB(mongo, username, dmname, filecontent):
# Parameters:
#######################################################
# username: str or unicode
# dmname: str or unicode
# filecontent : str, text content of the model filecontent
# pathid : str, uuid, direct to the corresponding output path
# Returns: bool, True if successfully saved
if type(filecontent) is not str:
return False
# to save file as bson, we need base64 encode first
b64content = base64.standard_b64encode(filecontent)
# get bson object
bincontent = binary.Binary(b64content)
fileid = mongo.db.filedb.insert({"file": bincontent})
#if this user doesn't upload history in database, create one
userresult = mongo.db.history.find_one({"username": username})
if userresult == None:
mongo.db.history.insert({"username": username, "uploads": []})
domainresult = mongo.db.history.find_one({
"username": username,
"uploads": {
"$elemMatch": {
"domainModelName": dmname
}
}
})
if domainresult == None:
mongo.db["history"].update({"username": username}, {
"$push": {
"uploads": {
"domainModelName": dmname,
"files": []
}
}
})
mongo.db["history"].update(
{
"username": username,
"uploads.domainModelName": dmname
}, {
"$push": {
"uploads.$.files": {
"file": fileid,
"date": dt.datetime.now(pytz.utc)
}
}
})
return fileid
def save_to_mongo(self, _hash, picture, url):
# mongo_gfs = GridFS(Mongo['screenshot'])
_file = '/tmp/%s.png' % _hash
_t_file = '/tmp/%s_t.png' % _hash
_pic = open(_file, 'w')
_pic.write(picture)
_pic.close()
size = 260, 150
im = Image.open(_file)
im.crop((0, 0, self.display_width, self.display_height)).save(_file)
im.thumbnail(size, Image.ANTIALIAS)
im.save(_t_file)
_thum = binary.Binary(open(_t_file, mode='rb').read())
_binary = binary.Binary(open(_file, mode='rb').read())
os.remove(_file)
os.remove(_t_file)
self.browser.close()
# self.display.stop()
return Mongo['screenshot']['store'].update_one({'hash': _hash}, {
'$set': {
'url': url,
'hash': _hash,
'uuid': str(uuid.uuid4()),
'file': _binary,
'thumbnail': _thum,
'content-type': 'image/png'
}
},
upsert=True)
def _store_features(fv, photo_id):
# store the extracted features
t0 = 0
if _DEBUG:
t0 = time.time()
num_segments = fv.shape[0]
bin_fv = bbin.Binary(pickle.dumps(fv, protocol=2), subtype=128)
col_seg_fv.insert_one({'Photo_id': photo_id,
'n_segments': num_segments,
'features': bin_fv})
if _DEBUG:
print 'feature storing time: ', time.time() - t0
""" loading the binary dump
def insert_file(self, zoom, row, column, path):
# 将data 存到mongo中
level = zoom[1:]
if int(level) <= self.max_level:
png_list = self.my_set.find({"x": row, "y": column, "z": zoom})
if png_list.count() == 0:
print('zoom:%s row:%s fileName:%s path:%s ' % (zoom, row, column, path))
file = open(path, mode='rb')
content = binary.Binary(file.read())
self.my_set.insert({"x": row, "y": column, "z": zoom, "img": content})
file.close()
else:
print('this file is exist')
else:
print('this zoom greater than max level')
def update_instance():
if request.method == 'POST':
oldfile_id = request.form['fileId']
domain_model_name = request.form['domainModelName']
file = request.files['file']
filename_str = ""
output_dir = os.path.join(config.get('Output', 'output_path')) + "/" + session['username']
if file and allowed_file(file.filename):
filename = secure_filename(file.filename)
filename_str = filename.split(".")[0]
all_content = file.read()
b64content = base64.standard_b64encode(all_content)
# get bson object
bincontent = binary.Binary(b64content)
#print all_content
# save file into database
#newfile_id = dbOps.saveFileToDB(mongo, current_user.username, filename_str, all_content)
#print newfile_id
# save file to path
output_dir = output_dir + "/" + filename_str + "/" + str(oldfile_id)
with fileOps.safe_open_w(output_dir + "/" + filename) as f:
f.write(all_content)
f.close()
username = session['username']
dbOps.updateInstanceDb(mongo, username, domain_model_name, oldfile_id, bincontent)
file_dir = os.path.join(config.get('Output', 'output_path')) + "/" + username + "/" + domain_model_name + "/" + str(oldfile_id)
# Parse XML and generate JSON
ana.DM_File_Analyze(output_dir, {'DM_Input_type': "Simple_XML"}, filename_str)
return redirect(url_for('index'))
#return redirect(url_for('index'))
return redirect(url_for('update_instance'))
def _fix_json(d):
if d in [None, [], {}]: #if not d: breaks empty Binary
return d
data_type = type(d)
if data_type == list:
data = []
for item in d:
data.append(_fix_json(item))
return data
elif data_type == dict:
data = {}
if '$binary' in d: #base64 encoded data
return binary.Binary(base64.decodestring(d['$binary']),
d['$type'])
else:
for k in d:
data[_fix_json(k)] = _fix_json(d[k])
return data
else:
return d
def update_apps_installed(self, client_id, user_app_data):
cond = {'_id': client_id}
apps = user_app_data.get('apps', None)
if apps is None:
return
apps = [
app.get('packageName', '') + ':' +
str(app.get('packageVersion', '')) for app in apps
]
data_str = '|'.join(apps)
data_bytes = data_str.encode('zip')
data_binary = binary.Binary(data_bytes)
self._db.apps.installed.update(cond, {
'_id': client_id,
'apps': data_binary
},
True,
w=0)
def save_file(f): content = StringIO(f.read()) try: mime = Image.open(content).format.lower() if mime not in allow_formats: raise IOError() except IOError: abort(400) sha1 = hashlib.sha1(content.getvalue()).hexdigest() c = dict( content=binary.Binary(content.getvalue()), mime=mime, time=datetime.datetime.utcnow(), sha1=sha1, ) try: db.files.save(c) except pymongo.errors.DuplicateKeyError: pass return sha1
def process_file_obj(*, doc, files):
for attr in doc.keys():
if attr in files.keys():
doc[attr] = []
for file in files[attr].values():
doc[attr].append({
'name':
file['name'],
'lastModified':
file['lastModified'],
'type':
file['type'],
'size':
file['size'],
'content':
binary.Binary(
bytes([
int(byte) for byte in file['content'].split(',')
])),
})
del files[attr]
return doc
def _Encode(document, value):
"""Encodes the value into the document.
Args:
document: The mogo document which will receive this new value.
value: A value to be encoded in the database.
Returns:
The modified document.
"""
if hasattr(value, "SerializeToDataStore"):
value = value.SerializeToDataStore()
elif hasattr(value, "SerializeToString"):
value = value.SerializeToString()
if isinstance(value, (long, int)):
document["int_value"] = value
elif isinstance(value, str):
document["value"] = binary.Binary(value)
else:
document["str_value"] = utils.SmartUnicode(value)
return document
def save_file(f):
content = BytesIO(f.read())
content.seek(0, os.SEEK_END)
size = content.tell()
content.seek(0, os.SEEK_SET)
try:
filename = secure_filename(f.filename)
if filename:
if '.' in filename:
mime = filename.rsplit('.', 1)[1]
else:
mime = filename
t = str(time.time())
str1 = filename + t
hash1 = hashlib.md5()
hash1.update(str1.encode('UTF-8'))
toHash = hash1.hexdigest()
id = toHash + "." + mime
if mime not in ALLOWED_EXTENSIONS.keys():
raise IOError()
except IOError:
return 0
c = dict(content=binary.Binary(content.getvalue()),
mime=mime,
time=datetime.datetime.utcnow(),
md5=toHash,
filename=filename,
size=size)
try:
file = File(**c)
file.save()
except pymongo.errors.DuplicateKeyError:
return jsonify({
"code": 400,
"msg": "数据保存失败",
})
return id
def random_binary(length):
b64 = base64.b64encode(random_string(length))
return binary.Binary(b64, global_prng.choice([0, 1]))
def decode(datas, hash):
from bson import binary
tt = []
dict = {}
data = datas
client = MongoClient()
db = client.local
collection = db.vizdata
bin = db.binaries
binData = binary.Binary(base64.b64encode(data))
print bin.insert({'s1' : hash, 'data' : binData})
print "Inserted binary"
for dat in collection.find():
print "Finding.."
if dat['s1'] == hash:
print "FILE ALREADY EXISTS"
return 0
print "DECODING TWOTUPLE DATA... " #TWO TUPLE
i = 0
for c in data:
tt.append(ord(c))
threetup = tt
for i in range(0, len(tt)-1):
dict.update({(tt[i], tt[i+1]):1})
for i in range(0,len(tt)-1):
dict.update({(tt[i], tt[i+1]):dict[(tt[i], tt[i+1])]+1})
tt = []
for x,y in dict:
tt.append((x,y,dict[x,y]))
dict = {}
print "Decoding histogram Data..." #HISTOGRAM
hist = []
import distorm3
from distorm3 import Decode, Decode16Bits, Decode32Bits, Decode64Bits
l = Decode(0xA30, data, Decode32Bits)
for i in l:
hist.append(i[2].split(' ')[0])
for i in hist:
dict.update({i:0})
for i in hist:
dict.update({i:(dict[i]+1)})
import math
hist = sorted(dict.items(), key=lambda val:val[0])
hist = hist[::]
print "Decoding Three Tuple Data...." #THREE TUPLE
slide = 3
points = 15000000000
x=y=z=0
dict = {}
for i in xrange(0, len(data)-2, slide):
dict.update({(threetup[i], threetup[i+1], threetup[i+2]):0})
for i in range(0,len(data)-2,slide):
dict.update({(threetup[i], threetup[i+1], threetup[i+2]):dict[(threetup[i], threetup[i+1],threetup[i+2])]+1})
num = int(len(dict)/points)+1
import numpy
arr = dict.keys()
threetup = []
if(num <= 1):
for x,y,z in dict:
threetup.append((x,y,z,dict[(x,y,z)]))
else:
for i in xrange(0,len(dict),num):
ctr =[0,0,0,0]
for j in range(0, num): #SMALLER BLOCK OF DATA TO BE AVERAGED
if i+j >= len(dict)-1 : break
for k in range(0,3):
ctr[k] = ctr[k]+arr[i+j][k]
ctr[3] = ctr[3]+dict[arr[i+j]]
for k in range(0,4):
ctr[k] = ctr[k]/num
threetup.append((ctr[0],ctr[1], ctr[2],ctr[3]))
print "Decoding ContactMap..." #CONTACT MAP DECODE
list = []
points = 225 #NO. OF POINTS ON CONTACT MAP
check = len(data)
size = check/points
for c in data:
b = ord(c)
b = float(b)
if b < 0 : b = b+256
list.append(b)
strings = []
for i in range(0, (len(list)/size)+1):
strings.append(list[size*i : (size*i)+size])
if len(strings[len(strings)-1]) < size :
for i in range(0, size-len(strings[len(strings)-1])):
strings[len(strings)-1].append(0)
import numpy
import scipy
import scipy.spatial
import math
val = [0,0,0]
length = len(strings)
cont = []
print length
i = j = 0
for i in xrange(0,length):
t1 = strings[i]
for j in range(0,length):
t2 = strings[j]
val[0] = round(100*scipy.spatial.distance.braycurtis(t1, t2), 2)
#val[0] = (100*scipy.spatial.distance.jaccard(strings[i], strings[j]))
#val[0] = (scipy.spatial.distance.sqeuclidean(strings[i], strings[j]))
#for k in range(0,2):
if math.isnan(val[0]) or (val[0] <= 10): val[0] = 0
else : cont.append((i, j,val[0]))#, val[1], val[2]])
if i%100 == 0 :
print i
print len(cont)
print cont[0:10]
import json,zlib
print "COmpressing data..." #DATA COMPRESSION
hist = json.dumps(hist)
hist = zlib.compress(hist)
tt = json.dumps(tt)
tt = zlib.compress(tt)
threetup = json.dumps(threetup)
threetup = zlib.compress(threetup)
cont = json.dumps(cont)
cont = zlib.compress(cont)
hist = base64.b64encode(hist)
tt = base64.b64encode(tt)
threetup = base64.b64encode(threetup)
cont = base64.b64encode(cont)
dict = {"s1" : hash, "hist" : hist, "t2" : tt ,"t3" : threetup, "cm":cont}
tt = threetup = hist = cont = []
print "Inserting..."
collection.insert(dict)
print "Database Created"
return 2
import pymongocrypt
from bson import binary
from bson.binary import (Binary, JAVA_LEGACY, STANDARD, UUID_SUBTYPE)
from bson.codec_options import CodecOptions
OPTS = CodecOptions(uuid_representation=STANDARD)
print(pymongocrypt.libmongocrypt_version())
client = MongoClient("localhost", 27017)
# ** A LOCAL TEST KEY SHOULD NEVER BE USED IN PRODUCTION, ONLY FOR DEVELOPMENT **
# Test key material generated on Mac & Linux with: echo $(head -c 96 /dev/urandom | base64 | tr -d '\n')
local_key = 'CgOcoan3c/wm2c+WsOO6fXOUlJgd7SLQ1vl///aEFX6vXN9+7VOAP+iHKheZiYlB09ZS7CDcAQhlPeTeQNz03xiGbiCJJvl3uj4lnG+5i/udSLJAcwgtgtaedkFD0ROq'
key_bin = binary.Binary(base64.b64decode(local_key))
kms_providers = {'local': {'key': key_bin}}
print(
"Please ensure mongocryptd is running (included in the standard enterprise download package)."
)
fle_opts = AutoEncryptionOpts(kms_providers,
"demoFLE.__keystore",
mongocryptd_bypass_spawn=True)
client = MongoClient("mongodb://localhost:27017/demoFLE",
auto_encryption_opts=fle_opts)
db = client.demoFLE
import base64
import os
from pymongo import MongoClient
from pymongo.errors import DuplicateKeyError
from pymongo.encryption import ClientEncryption
from pymongo.encryption_options import AutoEncryptionOpts
from bson import binary, ObjectId
from bson.codec_options import CodecOptions
# Setup.
codec_opts = CodecOptions(uuid_representation=binary.STANDARD)
if "LOCAL_MASTERKEY_BASE64" not in os.environ or not os.path.exists(
"key_uuid.txt"):
raise Exception("Prerequisites not met. Run setup.py")
local_key = os.environ["LOCAL_MASTERKEY_BASE64"]
masterkey = binary.Binary(base64.b64decode(local_key))
key_uuid = binary.Binary(base64.b64decode(open("key_uuid.txt", "r").read()), binary.UUID_SUBTYPE)
# Create a data key identified in the key vault collection lab7.key_vault.
from base64 import b64encode, b64decode
import os
from pymongo import MongoClient
from pymongo.encryption import ClientEncryption
from bson import binary
from bson.codec_options import CodecOptions
codec_opts = CodecOptions(uuid_representation=binary.STANDARD)
# Test key material generated by: echo $(head -c 96 /dev/urandom | base64 | tr -d '\n')
if "LOCAL_MASTER_KEY" not in os.environ:
raise Exception("Set LOCAL_MASTER_KEY env variable to 96 bytes of base64")
master_key = binary.Binary(b64decode(os.environ["LOCAL_MASTER_KEY"]))
# Reset the collection
key_vault_client = MongoClient("mongodb://localhost/")
key_vault_client.lab7.key_vault.drop()
# Configure a ClientEncryption object to create data keys
kms_providers = {"local": {"key": master_key}}
key_vault_client = MongoClient()
client_encryption = ClientEncryption(
kms_providers, "lab7.key_vault", key_vault_client, codec_opts)
key_uuid = client_encryption.create_data_key("local")
# Store the key id into a file for easy access
open("key_uuid.txt", "w").write(b64encode(key_uuid).decode("utf-8"))
def update_instance():
ana = p()
if request.method == 'POST':
oldfile_id = request.form['fileId']
domain_model_name = request.form['domainModelName']
file = request.files['file']
filename_str = ""
output_dir = os.path.join(config.get('Output', 'output_path')) + "/" + session['username']
if file and allowed_file(file.filename):
filename = secure_filename(file.filename)
filename_str = filename.split(".")[0]
all_content = file.read()
b64content = base64.standard_b64encode(all_content)
# get bson object
bincontent = binary.Binary(b64content)
#print all_content
# save file into database
#newfile_id = dbOps.saveFileToDB(mongo, current_user.username, filename_str, all_content)
#print newfile_id
# save file to path
output_dir = output_dir + "/" + filename_str + "/" + str(oldfile_id)
with fileOps.safe_open_w(output_dir + "/" + filename) as f:
f.write(all_content)
f.close()
username = session['username']
dbOps.updateInstanceDb(mgInstance.mongo, username, domain_model_name, oldfile_id, bincontent)
file_dir = os.path.join(config.get('Output', 'output_path')) + "/" + username + "/" + domain_model_name + "/" + str(oldfile_id)
# Parse XML and generate JSON
ana.DM_File_Analyze(output_dir, {'DM_Input_type': "Simple_XML"}, filename_str)
# Parse JSON and generate code
model_display_data, server_url = generate_code.generate_all(filename_str, output_dir)
authen_key = dbOps.getAuthenKey(mgInstance.mongo, session['username'])
# Pass required data to the template
description_data = {
"model_display_data": model_display_data,
"server_url": server_url,
"authen_key" : authen_key
}
# write description_data into json file
generate_code.write_description_to_file(filename_str, output_dir, description_data)
return redirect(url_for('main_bp.index'))
#return redirect(url_for('index'))
return redirect(url_for('main_bp.update_instance'))
def xueyansuo_tupian(self, url):
from bson import binary
data = requests.get(url, timeout=15).content
return binary.Binary(data)
