def _StorableObject(obj): # Mongo can handle integer types directly if isinstance(obj, (int, long)): return obj elif isinstance(obj, str): return binary.Binary(obj) try: return utils.SmartUnicode(obj) except UnicodeError: # We can store a binary object but regex dont apply to it: return binary.Binary(obj)
def test_mongodb_origin_simple_with_BSONBinary(sdc_builder, sdc_executor, mongodb): """ Create 3 simple documents consists with BSON Binary data type in MongoDB and confirm that MongoDB origin reads them. The pipeline looks like: mongodb_origin >> trash """ ORIG_BINARY_DOCS = [ {'data': binary.Binary(b'Binary Data Flute')}, {'data': binary.Binary(b'Binary Data Oboe')}, {'data': binary.Binary(b'Binary Data Violin')} ] pipeline_builder = sdc_builder.get_pipeline_builder() pipeline_builder.add_error_stage('Discard') mongodb_origin = pipeline_builder.add_stage('MongoDB', type='origin') mongodb_origin.set_attributes(capped_collection=False, database=get_random_string(ascii_letters, 5), collection=get_random_string(ascii_letters, 10)) trash = pipeline_builder.add_stage('Trash') mongodb_origin >> trash pipeline = pipeline_builder.build().configure_for_environment(mongodb) try: # MongoDB and PyMongo add '_id' to the dictionary entries e.g. docs_in_database # when used for inserting in collection. Hence the deep copy. docs_in_database = copy.deepcopy(ORIG_BINARY_DOCS) # Create documents in MongoDB using PyMongo. # First a database is created. Then a collection is created inside that database. # Then documents are created in that collection. logger.info('Adding documents into %s collection using PyMongo...', mongodb_origin.collection) mongodb_database = mongodb.engine[mongodb_origin.database] mongodb_collection = mongodb_database[mongodb_origin.collection] insert_list = [mongodb_collection.insert_one(doc) for doc in docs_in_database] assert len(insert_list) == len(docs_in_database) # Start pipeline and verify the documents using snaphot. sdc_executor.add_pipeline(pipeline) snapshot = sdc_executor.capture_snapshot(pipeline=pipeline, start_pipeline=True).snapshot sdc_executor.stop_pipeline(pipeline) rows_from_snapshot = [{'data': str(record.value2['data'])} for record in snapshot[mongodb_origin].output] assert rows_from_snapshot == [{'data': str(record.get('data'))} for record in ORIG_BINARY_DOCS] finally: logger.info('Dropping %s database...', mongodb_origin.database) mongodb.engine.drop_database(mongodb_origin.database)
def novel_edit(req, novel_id): obj_id = bson.objectid.ObjectId(oid=novel_id) obj = models.NovelModel.objects.get(id=obj_id) if req.method == "GET": return render(req, 'novel/novel_edit.html', {"obj": obj}) if req.method == "POST": title = req.POST.get('title', '') author = req.POST.get('author', '') tags = req.POST.get('tags', '') tag_list = [] if ' ' in tags: tag_list = tags.split(' ') else: tag_list.append(tags) file = req.FILES.get('img_content') if file: obj.update( title=title, author=author, tags=tag_list, img_content=binary.Binary(file.read()), ) else: obj.update( title=title, author=author, tags=tag_list, ) return redirect('/novel/novel_manager/') pass
def novel_add(req): ''' title author tags :param req: :return: ''' if req.method == 'POST': title = req.POST.get('title', '') author = req.POST.get('author', '') tags = req.POST.get('tags', '') tag_list = [] if ' ' in tags: tag_list = tags.split(' ') else: tag_list.append(tags) file = req.FILES.get('img_content') models.NovelModel.objects.create(title=title, author=author, tags=tag_list, img_content=binary.Binary( file.read())) return redirect('/novel/novel_manager/') pass
def close_spider(self, spider): # assume map image is saved before crawling text with open('airportmap.png', 'rb') as f: data = f.read() self.document['image'] = binary.Binary(data) self.mcol.insert_one(self.document) self.mclient.close() os.remove('airportmap.png')
def save_img_bson(self,item): client = pymongo.MongoClient("mongodb://localhost:27017/") db = client['doubanTop'] img_collection = db['movie_bson'] img_binary = requests.get(item['m_img_url'],headers=self.headers,timeout=10).content if not img_collection.find_one({'img_url':item['m_img_url']}): item['img_binary'] = binary.Binary(img_binary) img_collection.insert_one(item) print('Save successfully!')
def mongoTest02(): client = MongoClient('localhost', 27017) db = client.infos image_collection = db.images dic = {} data = requests.get(dic['photo_url'], timeout=10).content if not image_collection.find_one({'photo_url': dic['photo_url']}): dic['imagecontent'] = binary.Binary(data) image_collection.insert(dic)
def getdata(self, docid, url): #获取数据 rsp = self.s.get(url=url, verify=False) data = rsp.content #now=time.time() #crawl_time = time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(now)) ##抓取时间 dc = {'docid': docid, 'thumb_url': url} dc['data'] = binary.Binary(data) self.my_set.insert(dc)
def post_image(): f = request.files['photo'] if f: imgstr = cStringIO.StringIO(f.stream.read()) img = Image.open(imgstr) img.convert('RGB') img.thumbnail((300, 300)) imgbuf = cStringIO.StringIO() img.save(imgbuf, format='JPEG') gitshot = dict(img=binary.Binary(imgbuf.getvalue())) return str(mongo.db.gitshots.insert(gitshot)) return 400
def encode_item(trait, value): if value is None: return value elif hasattr(trait, 'reference'): value = trait.reference(value) elif 'savedict' in dir(value): value = value.savedict elif isinstance(trait, traitlets.Container): value = [Document.encode_item(trait._trait,elem) for elem in value] elif not isinstance(value, SAME_TYPES): value = binary.Binary(pickle.dumps(value)) return value
def add_pubkey(self, user, blob, name="unnamed", fingerprint=""): """Add a new public key for a user""" if self.user_has_pubkey(user, blob): return False self.database.pubkeys.insert({ 'userid': user.userid, 'blob': binary.Binary(blob), 'name': name, 'fingerprint': fingerprint }) return True
def test_sort_key_fetcher(): doc1 = {'a': ['a', 'b', 'c', binary.Binary('AAAA', 0)]} doc2 = {'a': [{'b': 1, 'c': 1}, {'b': 2, 'c': 2}]} sort1 = [('a', True)] sort2 = [('a.c', True), ('a.b', False)] # query1 = {'a.b': {'$gt':2}} query1 = {'a': {'$type': 2}} fetcher = SortKeyFetcher(query1, doc1, ModelOptions('')) result = tuple([fetcher.get_sort_value(k, dir) for (k, dir) in sort1]) print result
def saveFileToDB(mongo, username, dmname, filecontent): # Parameters: ####################################################### # username: str or unicode # dmname: str or unicode # filecontent : str, text content of the model filecontent # pathid : str, uuid, direct to the corresponding output path # Returns: bool, True if successfully saved if type(filecontent) is not str: return False # to save file as bson, we need base64 encode first b64content = base64.standard_b64encode(filecontent) # get bson object bincontent = binary.Binary(b64content) fileid = mongo.db.filedb.insert({"file": bincontent}) #if this user doesn't upload history in database, create one userresult = mongo.db.history.find_one({"username": username}) if userresult == None: mongo.db.history.insert({"username": username, "uploads": []}) domainresult = mongo.db.history.find_one({ "username": username, "uploads": { "$elemMatch": { "domainModelName": dmname } } }) if domainresult == None: mongo.db["history"].update({"username": username}, { "$push": { "uploads": { "domainModelName": dmname, "files": [] } } }) mongo.db["history"].update( { "username": username, "uploads.domainModelName": dmname }, { "$push": { "uploads.$.files": { "file": fileid, "date": dt.datetime.now(pytz.utc) } } }) return fileid
def save_to_mongo(self, _hash, picture, url): # mongo_gfs = GridFS(Mongo['screenshot']) _file = '/tmp/%s.png' % _hash _t_file = '/tmp/%s_t.png' % _hash _pic = open(_file, 'w') _pic.write(picture) _pic.close() size = 260, 150 im = Image.open(_file) im.crop((0, 0, self.display_width, self.display_height)).save(_file) im.thumbnail(size, Image.ANTIALIAS) im.save(_t_file) _thum = binary.Binary(open(_t_file, mode='rb').read()) _binary = binary.Binary(open(_file, mode='rb').read()) os.remove(_file) os.remove(_t_file) self.browser.close() # self.display.stop() return Mongo['screenshot']['store'].update_one({'hash': _hash}, { '$set': { 'url': url, 'hash': _hash, 'uuid': str(uuid.uuid4()), 'file': _binary, 'thumbnail': _thum, 'content-type': 'image/png' } }, upsert=True)
def _store_features(fv, photo_id): # store the extracted features t0 = 0 if _DEBUG: t0 = time.time() num_segments = fv.shape[0] bin_fv = bbin.Binary(pickle.dumps(fv, protocol=2), subtype=128) col_seg_fv.insert_one({'Photo_id': photo_id, 'n_segments': num_segments, 'features': bin_fv}) if _DEBUG: print 'feature storing time: ', time.time() - t0 """ loading the binary dump
def insert_file(self, zoom, row, column, path): # 将data 存到mongo中 level = zoom[1:] if int(level) <= self.max_level: png_list = self.my_set.find({"x": row, "y": column, "z": zoom}) if png_list.count() == 0: print('zoom:%s row:%s fileName:%s path:%s ' % (zoom, row, column, path)) file = open(path, mode='rb') content = binary.Binary(file.read()) self.my_set.insert({"x": row, "y": column, "z": zoom, "img": content}) file.close() else: print('this file is exist') else: print('this zoom greater than max level')
def update_instance(): if request.method == 'POST': oldfile_id = request.form['fileId'] domain_model_name = request.form['domainModelName'] file = request.files['file'] filename_str = "" output_dir = os.path.join(config.get('Output', 'output_path')) + "/" + session['username'] if file and allowed_file(file.filename): filename = secure_filename(file.filename) filename_str = filename.split(".")[0] all_content = file.read() b64content = base64.standard_b64encode(all_content) # get bson object bincontent = binary.Binary(b64content) #print all_content # save file into database #newfile_id = dbOps.saveFileToDB(mongo, current_user.username, filename_str, all_content) #print newfile_id # save file to path output_dir = output_dir + "/" + filename_str + "/" + str(oldfile_id) with fileOps.safe_open_w(output_dir + "/" + filename) as f: f.write(all_content) f.close() username = session['username'] dbOps.updateInstanceDb(mongo, username, domain_model_name, oldfile_id, bincontent) file_dir = os.path.join(config.get('Output', 'output_path')) + "/" + username + "/" + domain_model_name + "/" + str(oldfile_id) # Parse XML and generate JSON ana.DM_File_Analyze(output_dir, {'DM_Input_type': "Simple_XML"}, filename_str) return redirect(url_for('index')) #return redirect(url_for('index')) return redirect(url_for('update_instance'))
def _fix_json(d): if d in [None, [], {}]: #if not d: breaks empty Binary return d data_type = type(d) if data_type == list: data = [] for item in d: data.append(_fix_json(item)) return data elif data_type == dict: data = {} if '$binary' in d: #base64 encoded data return binary.Binary(base64.decodestring(d['$binary']), d['$type']) else: for k in d: data[_fix_json(k)] = _fix_json(d[k]) return data else: return d
def update_apps_installed(self, client_id, user_app_data): cond = {'_id': client_id} apps = user_app_data.get('apps', None) if apps is None: return apps = [ app.get('packageName', '') + ':' + str(app.get('packageVersion', '')) for app in apps ] data_str = '|'.join(apps) data_bytes = data_str.encode('zip') data_binary = binary.Binary(data_bytes) self._db.apps.installed.update(cond, { '_id': client_id, 'apps': data_binary }, True, w=0)
def save_file(f): content = StringIO(f.read()) try: mime = Image.open(content).format.lower() if mime not in allow_formats: raise IOError() except IOError: abort(400) sha1 = hashlib.sha1(content.getvalue()).hexdigest() c = dict( content=binary.Binary(content.getvalue()), mime=mime, time=datetime.datetime.utcnow(), sha1=sha1, ) try: db.files.save(c) except pymongo.errors.DuplicateKeyError: pass return sha1
def process_file_obj(*, doc, files): for attr in doc.keys(): if attr in files.keys(): doc[attr] = [] for file in files[attr].values(): doc[attr].append({ 'name': file['name'], 'lastModified': file['lastModified'], 'type': file['type'], 'size': file['size'], 'content': binary.Binary( bytes([ int(byte) for byte in file['content'].split(',') ])), }) del files[attr] return doc
def _Encode(document, value): """Encodes the value into the document. Args: document: The mogo document which will receive this new value. value: A value to be encoded in the database. Returns: The modified document. """ if hasattr(value, "SerializeToDataStore"): value = value.SerializeToDataStore() elif hasattr(value, "SerializeToString"): value = value.SerializeToString() if isinstance(value, (long, int)): document["int_value"] = value elif isinstance(value, str): document["value"] = binary.Binary(value) else: document["str_value"] = utils.SmartUnicode(value) return document
def save_file(f): content = BytesIO(f.read()) content.seek(0, os.SEEK_END) size = content.tell() content.seek(0, os.SEEK_SET) try: filename = secure_filename(f.filename) if filename: if '.' in filename: mime = filename.rsplit('.', 1)[1] else: mime = filename t = str(time.time()) str1 = filename + t hash1 = hashlib.md5() hash1.update(str1.encode('UTF-8')) toHash = hash1.hexdigest() id = toHash + "." + mime if mime not in ALLOWED_EXTENSIONS.keys(): raise IOError() except IOError: return 0 c = dict(content=binary.Binary(content.getvalue()), mime=mime, time=datetime.datetime.utcnow(), md5=toHash, filename=filename, size=size) try: file = File(**c) file.save() except pymongo.errors.DuplicateKeyError: return jsonify({ "code": 400, "msg": "数据保存失败", }) return id
def random_binary(length): b64 = base64.b64encode(random_string(length)) return binary.Binary(b64, global_prng.choice([0, 1]))
def decode(datas, hash): from bson import binary tt = [] dict = {} data = datas client = MongoClient() db = client.local collection = db.vizdata bin = db.binaries binData = binary.Binary(base64.b64encode(data)) print bin.insert({'s1' : hash, 'data' : binData}) print "Inserted binary" for dat in collection.find(): print "Finding.." if dat['s1'] == hash: print "FILE ALREADY EXISTS" return 0 print "DECODING TWOTUPLE DATA... " #TWO TUPLE i = 0 for c in data: tt.append(ord(c)) threetup = tt for i in range(0, len(tt)-1): dict.update({(tt[i], tt[i+1]):1}) for i in range(0,len(tt)-1): dict.update({(tt[i], tt[i+1]):dict[(tt[i], tt[i+1])]+1}) tt = [] for x,y in dict: tt.append((x,y,dict[x,y])) dict = {} print "Decoding histogram Data..." #HISTOGRAM hist = [] import distorm3 from distorm3 import Decode, Decode16Bits, Decode32Bits, Decode64Bits l = Decode(0xA30, data, Decode32Bits) for i in l: hist.append(i[2].split(' ')[0]) for i in hist: dict.update({i:0}) for i in hist: dict.update({i:(dict[i]+1)}) import math hist = sorted(dict.items(), key=lambda val:val[0]) hist = hist[::] print "Decoding Three Tuple Data...." #THREE TUPLE slide = 3 points = 15000000000 x=y=z=0 dict = {} for i in xrange(0, len(data)-2, slide): dict.update({(threetup[i], threetup[i+1], threetup[i+2]):0}) for i in range(0,len(data)-2,slide): dict.update({(threetup[i], threetup[i+1], threetup[i+2]):dict[(threetup[i], threetup[i+1],threetup[i+2])]+1}) num = int(len(dict)/points)+1 import numpy arr = dict.keys() threetup = [] if(num <= 1): for x,y,z in dict: threetup.append((x,y,z,dict[(x,y,z)])) else: for i in xrange(0,len(dict),num): ctr =[0,0,0,0] for j in range(0, num): #SMALLER BLOCK OF DATA TO BE AVERAGED if i+j >= len(dict)-1 : break for k in range(0,3): ctr[k] = ctr[k]+arr[i+j][k] ctr[3] = ctr[3]+dict[arr[i+j]] for k in range(0,4): ctr[k] = ctr[k]/num threetup.append((ctr[0],ctr[1], ctr[2],ctr[3])) print "Decoding ContactMap..." #CONTACT MAP DECODE list = [] points = 225 #NO. OF POINTS ON CONTACT MAP check = len(data) size = check/points for c in data: b = ord(c) b = float(b) if b < 0 : b = b+256 list.append(b) strings = [] for i in range(0, (len(list)/size)+1): strings.append(list[size*i : (size*i)+size]) if len(strings[len(strings)-1]) < size : for i in range(0, size-len(strings[len(strings)-1])): strings[len(strings)-1].append(0) import numpy import scipy import scipy.spatial import math val = [0,0,0] length = len(strings) cont = [] print length i = j = 0 for i in xrange(0,length): t1 = strings[i] for j in range(0,length): t2 = strings[j] val[0] = round(100*scipy.spatial.distance.braycurtis(t1, t2), 2) #val[0] = (100*scipy.spatial.distance.jaccard(strings[i], strings[j])) #val[0] = (scipy.spatial.distance.sqeuclidean(strings[i], strings[j])) #for k in range(0,2): if math.isnan(val[0]) or (val[0] <= 10): val[0] = 0 else : cont.append((i, j,val[0]))#, val[1], val[2]]) if i%100 == 0 : print i print len(cont) print cont[0:10] import json,zlib print "COmpressing data..." #DATA COMPRESSION hist = json.dumps(hist) hist = zlib.compress(hist) tt = json.dumps(tt) tt = zlib.compress(tt) threetup = json.dumps(threetup) threetup = zlib.compress(threetup) cont = json.dumps(cont) cont = zlib.compress(cont) hist = base64.b64encode(hist) tt = base64.b64encode(tt) threetup = base64.b64encode(threetup) cont = base64.b64encode(cont) dict = {"s1" : hash, "hist" : hist, "t2" : tt ,"t3" : threetup, "cm":cont} tt = threetup = hist = cont = [] print "Inserting..." collection.insert(dict) print "Database Created" return 2
import pymongocrypt from bson import binary from bson.binary import (Binary, JAVA_LEGACY, STANDARD, UUID_SUBTYPE) from bson.codec_options import CodecOptions OPTS = CodecOptions(uuid_representation=STANDARD) print(pymongocrypt.libmongocrypt_version()) client = MongoClient("localhost", 27017) # ** A LOCAL TEST KEY SHOULD NEVER BE USED IN PRODUCTION, ONLY FOR DEVELOPMENT ** # Test key material generated on Mac & Linux with: echo $(head -c 96 /dev/urandom | base64 | tr -d '\n') local_key = 'CgOcoan3c/wm2c+WsOO6fXOUlJgd7SLQ1vl///aEFX6vXN9+7VOAP+iHKheZiYlB09ZS7CDcAQhlPeTeQNz03xiGbiCJJvl3uj4lnG+5i/udSLJAcwgtgtaedkFD0ROq' key_bin = binary.Binary(base64.b64decode(local_key)) kms_providers = {'local': {'key': key_bin}} print( "Please ensure mongocryptd is running (included in the standard enterprise download package)." ) fle_opts = AutoEncryptionOpts(kms_providers, "demoFLE.__keystore", mongocryptd_bypass_spawn=True) client = MongoClient("mongodb://localhost:27017/demoFLE", auto_encryption_opts=fle_opts) db = client.demoFLE
import base64 import os from pymongo import MongoClient from pymongo.errors import DuplicateKeyError from pymongo.encryption import ClientEncryption from pymongo.encryption_options import AutoEncryptionOpts from bson import binary, ObjectId from bson.codec_options import CodecOptions # Setup. codec_opts = CodecOptions(uuid_representation=binary.STANDARD) if "LOCAL_MASTERKEY_BASE64" not in os.environ or not os.path.exists( "key_uuid.txt"): raise Exception("Prerequisites not met. Run setup.py") local_key = os.environ["LOCAL_MASTERKEY_BASE64"] masterkey = binary.Binary(base64.b64decode(local_key)) key_uuid = binary.Binary(base64.b64decode(open("key_uuid.txt", "r").read()), binary.UUID_SUBTYPE)
# Create a data key identified in the key vault collection lab7.key_vault. from base64 import b64encode, b64decode import os from pymongo import MongoClient from pymongo.encryption import ClientEncryption from bson import binary from bson.codec_options import CodecOptions codec_opts = CodecOptions(uuid_representation=binary.STANDARD) # Test key material generated by: echo $(head -c 96 /dev/urandom | base64 | tr -d '\n') if "LOCAL_MASTER_KEY" not in os.environ: raise Exception("Set LOCAL_MASTER_KEY env variable to 96 bytes of base64") master_key = binary.Binary(b64decode(os.environ["LOCAL_MASTER_KEY"])) # Reset the collection key_vault_client = MongoClient("mongodb://localhost/") key_vault_client.lab7.key_vault.drop() # Configure a ClientEncryption object to create data keys kms_providers = {"local": {"key": master_key}} key_vault_client = MongoClient() client_encryption = ClientEncryption( kms_providers, "lab7.key_vault", key_vault_client, codec_opts) key_uuid = client_encryption.create_data_key("local") # Store the key id into a file for easy access open("key_uuid.txt", "w").write(b64encode(key_uuid).decode("utf-8"))
def update_instance(): ana = p() if request.method == 'POST': oldfile_id = request.form['fileId'] domain_model_name = request.form['domainModelName'] file = request.files['file'] filename_str = "" output_dir = os.path.join(config.get('Output', 'output_path')) + "/" + session['username'] if file and allowed_file(file.filename): filename = secure_filename(file.filename) filename_str = filename.split(".")[0] all_content = file.read() b64content = base64.standard_b64encode(all_content) # get bson object bincontent = binary.Binary(b64content) #print all_content # save file into database #newfile_id = dbOps.saveFileToDB(mongo, current_user.username, filename_str, all_content) #print newfile_id # save file to path output_dir = output_dir + "/" + filename_str + "/" + str(oldfile_id) with fileOps.safe_open_w(output_dir + "/" + filename) as f: f.write(all_content) f.close() username = session['username'] dbOps.updateInstanceDb(mgInstance.mongo, username, domain_model_name, oldfile_id, bincontent) file_dir = os.path.join(config.get('Output', 'output_path')) + "/" + username + "/" + domain_model_name + "/" + str(oldfile_id) # Parse XML and generate JSON ana.DM_File_Analyze(output_dir, {'DM_Input_type': "Simple_XML"}, filename_str) # Parse JSON and generate code model_display_data, server_url = generate_code.generate_all(filename_str, output_dir) authen_key = dbOps.getAuthenKey(mgInstance.mongo, session['username']) # Pass required data to the template description_data = { "model_display_data": model_display_data, "server_url": server_url, "authen_key" : authen_key } # write description_data into json file generate_code.write_description_to_file(filename_str, output_dir, description_data) return redirect(url_for('main_bp.index')) #return redirect(url_for('index')) return redirect(url_for('main_bp.update_instance'))
def xueyansuo_tupian(self, url): from bson import binary data = requests.get(url, timeout=15).content return binary.Binary(data)