def test_GridFsIndexesCreation(self): """ Tests gridfs indexes creation""" conn = yield txmongo.MongoConnection(mongo_host, mongo_port) db = conn.test yield self._drop_gridfs(db) # Create a new GridFS instance should trigger indexes creation gfs = GridFS(db) # Multiple calls should return multiple defer not to mix between them self.assertNotEqual(gfs.indexes_created(), gfs.indexes_created()) yield gfs.indexes_created() indexes = yield db.fs.files.index_information() self.assertTrue( any(key["key"] == SON([("filename", 1), ("uploadDate", 1)]) for key in indexes.values())) indexes = yield db.fs.chunks.index_information() self.assertTrue( any(key["key"] == SON([("files_id", 1), ("n", 1)]) for key in indexes.values())) yield conn.disconnect()
def test_document_from_ip_address(): streamer = Streamer('test_document_from_ip_address') host = 'http://list-iptv.com' url1 = 'http://62.210.245.19:8000/live/testapp/testapp/2.ts' url2 = 'http://clientportalpro.com:2500/live/VE5DWv4Ait/7KHLqRRZ9E/2160.ts' url3 = 'http://ndasat.pro:8000/live/exch/exch/1227.ts' for url in [url1, url2, url3]: netloc = um.prepare_netloc(url) ip_addresses = socket.gethostbyname_ex(um.remove_schema(netloc))[2] for ip_address in ip_addresses: data = { 'ip_address': ip_address, 'network_locations': [ SON([('network_location', netloc), ('linked_by', [host]), ('working_link', True)]) ] } streamer.collection().insert(data) doc = streamer.document_from_ip_address(ip_address) assert doc['ip_address'] == ip_address assert doc['network_locations'] == [ SON([('network_location', netloc), ('linked_by', [host]), ('working_link', True)]) ] streamer.delete()
def delete(self, user='******'): history_collection = DB.handle[self.record_type + '_history'] record_history = history_collection.find_one({'_id': self.id}) or SON() record_history['deleted'] = SON({'user': user, 'time': datetime.utcnow()}) history_collection.replace_one({'_id': self.id}, record_history, upsert=True) return type(self).handle().delete_one({'_id': self.id})
def test_delta_for_dynamic_documents(self): class Person(DynamicDocument): name = StringField() meta = {'allow_inheritance': True} Person.drop_collection() p = Person(name="James", age=34) self.assertEqual(p._delta(), ( SON([('_cls', 'Person'), ('name', 'James'), ('age', 34)]), {})) p.doc = 123 del(p.doc) self.assertEqual(p._delta(), ( SON([('_cls', 'Person'), ('name', 'James'), ('age', 34)]), {})) p = Person() p.name = "Dean" p.age = 22 p.save() p.age = 24 self.assertEqual(p.age, 24) self.assertEqual(p._get_changed_fields(), ['age']) self.assertEqual(p._delta(), ({'age': 24}, {})) p = Person.objects(age=22).get() p.age = 24 self.assertEqual(p.age, 24) self.assertEqual(p._get_changed_fields(), ['age']) self.assertEqual(p._delta(), ({'age': 24}, {})) p.save() self.assertEqual(1, Person.objects(age=24).count())
def get_punctuation(db, collection, city): df = pd.DataFrame([[d['name'], d['number_of_employees'], d['loc']] for d in db[collection].find({'city': city})], columns=['name', 'employees', 'loc']) distances = {'airports': 30000, 'restaurants': 1000, 'starbucks': 500} data = { 'airports': src.yelp.get_airports(city), 'restaurants': src.yelp.get_vegan_restaurants(city), 'starbucks': src.yelp.get_starbucks(city) } for kind, listing in data.items(): df[kind] = 0 for element in listing: results = db[collection].find({ 'loc': { '$near': SON([('$geometry', SON([('type', 'Point'), ('coordinates', [ element['coordinates']['longitude'], element['coordinates']['latitude'] ])])), ('$maxDistance', distances[kind])]) } }) for r in results: df.at[df.loc[df['name'] == r['name']].index, kind] = 1 df = df.astype({ 'airports': 'int32', 'starbucks': 'int32', 'restaurants': 'int32' }) df['total'] = df.iloc[:, -3:-1].sum(axis=1) return df
def authenticate_scram_sha1(self, database_name, username, password): # Totally stolen from pymongo.auth user = username.replace('=', "=3D").replace(',', "=2C") nonce = base64.standard_b64encode( str(SystemRandom().random()).encode('ascii'))[2:] first_bare = "n={0},r={1}".format(user, nonce.decode()).encode('ascii') cmd = SON([("saslStart", 1), ("mechanism", "SCRAM-SHA-1"), ("autoAuthorize", 1), ("payload", Binary(b"n,," + first_bare))]) result = yield self.__run_command(database_name, cmd) server_first = result["payload"] parsed = auth._parse_scram_response(server_first) iterations = int(parsed[b'i']) salt = parsed[b's'] rnonce = parsed[b'r'] if not rnonce.startswith(nonce): raise MongoAuthenticationError( "TxMongo: server returned an invalid nonce.") without_proof = b"c=biws,r=" + rnonce salted_pass = auth._hi( auth._password_digest(username, password).encode("utf-8"), base64.standard_b64decode(salt), iterations) client_key = hmac.HMAC(salted_pass, b"Client Key", sha1).digest() stored_key = sha1(client_key).digest() auth_msg = b','.join((first_bare, server_first, without_proof)) client_sig = hmac.HMAC(stored_key, auth_msg, sha1).digest() client_proof = b"p=" + base64.standard_b64encode( auth._xor(client_key, client_sig)) client_final = b','.join((without_proof, client_proof)) server_key = hmac.HMAC(salted_pass, b"Server Key", sha1).digest() server_sig = base64.standard_b64encode( hmac.HMAC(server_key, auth_msg, sha1).digest()) cmd = SON([("saslContinue", 1), ("conversationId", result["conversationId"]), ("payload", Binary(client_final))]) result = yield self.__run_command(database_name, cmd) if not result["ok"]: raise MongoAuthenticationError("TxMongo: authentication failed.") parsed = auth._parse_scram_response(result["payload"]) if parsed[b'v'] != server_sig: raise MongoAuthenticationError( "TxMongo: server returned an invalid signature.") # Depending on how it's configured, Cyrus SASL (which the server uses) # requires a third empty challenge. if not result["done"]: cmd = SON([("saslContinue", 1), ("conversationId", result["conversationId"]), ("payload", Binary(b''))]) result = yield self.__run_command(database_name, cmd) if not result["done"]: raise MongoAuthenticationError( "TxMongo: SASL conversation failed to complete.")
def pharmacy_search(): if check_root != 2: return render_template('error.html', id="로컬이 아닌 잘못된 접근입니다.") client = MyMongoClient("kim_db", "pharmacy") client2 = MyMongoClient("kim_db", "modify_pharmacy") modify_pharmacy = client2.get_collection() cursor = client.get_collection().find() for cur in cursor: if cur.get('경도') == None: continue phar_loc = {'type': 'Point'} phar_loc['coordinates'] = [float(cur.get('경도')), float(cur.get('위도'))] bookJson = {'pharmacyName': cur.get('약국명'), 'location': phar_loc} modify_pharmacy.insert_one(bookJson) modify_pharmacy.create_index([('location', GEOSPHERE)]) query = { 'location': { '$near': SON([('$geometry', SON([('type', 'Point'), ('coordinates', [128.62707, 35.88107])])), ('$maxDistance', 1000)]) } } pharmacy = modify_pharmacy.find(query) return render_template('hkit_pharmacy.html', pharmacy=pharmacy)
def make_pipeline(criterions): if not criterions: return [ { '$match': {'skip': {'$exists': False}} }, { '$limit': 10 } ] return [ { '$match': {'skip': {'$exists': False}} }, { '$project': SON( [(name, chose_project_method(name, value)) for name, value in criterions] + [('id', 1), ('domain', 1)] ) }, { '$sort': SON( [(name, -1) for name, value in criterions] ) }, { '$limit': 10 } ]
async def generateRandomPlay(db, state): now = str(datetime.datetime.now()) if len(state["cards_left"]) < 2: # Start a new round! if state["round"] == state["board"]["numRounds"] - 1: # end of game state["in_progress"] = False state["time_ended"] = now state["last_updated"] = now await db.games.update_one({"_id": state["_id"]}, SON([ ("$set", SON([("last_updated", state["last_updated"]), \ ("in_progress", state["in_progress"]), \ ("time_ended", state["time_ended"]), ]))])) return False state["round"] = state["round"] + 1 state["cards_left"] = state["cards_list"].copy() for player in state["players"]: prev_score = player["score"]["target_rounds"][state["round"] - 1] player["score"]["target_rounds"].append(prev_score) newPlay = random.sample(state["cards_left"], 2) state["cards_left"].remove(newPlay[0]) state["cards_left"].remove(newPlay[1]) state["plays"].append(newPlay) state["last_updated"] = now await db.games.update_one({"_id": state["_id"]}, SON([("$push", SON([("plays", newPlay)])), \ ("$set", SON([("last_updated", state["last_updated"]), \ ("cards_left", state["cards_left"]), \ ("round", state["round"]), \ ("players", state["players"])]))])) return True
def test_forward__if_name_is_set_and_changed_and_field_spec_also_changed__should_create_index( self, test_db, left_schema): fields1 = [('field2', pymongo.ASCENDING)] fields2 = [('field1', pymongo.ASCENDING), ('field2', pymongo.DESCENDING)] test_db['document1'].create_index(fields1, name='index_old', sparse=False) action = AlterIndex('Document1', 'index2', fields=fields2, name='index2', sparse=True) action.prepare(test_db, left_schema, MigrationPolicy.strict) action.run_forward() indexes1 = [ x for x in test_db['document1'].list_indexes() if x['key'] == SON(fields1) ] assert len(indexes1) == 1 assert indexes1[0]['sparse'] is False assert indexes1[0]['name'] == 'index_old' indexes2 = [ x for x in test_db['document1'].list_indexes() if x['key'] == SON(fields2) ] assert len(indexes2) == 1 assert indexes2[0]['name'] == 'index2' assert indexes2[0]['sparse'] is True
def _encode_binary(data, subtype, json_options): if json_options.json_mode == JSONMode.LEGACY: return SON([ ('$binary', base64.b64encode(data).decode()), ('$type', "%02x" % subtype)]) return {'$binary': SON([ ('base64', base64.b64encode(data).decode()), ('subType', "%02x" % subtype)])}
def nearest_neighbors_query(cls, coordinate_field, lat, lon): return { coordinate_field: SON([("$near", { "$geometry": SON([("type", "Point"), ("coordinates", [lon, lat])]) })]) }
def default(obj): # We preserve key order when rendering SON, DBRef, etc. as JSON by # returning a SON for those types instead of a dict. This works with # the "json" standard library in Python 2.6+ and with simplejson # 2.1.0+ in Python 2.5+, because those libraries iterate the SON # using PyIter_Next. Python 2.4 must use simplejson 2.0.9 or older, # and those versions of simplejson use the lower-level PyDict_Next, # which bypasses SON's order-preserving iteration, so we lose key # order in Python 2.4. if isinstance(obj, ObjectId): return {"$oid": str(obj)} if isinstance(obj, DBRef): return _json_convert(obj.as_doc()) if isinstance(obj, datetime.datetime): # TODO share this code w/ bson.py? if obj.utcoffset() is not None: obj = obj - obj.utcoffset() millis = int(calendar.timegm(obj.timetuple()) * 1000 + obj.microsecond / 1000) return {"$date": millis} if isinstance(obj, (RE_TYPE, Regex)): flags = "" if obj.flags & re.IGNORECASE: flags += "i" if obj.flags & re.LOCALE: flags += "l" if obj.flags & re.MULTILINE: flags += "m" if obj.flags & re.DOTALL: flags += "s" if obj.flags & re.UNICODE: flags += "u" if obj.flags & re.VERBOSE: flags += "x" if isinstance(obj.pattern, str): pattern = obj.pattern else: pattern = obj.pattern.decode('utf-8') return SON([("$regex", pattern), ("$options", flags)]) if isinstance(obj, MinKey): return {"$minKey": 1} if isinstance(obj, MaxKey): return {"$maxKey": 1} if isinstance(obj, Timestamp): return SON([("t", obj.time), ("i", obj.inc)]) if isinstance(obj, Code): return SON([('$code', str(obj)), ('$scope', obj.scope)]) if isinstance(obj, Binary): return SON([ ('$binary', base64.b64encode(obj).decode()), ('$type', "%02x" % obj.subtype)]) if PY3 and isinstance(obj, binary_type): return SON([ ('$binary', base64.b64encode(obj).decode()), ('$type', "00")]) if bson.has_uuid() and isinstance(obj, bson.uuid.UUID): return {"$uuid": obj.hex} raise TypeError("%r is not JSON serializable" % obj)
def MRsimple(collection, FunMap, FunReduce=None, query={}, out={"replace": 'mr_tmp'}, finalize=None, scope={}, sort=None, jsMode=False, verbose=1): """ simplified generic Map Reduce see: http://docs.mongodb.org/manual/reference/method/db.collection.mapReduce/ returns (MR response object, results collection or results list if out={"inline":1}) Reduce function defaults to one that increments value count optimize by sorting on emit fields see: http://edgystuff.tumblr.com/post/7624019777/optimizing-map-reduce-with-mongodb docs.mongodb.org/manual/reference/method/db.collection.mapReduce/#db.collection.mapReduce sort i.e: sort= { "_id":1 } jsMode should be False if we expect more than 500K dictinct ids """ if len(out.viewkeys()) > 1: command = MRCommand_(out) out = SON([(command, out[command]), ('db', out.get('db')), ('nonAtomic', out.get('nonAtomic', False))]) #nonAtomic not allowed on replace FunMap = Code(FunMap, {}) if FunReduce is None: FunReduce = u"""function (key, values) {var total = 0; for (var i = 0; i < values.length; i++) { total += values[i]; } return total;} """ FunReduce = Code(FunReduce, {}) if verbose > 2: print "Start MRsimple collection = %s"\ "query = %s\nMap=\n%s\nReduce=\n%s\nFinalize=%s\nscope=%s sort=%s" \ % tuple(map(str, (out, query, FunMap, FunReduce, finalize, scope, sort))) if sort: sort = SON(sort) r = collection.map_reduce(FunMap, FunReduce, out=out, query=query, finalize=finalize, scope=scope, sort=sort, full_response=True) if verbose > 1: print "End MRsimple collection=%s, query=%s\nresulsts=\n %s"\ % (collection.name, str(query), str(r)) if 'db' in out.viewkeys(): #@note: can be dict or SON, either way it has property viewkeys results = collection.database.connection[r['result']['db']][ r['result']['collection']] else: results = r['results'] if out.keys( )[0] == 'inline' else collection.database[r['result']] #@note: results is a list if inline else a collection return r, results
def test_sort(self): self.assertEqual(pql.sort('a'), [{ '$sort': SON([('a', pymongo.ASCENDING)]) }]) self.assertEqual(pql.sort(['a', '-b', '+c']), [{ '$sort': SON([('a', pymongo.ASCENDING), ('b', pymongo.DESCENDING), ('c', pymongo.ASCENDING)]) }])
async def startGame(db, state): state["in_progress"] = True now = str(datetime.datetime.now()) state["last_updated"] = now await db.games.update_one({"_id": state["_id"]}, SON([("$set", SON([("in_progress", True), ("last_updated", state["last_updated"])]))])) await generateRandomPlay(db, state)
def map_response_data_to_census(): response_census = mongo_db.response_census x = response_census.find({ "loc": SON([({ '$geoWithin': { '$geometry': SON([("type", "Polygon"), ("coordinates", 'loc')]) } })]) }) return x
def _make_bson_doc(uid: str, df: pd.DataFrame, metadata) -> SON: """ Takes a DataFrame and makes a BSON document ready to be inserted into MongoDB. Given Conritick's focus on timeseries data, the input DataFrame index must be a DatetimeIndex. Column names are kept and saved as strings. Index name is explicitly discarded and not saved. :param uid: Unique ID for the timeseries represented by the input DataFrame :param df: Input DataFrame :param metadata: Any BSON-able objects to be attached to document as metadata :return: BSON document """ mem_usage = df.memory_usage().sum() df = df.sort_index(ascending=True) if df.index.tzinfo is None: if not all(ix.time() == datetime.time(0, 0) for ix in df.index[:100]): # Issue warning only if DataFrame doesn't look like EOD based. warnings.warn('DatetimeIndex is timezone-naive. Assuming to be in UTC.') offset = None else: offset = df.index.tzinfo._utcoffset.total_seconds() / 60 # Remove invalid MongoDB field characters df = df.rename(columns=lambda x: re.sub('\.', '', str(x))) index = _make_bson_column(df.index) columns = SON() for col in df.columns: columns[col] = _make_bson_column(df[col]) nrows = len(df) binary_size = sum([columns[col]['size'] for col in df.columns]) binary_size += index['size'] compression_ratio = binary_size / mem_usage if binary_size > 0.95 * MAX_BSON_SIZE: msg = f'Binary data size is too large ({binary_size:,} / {compression_ratio:.1%})' raise InvalidBSON(msg, compression_ratio) logger.debug(f'{uid} document: {binary_size:,} bytes ({compression_ratio:.1%}), {nrows} rows') add_meta = {'nrows': nrows, 'binary_size': binary_size, 'utc_offset': offset} metadata = {**metadata, **add_meta} doc = SON([ ('uid', uid), ('start', df.index[0]), ('end', df.index[-1]), ('metadata', metadata), ('index', index), ('columns', columns)]) return doc
async def msg_submit_form(request): form = await request.post() #get the form text = form['msg_input'] # get the form data output_result.append(text) # store the previous data in an array db = request.app['db'] # request the db try: #check if the input from form matches any of the data in db msg_db = await db.command( SON([("distinct", "collection"), ("key", "msg"), ("query", { "msg": text })])) try: msg_db = msg_db['values'][0] except Exception as e: print(e, 1) result = "can you say it again" return {"result": result, "output_result": output_result} if msg_db == text: try: # if it matches, retrieve the required data from the collection option1 = await db.command( SON([("distinct", "collection"), ("key", "option1"), ("query", { "msg": text })])) option2 = await db.command( SON([("distinct", "collection"), ("key", "option2"), ("query", { "msg": text })])) option1 = option1['values'][0] option2 = option2['values'][0] return { "form": [option1, option2], "output_result": output_result } except: result = "SON failure" return {"result": result, "output_result": output_result} except Exception as e: print(e, 2) result = "can you say it again" return {"result": result, "output_result": output_result}
async def addNewPlayerToGame(db, state, player_id, playerName): newPlayer = { "id": player_id, "name": playerName, "score": { "targets_current_round": 0, "target_rounds": [0], "targets_previous_rounds": [], "shops_joined": [], "bonuses": [], "saladcop_bonus": 0 }, "moves": [], "numBonusMovesPlayed": 0, "cells_connected_to_shops": {}, "targets_connected_to_shops": {}, "bonusLines": 0, "connected_targets": {}, "connected_target_types": {}, "connected_shops": {}, "shop_pairs": {}, "active_cells": {} } # Add permutation perms = db.permutations.aggregate([{"$sample": {"size": 1}}]) permlist = await perms.to_list(1) myPerm = permlist[0]["perm"] newPlayer["perm"] = myPerm # Add pairs of shops for letter in list(set(myPerm)): i = myPerm.index(letter) j = myPerm.index(letter, i + 1) shopi = "shop" + str(i) shopj = "shop" + str(j) newPlayer["shop_pairs"][shopi] = shopj newPlayer["shop_pairs"][shopj] = shopi state["players"].append(newPlayer) now = str(datetime.datetime.now()) state["last_updated"] = now await db.games.update_one({"_id": state["_id"]}, SON([("$push", SON([("players", newPlayer)])), ("$set", SON([ ("last_updated", state["last_updated"]), ]))]))
def default(obj): # We preserve key order when rendering SON, DBRef, etc. as JSON by # returning a SON for those types instead of a dict. if isinstance(obj, ObjectId): return {"$oid": str(obj)} if isinstance(obj, DBRef): return _json_convert(obj.as_doc()) if isinstance(obj, datetime.datetime): # TODO share this code w/ bson.py? if obj.utcoffset() is not None: obj = obj - obj.utcoffset() millis = int( calendar.timegm(obj.timetuple()) * 1000 + obj.microsecond / 1000) return {"$date": millis} if isinstance(obj, (RE_TYPE, Regex)): flags = "" if obj.flags & re.IGNORECASE: flags += "i" if obj.flags & re.LOCALE: flags += "l" if obj.flags & re.MULTILINE: flags += "m" if obj.flags & re.DOTALL: flags += "s" if obj.flags & re.UNICODE: flags += "u" if obj.flags & re.VERBOSE: flags += "x" if isinstance(obj.pattern, text_type): pattern = obj.pattern else: pattern = obj.pattern.decode('utf-8') return SON([("$regex", pattern), ("$options", flags)]) if isinstance(obj, MinKey): return {"$minKey": 1} if isinstance(obj, MaxKey): return {"$maxKey": 1} if isinstance(obj, Timestamp): return {"$timestamp": SON([("t", obj.time), ("i", obj.inc)])} if isinstance(obj, Code): return SON([('$code', str(obj)), ('$scope', obj.scope)]) if isinstance(obj, Binary): return SON([('$binary', base64.b64encode(obj).decode()), ('$type', "%02x" % obj.subtype)]) if PY3 and isinstance(obj, bytes): return SON([('$binary', base64.b64encode(obj).decode()), ('$type', "00")]) if isinstance(obj, uuid.UUID): return {"$uuid": obj.hex} raise TypeError("%r is not JSON serializable" % obj)
def _start_authentication(self, response, error=None): # this is the nonce response if error: logging.error(error) logging.error(response) raise AuthenticationError(error) nonce = response['data'][0]['nonce'] key = helpers._auth_key(nonce, self.__dbuser, self.__dbpass) self.__callback = self._finish_authentication self._send_message( message.query( 0, "%s.$cmd" % self.__pool._dbname, 0, 1, SON([('authenticate', 1), ('user', self.__dbuser), ('nonce', nonce), ('key', key)]), SON({})))
def test_to_mongo(self): doc = EmbeddedDocument(name='Bob') value = self.field.to_mongo(doc) self.assertIsInstance(value, SON) self.assertEqual(value, SON({'name': 'Bob'})) son = value value = self.field.to_mongo(son) self.assertIsInstance(value, SON) self.assertEqual(value, SON({'name': 'Bob'})) value = self.field.to_mongo({'name': 'Bob'}) self.assertIsInstance(value, SON) self.assertEqual(value, SON({'name': 'Bob'}))
def from_json(cls, json_str, created=False, *args, **kwargs): """ Decode from human-readable json. Parameters: json_str: JSON string that should be passed to the serialized created: a parameter that is passed to cls._from_son. *args, **kwargs: Any additional arguments that is passed to json.loads. """ from .fields import FollowReferenceField kwargs.setdefault("object_hook", generate_object_hook(cls)) dct = json.loads(json_str, *args, **kwargs) for name, fld in cls._fields.items(): if any([ getattr(fld, "exclude_from_json", None), getattr(fld, "exclude_json", None) ]): dct.pop(name, None) from_son_result = cls._from_son(SON(dct), created=created) for fldname, fld in cls._fields.items(): target = fld.field if isinstance(fld, db.ListField) else fld if not isinstance(target, db.ReferenceField) or \ isinstance(target, FollowReferenceField): continue value = dct.get(fldname) setattr(from_son_result, fldname, normalize_reference(getattr(value, "id", value), target)) return from_son_result
def get_result2(cls, collection, fields_dict, match, limit, skip): sum_field = cls.create_sum_field(fields_dict) groups2 = cls.create_groups2(fields_dict, sum_field) total_count = collection.aggregate([ { "$match": match }, { "$group": groups2 }, { "$sort": SON([("_id", 1)]) }, ]).get('result').__len__() if not skip else 0 res = collection.aggregate([{ "$match": match }, { "$group": groups2 }, { "$sort": { 'day': 1 } }, { "$skip": skip }, { "$limit": limit }]) result = res.get('result') for r in result: fields = r.get('_id') r.pop('_id') r.update(fields) return total_count, result
def test_get_more_iteration(self): # Insert 5 documents with WC majority. self.coll.insert_many([{'data': k} for k in range(5)]) # Start a find operation and retrieve first batch of results. batch_size = 2 cursor = self.coll.find(batch_size=batch_size) for _ in range(batch_size): cursor.next() # Force step-down the primary. res = self.client.admin.command( SON([("replSetStepDown", 5), ("force", True)])) self.assertEqual(res["ok"], 1.0) # Get next batch of results. for _ in range(batch_size): cursor.next() # Verify pool not cleared. self.verify_pool_not_cleared() # Attempt insertion to mark server description as stale and prevent a # notMaster error on the subsequent operation. try: self.coll.insert_one({}) except NotMasterError: pass # Next insert should succeed on the new primary without clearing pool. self.coll.insert_one({}) self.verify_pool_not_cleared()
def _index_document(index_list): """Helper to generate an index specifying document. Takes a list of (key, direction) pairs. """ if isinstance(index_list, abc.Mapping): raise TypeError("passing a dict to sort/create_index/hint is not " "allowed - use a list of tuples instead. did you " "mean %r?" % list(iteritems(index_list))) elif not isinstance(index_list, (list, tuple)): raise TypeError("must use a list of (key, direction) pairs, " "not: " + repr(index_list)) if not len(index_list): raise ValueError("key_or_list must not be the empty list") index = SON() for (key, value) in index_list: if not isinstance(key, string_type): raise TypeError("first item in each key pair must be a string") if not isinstance(value, (string_type, int, abc.Mapping)): raise TypeError("second item in each key pair must be 1, -1, " "'2d', 'geoHaystack', or another valid MongoDB " "index specifier.") index[key] = value return index
def update(self, document_id, update_spec, namespace, timestamp): """Apply updates given in update_spec to the document whose id matches that of doc. """ db, coll = self._db_and_collection(namespace) meta_collection_name = self._get_meta_collection(namespace) self.meta_database[meta_collection_name].replace_one( { self.id_field: document_id, "ns": namespace }, { self.id_field: document_id, "_ts": timestamp, "ns": namespace }, upsert=True) no_obj_error = "No matching object found" updated = self.mongo[db].command(SON([('findAndModify', coll), ('query', { '_id': document_id }), ('update', update_spec), ('new', True)]), allowable_errors=[no_obj_error ])['value'] return updated
def messages_per_hours(db): """ Number of messages per hour of the day :param db: database connection :return: """ pipeline = [{ "$project": { "hour": { "$hour": "$date" } } }, { "$unwind": "$hour" }, { "$group": { "_id": "$hour", "count": { "$sum": 1 } } }, { "$sort": SON([("_id", 1)]) }] data = list(db.messages.aggregate(pipeline)) return ChartData(map(operator.itemgetter('_id'), data), map(operator.itemgetter('count'), data))
def most_mentions(db): print( "3. Who is are the most mentioned Twitter users? (Provide the top five.)" ) pipeline = [{ "$match": { "text": { "$regex": "@\\w*" } } }, { "$group": { "_id": { "$substrCP": [ "$text", { "$indexOfCP": ["$text", "@"] }, { "$indexOfCP": ["$text", " "] } ] }, "count": { "$sum": 1 } } }, { "$sort": SON([("count", -1), ("user", -1)]) }, { "$limit": 5 }] result = db.tweets.aggregate(pipeline) pp_all(result) """