def test_slave_okay_metadata_commands(self):
secondaries = iter(self._get_client().secondaries)
host, port = secondaries.next()
# Direct connection to a secondary.
client = MongoClient(host, port)
self.assertFalse(client.is_primary)
self.assertEqual(client.read_preference, ReadPreference.PRIMARY)
# No error.
client.database_names()
client.pymongo_test.collection_names()
client.pymongo_test.test.options()
client.pymongo_test.test.index_information()
def test_ipv6(self):
c = MongoClient("mongodb://[::1]:%d" % (port, ), replicaSet=self.name)
# Client switches to IPv4 once it has first ismaster response.
msg = 'discovered primary with IPv4 address "%r"' % (self.primary, )
wait_until(lambda: c.primary == self.primary, msg)
# Same outcome with both IPv4 and IPv6 seeds.
c = MongoClient("[::1]:%d,localhost:%d" % (port, port),
replicaSet=self.name)
wait_until(lambda: c.primary == self.primary, msg)
if client_context.auth_enabled:
auth_str = "%s:%s@" % (db_user, db_pwd)
else:
auth_str = ""
uri = "mongodb://%slocalhost:%d,[::1]:%d" % (auth_str, port, port)
client = MongoClient(uri, replicaSet=self.name)
client.pymongo_test.test.insert_one({"dummy": u("object")})
client.pymongo_test_bernie.test.insert_one({"dummy": u("object")})
dbs = client.database_names()
self.assertTrue("pymongo_test" in dbs)
self.assertTrue("pymongo_test_bernie" in dbs)
client.close()
def test_unix_socket(self):
if not hasattr(socket, "AF_UNIX"):
raise SkipTest("UNIX-sockets are not supported on this system")
client = MongoClient(host, port)
if (sys.platform == 'darwin' and
server_started_with_auth(client) and
not version.at_least(client, (2, 7, 1))):
raise SkipTest("SERVER-8492")
mongodb_socket = '/tmp/mongodb-27017.sock'
if not os.access(mongodb_socket, os.R_OK):
raise SkipTest("Socket file is not accessable")
self.assertTrue(MongoClient("mongodb://%s" % mongodb_socket))
client = MongoClient("mongodb://%s" % mongodb_socket)
client.pymongo_test.test.save({"dummy": "object"})
# Confirm we can read via the socket
dbs = client.database_names()
self.assertTrue("pymongo_test" in dbs)
# Confirm it fails with a missing socket
self.assertRaises(ConnectionFailure, MongoClient,
"mongodb:///tmp/none-existent.sock")
def test_ipv6(self):
c = MongoClient("mongodb://[::1]:%d" % (port,), replicaSet=self.name)
# Client switches to IPv4 once it has first ismaster response.
msg = 'discovered primary with IPv4 address "%r"' % (self.primary,)
wait_until(lambda: c.primary == self.primary, msg)
# Same outcome with both IPv4 and IPv6 seeds.
c = MongoClient("[::1]:%d,localhost:%d" % (port, port),
replicaSet=self.name)
wait_until(lambda: c.primary == self.primary, msg)
if client_context.auth_enabled:
auth_str = "%s:%s@" % (db_user, db_pwd)
else:
auth_str = ""
uri = "mongodb://%slocalhost:%d,[::1]:%d" % (auth_str, port, port)
client = MongoClient(uri, replicaSet=self.name)
client.pymongo_test.test.insert_one({"dummy": u("object")})
client.pymongo_test_bernie.test.insert_one({"dummy": u("object")})
dbs = client.database_names()
self.assertTrue("pymongo_test" in dbs)
self.assertTrue("pymongo_test_bernie" in dbs)
client.close()
def test_unix_socket(self):
if not hasattr(socket, "AF_UNIX"):
raise SkipTest("UNIX-sockets are not supported on this system")
mongodb_socket = '/tmp/mongodb-27017.sock'
encoded_socket = '%2Ftmp%2Fmongodb-27017.sock'
if not os.access(mongodb_socket, os.R_OK):
raise SkipTest("Socket file is not accessible")
if client_context.auth_enabled:
uri = "mongodb://%s:%s@%s" % (db_user, db_pwd, encoded_socket)
else:
uri = "mongodb://%s" % encoded_socket
# Confirm we can do operations via the socket.
client = MongoClient(uri)
client.pymongo_test.test.insert_one({"dummy": "object"})
dbs = client.database_names()
self.assertTrue("pymongo_test" in dbs)
# Confirm it fails with a missing socket.
self.assertRaises(
ConnectionFailure, connected,
MongoClient("mongodb://%2Ftmp%2Fnon-existent.sock",
serverSelectionTimeoutMS=100))
def test_unix_socket(self):
if not hasattr(socket, "AF_UNIX"):
raise SkipTest("UNIX-sockets are not supported on this system")
mongodb_socket = '/tmp/mongodb-27017.sock'
encoded_socket = '%2Ftmp%2Fmongodb-27017.sock'
if not os.access(mongodb_socket, os.R_OK):
raise SkipTest("Socket file is not accessible")
if client_context.auth_enabled:
uri = "mongodb://%s:%s@%s" % (db_user, db_pwd, encoded_socket)
else:
uri = "mongodb://%s" % encoded_socket
# Confirm we can do operations via the socket.
client = MongoClient(uri)
client.pymongo_test.test.insert_one({"dummy": "object"})
dbs = client.database_names()
self.assertTrue("pymongo_test" in dbs)
# Confirm it fails with a missing socket.
self.assertRaises(
ConnectionFailure,
connected, MongoClient("mongodb://%2Ftmp%2Fnon-existent.sock",
serverSelectionTimeoutMS=100))
def test_database_names(self):
client = MongoClient(host, port)
client.pymongo_test.test.save({"dummy": u"object"})
client.pymongo_test_mike.test.save({"dummy": u"object"})
dbs = client.database_names()
self.assertTrue("pymongo_test" in dbs)
self.assertTrue("pymongo_test_mike" in dbs)
def test_database_names(self):
client = MongoClient(host, port)
client.pymongo_test.test.save({"dummy": u"object"})
client.pymongo_test_mike.test.save({"dummy": u"object"})
dbs = client.database_names()
self.assertTrue("pymongo_test" in dbs)
self.assertTrue("pymongo_test_mike" in dbs)
def test_drop_database(self):
client = MongoClient(host, port)
self.assertRaises(TypeError, client.drop_database, 5)
self.assertRaises(TypeError, client.drop_database, None)
raise SkipTest("This test often fails due to SERVER-2329")
client.pymongo_test.test.save({"dummy": u"object"})
dbs = client.database_names()
self.assertTrue("pymongo_test" in dbs)
client.drop_database("pymongo_test")
dbs = client.database_names()
self.assertTrue("pymongo_test" not in dbs)
client.pymongo_test.test.save({"dummy": u"object"})
dbs = client.database_names()
self.assertTrue("pymongo_test" in dbs)
client.drop_database(client.pymongo_test)
dbs = client.database_names()
self.assertTrue("pymongo_test" not in dbs)
def test_drop_database(self):
client = MongoClient(host, port)
self.assertRaises(TypeError, client.drop_database, 5)
self.assertRaises(TypeError, client.drop_database, None)
raise SkipTest("This test often fails due to SERVER-2329")
client.pymongo_test.test.save({"dummy": u"object"})
dbs = client.database_names()
self.assertTrue("pymongo_test" in dbs)
client.drop_database("pymongo_test")
dbs = client.database_names()
self.assertTrue("pymongo_test" not in dbs)
client.pymongo_test.test.save({"dummy": u"object"})
dbs = client.database_names()
self.assertTrue("pymongo_test" in dbs)
client.drop_database(client.pymongo_test)
dbs = client.database_names()
self.assertTrue("pymongo_test" not in dbs)
def test_copy_db(self):
c = MongoClient(host, port)
# Due to SERVER-2329, databases may not disappear
# from a master in a master-slave pair.
if server_is_master_with_slave(c):
raise SkipTest("SERVER-2329")
ctx = catch_warnings()
try:
warnings.simplefilter("ignore", DeprecationWarning)
self.assertRaises(TypeError, c.copy_database, 4, "foo")
self.assertRaises(TypeError, c.copy_database, "foo", 4)
self.assertRaises(InvalidName, c.copy_database, "foo", "$foo")
c.pymongo_test.test.drop()
c.pymongo_test.test.insert({"foo": "bar"})
c.drop_database("pymongo_test1")
self.assertFalse("pymongo_test1" in c.database_names())
c.copy_database("pymongo_test", "pymongo_test1")
self.assertTrue("pymongo_test1" in c.database_names())
self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])
c.drop_database("pymongo_test1")
# XXX - SERVER-15318
if not (version.at_least(c, (2, 6, 4)) and is_mongos(c)):
self.assertFalse(c.in_request())
c.copy_database("pymongo_test", "pymongo_test1",
"%s:%d" % (host, port))
# copy_database() didn't accidentally restart the request
self.assertFalse(c.in_request())
self.assertTrue("pymongo_test1" in c.database_names())
self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])
c.drop_database("pymongo_test1")
finally:
ctx.exit()
def test_copy_db(self):
c = MongoClient(host, port)
# Due to SERVER-2329, databases may not disappear
# from a master in a master-slave pair.
if server_is_master_with_slave(c):
raise SkipTest("SERVER-2329")
ctx = catch_warnings()
try:
warnings.simplefilter("ignore", DeprecationWarning)
self.assertRaises(TypeError, c.copy_database, 4, "foo")
self.assertRaises(TypeError, c.copy_database, "foo", 4)
self.assertRaises(InvalidName, c.copy_database, "foo", "$foo")
c.pymongo_test.test.drop()
c.pymongo_test.test.insert({"foo": "bar"})
c.drop_database("pymongo_test1")
self.assertFalse("pymongo_test1" in c.database_names())
c.copy_database("pymongo_test", "pymongo_test1")
self.assertTrue("pymongo_test1" in c.database_names())
self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])
c.drop_database("pymongo_test1")
# XXX - SERVER-15318
if not (version.at_least(c, (2, 6, 4)) and is_mongos(c)):
self.assertFalse(c.in_request())
c.copy_database("pymongo_test", "pymongo_test1",
"%s:%d" % (host, port))
# copy_database() didn't accidentally restart the request
self.assertFalse(c.in_request())
self.assertTrue("pymongo_test1" in c.database_names())
self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])
c.drop_database("pymongo_test1")
finally:
ctx.exit()
def get_empty_db_da(_database_name, _json_schema_folders=None, _uri_handlers = None):
"""
Create an empty database. Drops any existing.
:param _database_name: The name of the database
:return: A database access object for the database
:param _json_schema_folders: A list of application specific JSON schema folders
"""
_client = MongoClient()
if _database_name in _client.database_names():
_client.drop_database(_client[_database_name])
_database = _client[_database_name]
return DatabaseAccess(_database=_database, _json_schema_folders=_json_schema_folders, _uri_handlers=_uri_handlers)
def test_ipv6(self):
try:
client = MongoClient("[::1]")
except:
# Either mongod was started without --ipv6
# or the OS doesn't support it (or both).
raise SkipTest("No IPv6")
# Try a few simple things
MongoClient("mongodb://[::1]:%d" % (port,))
MongoClient("mongodb://[::1]:%d/?w=0" % (port,))
MongoClient("[::1]:%d,localhost:%d" % (port, port))
client = MongoClient("localhost:%d,[::1]:%d" % (port, port))
client.pymongo_test.test.save({"dummy": u"object"})
client.pymongo_test_bernie.test.save({"dummy": u"object"})
dbs = client.database_names()
self.assertTrue("pymongo_test" in dbs)
self.assertTrue("pymongo_test_bernie" in dbs)
def test_ipv6(self):
try:
client = MongoClient("[::1]")
except:
# Either mongod was started without --ipv6
# or the OS doesn't support it (or both).
raise SkipTest("No IPv6")
# Try a few simple things
MongoClient("mongodb://[::1]:%d" % (port, ))
MongoClient("mongodb://[::1]:%d/?slaveOk=true" % (port, ))
MongoClient("[::1]:%d,localhost:%d" % (port, port))
client = MongoClient("localhost:%d,[::1]:%d" % (port, port))
client.pymongo_test.test.save({"dummy": u"object"})
client.pymongo_test_bernie.test.save({"dummy": u"object"})
dbs = client.database_names()
self.assertTrue("pymongo_test" in dbs)
self.assertTrue("pymongo_test_bernie" in dbs)
def test_unix_socket(self):
if not hasattr(socket, "AF_UNIX"):
raise SkipTest("UNIX-sockets are not supported on this system")
if (sys.platform == 'darwin'
and server_started_with_auth(MongoClient(host, port))):
raise SkipTest("SERVER-8492")
mongodb_socket = '/tmp/mongodb-27017.sock'
if not os.access(mongodb_socket, os.R_OK):
raise SkipTest("Socket file is not accessable")
self.assertTrue(MongoClient("mongodb://%s" % mongodb_socket))
client = MongoClient("mongodb://%s" % mongodb_socket)
client.pymongo_test.test.save({"dummy": "object"})
# Confirm we can read via the socket
dbs = client.database_names()
self.assertTrue("pymongo_test" in dbs)
# Confirm it fails with a missing socket
self.assertRaises(ConnectionFailure, MongoClient,
"mongodb:///tmp/none-existent.sock")
def test_copy_db(self):
c = MongoClient(self.host, self.port)
# We test copy twice; once starting in a request and once not. In
# either case the copy should succeed (because it starts a request
# internally) and should leave us in the same state as before the copy.
c.start_request()
self.assertRaises(TypeError, c.copy_database, 4, "foo")
self.assertRaises(TypeError, c.copy_database, "foo", 4)
self.assertRaises(InvalidName, c.copy_database, "foo", "$foo")
c.pymongo_test.test.drop()
c.drop_database("pymongo_test1")
c.drop_database("pymongo_test2")
c.pymongo_test.test.insert({"foo": "bar"})
# Due to SERVER-2329, databases may not disappear from a master in a
# master-slave pair
if not server_is_master_with_slave(c):
self.assertFalse("pymongo_test1" in c.database_names())
self.assertFalse("pymongo_test2" in c.database_names())
c.copy_database("pymongo_test", "pymongo_test1")
# copy_database() didn't accidentally end the request
self.assertTrue(c.in_request())
self.assertTrue("pymongo_test1" in c.database_names())
self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])
c.end_request()
self.assertFalse(c.in_request())
c.copy_database("pymongo_test", "pymongo_test2",
"%s:%d" % (self.host, self.port))
# copy_database() didn't accidentally restart the request
self.assertFalse(c.in_request())
self.assertTrue("pymongo_test2" in c.database_names())
self.assertEqual("bar", c.pymongo_test2.test.find_one()["foo"])
if version.at_least(c, (1, 3, 3, 1)):
c.drop_database("pymongo_test1")
c.pymongo_test.add_user("mike", "password")
self.assertRaises(OperationFailure, c.copy_database,
"pymongo_test", "pymongo_test1",
username="******", password="******")
if not server_is_master_with_slave(c):
self.assertFalse("pymongo_test1" in c.database_names())
self.assertRaises(OperationFailure, c.copy_database,
"pymongo_test", "pymongo_test1",
username="******", password="******")
if not server_is_master_with_slave(c):
self.assertFalse("pymongo_test1" in c.database_names())
if not is_mongos(c):
# See SERVER-6427
c.copy_database("pymongo_test", "pymongo_test1",
username="******", password="******")
self.assertTrue("pymongo_test1" in c.database_names())
self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])
def get_empty_db_da(_database_name, _json_schema_folder):
_client = MongoClient()
if _database_name in _client.database_names():
_client.drop_database(_client[_database_name])
_database = _client[_database_name]
return DatabaseAccess(_json_schema_folder=None, _database=_database)
def test_copy_db(self):
c = MongoClient(host, port)
# We test copy twice; once starting in a request and once not. In
# either case the copy should succeed (because it starts a request
# internally) and should leave us in the same state as before the copy.
c.start_request()
self.assertRaises(TypeError, c.copy_database, 4, "foo")
self.assertRaises(TypeError, c.copy_database, "foo", 4)
self.assertRaises(InvalidName, c.copy_database, "foo", "$foo")
c.pymongo_test.test.drop()
c.drop_database("pymongo_test1")
c.drop_database("pymongo_test2")
c.pymongo_test.test.insert({"foo": "bar"})
# Due to SERVER-2329, databases may not disappear from a master in a
# master-slave pair
if not server_is_master_with_slave(c):
self.assertFalse("pymongo_test1" in c.database_names())
self.assertFalse("pymongo_test2" in c.database_names())
c.copy_database("pymongo_test", "pymongo_test1")
# copy_database() didn't accidentally end the request
self.assertTrue(c.in_request())
self.assertTrue("pymongo_test1" in c.database_names())
self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])
c.end_request()
self.assertFalse(c.in_request())
c.copy_database("pymongo_test", "pymongo_test2",
"%s:%d" % (host, port))
# copy_database() didn't accidentally restart the request
self.assertFalse(c.in_request())
self.assertTrue("pymongo_test2" in c.database_names())
self.assertEqual("bar", c.pymongo_test2.test.find_one()["foo"])
if version.at_least(c, (1, 3, 3, 1)):
c.drop_database("pymongo_test1")
c.pymongo_test.add_user("mike", "password")
self.assertRaises(OperationFailure,
c.copy_database,
"pymongo_test",
"pymongo_test1",
username="******",
password="******")
if not server_is_master_with_slave(c):
self.assertFalse("pymongo_test1" in c.database_names())
self.assertRaises(OperationFailure,
c.copy_database,
"pymongo_test",
"pymongo_test1",
username="******",
password="******")
if not server_is_master_with_slave(c):
self.assertFalse("pymongo_test1" in c.database_names())
if not is_mongos(c):
# See SERVER-6427
c.copy_database("pymongo_test",
"pymongo_test1",
username="******",
password="******")
self.assertTrue("pymongo_test1" in c.database_names())
self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])
class RGCRequestParser(object):
def __init__(self):
# uri = "mongodb://*****:*****@172.22.164.85/?authSource=admin"
# master
# slave
self.client = MongoClient(self.uri)
self.db = self.client[u'rgctest']
self.posts = self.db.posts
# client = MongoClient('172.22.164.85')
# result = client.the_database.authenticate('one', '12345', mechanism='SCRAM-SHA-1')
# print(result)
self.post = {"author": "Mike",
"text": "My first blog post!",
"tags": ["mongodb", "python", "pymongo"],
"date": datetime.datetime.utcnow()}
self.__logger = mlogger
self.__logger.__name__ = 'LBS log analyser'
def get(self,post_id):
document = self.posts.find_one({'_id':ObjectId(post_id)})
print(document)
def testDB(self):
print self.client.database_names()
# post_id = self.posts.insert_one(self.post).inserted_id
# print(post_id)
# print self.posts.find_one()
# self.get(post_id)
for p in self.posts.find():
print(p)
def __analyseAndStore(self, lines):
geocoder = Geocoder()
rgc = RGC()
for line in lines:
if "/geocoder/v2/" in line:
#here deal with the web api request.
try:
bson = geocoder.getJson(line)
if bson is None:
continue
print(bson)
post_id = self.posts.insert_one(bson).inserted_id
except Exception as e:
self.__logger.logerror(str(e))
elif "qt=rgc" in line:
#here deal with the qt=rgc request.
try:
json = rgc.getJson(line)
if json is None:
continue
print(json)
post_id = self.posts.insert_one(json).inserted_id
except Exception as e:
self.__logger.logerror(str(e))
else:
#other types such as qt=rgc_stand
print()
def analyzelogfolder(self, folder):
for root, dirs, files in os.walk(folder):
for f in files:
filename = root + os.sep + f
file_read = open(filename, 'r')
lines = file_read.readlines()
self.__analyseAndStore(lines)
file_read.close()
lines = []
time.sleep(10)
def test_copy_db(self):
c = MongoClient(host, port)
# Due to SERVER-2329, databases may not disappear
# from a master in a master-slave pair.
if server_is_master_with_slave(c):
raise SkipTest("SERVER-2329")
if (not version.at_least(c, (2, 6, 0)) and
is_mongos(c) and server_started_with_auth(c)):
raise SkipTest("Need mongos >= 2.6.0 to test with authentication")
# We test copy twice; once starting in a request and once not. In
# either case the copy should succeed (because it starts a request
# internally) and should leave us in the same state as before the copy.
c.start_request()
self.assertRaises(TypeError, c.copy_database, 4, "foo")
self.assertRaises(TypeError, c.copy_database, "foo", 4)
self.assertRaises(InvalidName, c.copy_database, "foo", "$foo")
c.pymongo_test.test.drop()
c.drop_database("pymongo_test1")
c.drop_database("pymongo_test2")
self.assertFalse("pymongo_test1" in c.database_names())
self.assertFalse("pymongo_test2" in c.database_names())
c.pymongo_test.test.insert({"foo": "bar"})
c.copy_database("pymongo_test", "pymongo_test1")
# copy_database() didn't accidentally end the request
self.assertTrue(c.in_request())
self.assertTrue("pymongo_test1" in c.database_names())
self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])
c.end_request()
self.assertFalse(c.in_request())
if not c.is_mongos:
# On mongos, this would try to copydb from the mongos itself
# to the mongod that's the target. The mongod would try to
# begin a transaction on the mongos, which would fail.
c.copy_database("pymongo_test", "pymongo_test2",
"%s:%d" % (host, port))
# copy_database() didn't accidentally restart the request
self.assertFalse(c.in_request())
self.assertTrue("pymongo_test2" in c.database_names())
self.assertEqual("bar", c.pymongo_test2.test.find_one()["foo"])
# See SERVER-6427 for mongos
if not is_mongos(c) and server_started_with_auth(c):
c.drop_database("pymongo_test1")
c.admin.add_user("admin", "password")
c.admin.authenticate("admin", "password")
try:
c.pymongo_test.add_user("mike", "password")
self.assertRaises(OperationFailure, c.copy_database,
"pymongo_test", "pymongo_test1",
username="******", password="******")
self.assertFalse("pymongo_test1" in c.database_names())
self.assertRaises(OperationFailure, c.copy_database,
"pymongo_test", "pymongo_test1",
username="******", password="******")
self.assertFalse("pymongo_test1" in c.database_names())
c.copy_database("pymongo_test", "pymongo_test1",
username="******", password="******")
self.assertTrue("pymongo_test1" in c.database_names())
self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])
finally:
# Cleanup
remove_all_users(c.pymongo_test)
c.admin.remove_user("admin")
c.disconnect()
def test_copy_db(self):
c = MongoClient(host, port)
# Due to SERVER-2329, databases may not disappear
# from a master in a master-slave pair.
if server_is_master_with_slave(c):
raise SkipTest("SERVER-2329")
# We test copy twice; once starting in a request and once not. In
# either case the copy should succeed (because it starts a request
# internally) and should leave us in the same state as before the copy.
c.start_request()
self.assertRaises(TypeError, c.copy_database, 4, "foo")
self.assertRaises(TypeError, c.copy_database, "foo", 4)
self.assertRaises(InvalidName, c.copy_database, "foo", "$foo")
c.pymongo_test.test.drop()
c.drop_database("pymongo_test1")
c.drop_database("pymongo_test2")
self.assertFalse("pymongo_test1" in c.database_names())
self.assertFalse("pymongo_test2" in c.database_names())
c.pymongo_test.test.insert({"foo": "bar"})
c.copy_database("pymongo_test", "pymongo_test1")
# copy_database() didn't accidentally end the request
self.assertTrue(c.in_request())
self.assertTrue("pymongo_test1" in c.database_names())
self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])
c.end_request()
self.assertFalse(c.in_request())
c.copy_database("pymongo_test", "pymongo_test2",
"%s:%d" % (host, port))
# copy_database() didn't accidentally restart the request
self.assertFalse(c.in_request())
self.assertTrue("pymongo_test2" in c.database_names())
self.assertEqual("bar", c.pymongo_test2.test.find_one()["foo"])
# See SERVER-6427 for mongos
if (version.at_least(c, (1, 3, 3, 1)) and
not is_mongos(c) and server_started_with_auth(c)):
c.drop_database("pymongo_test1")
c.admin.add_user("admin", "password")
c.admin.authenticate("admin", "password")
try:
c.pymongo_test.add_user("mike", "password")
self.assertRaises(OperationFailure, c.copy_database,
"pymongo_test", "pymongo_test1",
username="******", password="******")
self.assertFalse("pymongo_test1" in c.database_names())
self.assertRaises(OperationFailure, c.copy_database,
"pymongo_test", "pymongo_test1",
username="******", password="******")
self.assertFalse("pymongo_test1" in c.database_names())
c.copy_database("pymongo_test", "pymongo_test1",
username="******", password="******")
self.assertTrue("pymongo_test1" in c.database_names())
self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])
finally:
# Cleanup
remove_all_users(c.pymongo_test)
c.admin.remove_user("admin")
c.disconnect()
class RGCRequestParser(object):
def __init__(self):
# uri = "mongodb://*****:*****@host/?authSource=admin"
#self.uri = "mongodb://*****:*****@host/?authSource=admin"
self.uri = "mongodb://*****:*****@host/?authSource=admin"
# slave
# uri = "mongodb://*****:*****@host/?authSource=admin"
self.client = MongoClient(self.uri)
self.db = self.client[u'rgconehour']
self.posts = self.db.rgconehour
# client = MongoClient('172.22.164.85')
# result = client.the_database.authenticate('one', '12345', mechanism='SCRAM-SHA-1')
# print(result)
self.post = {"author": "Mike",
"text": "My first blog post!",
"tags": ["mongodb", "python", "pymongo"],
"date": datetime.datetime.utcnow()}
self.__logger = mlogger
self.__logger.__name__ = 'LBS log analyser'
self.threadsize = 16
self.tp = threadPool.ThreadPool(self.threadsize)
def get(self,post_id):
document = self.posts.find_one({'_id':ObjectId(post_id)})
print(document)
def testDB(self):
print self.client.database_names()
# post_id = self.posts.insert_one(self.post).inserted_id
# print(post_id)
# print self.posts.find_one()
# self.get(post_id)
for p in self.posts.find():
print(p)
def __analyseAndStore(self, lines):
geocoder = Geocoder()
rgc = RGC()
for line in lines:
try:
#urls = self.htmlparser.parse_url(urlQueue.pop(), output_directory, target_url, crawl_interval, crawl_timeout)
#self.tp.add_job(self.__anslyseAndStoreSingle, line, geocoder, rgc)
self.__anslyseAndStoreSingle(line, geocoder, rgc)
except Exception as ex:
self.__logger.logerror(str(ex))
self.__logger.logerror(line)
# self.__anslyseAndStoreSingle(line, geocoder, rgc)
# if (count % self.threadsize) == 0:
# self.tp.wait_for_complete()
def __anslyseAndStoreSingle(self,line, geocoder,rgc):
print('in thread')
if "/geocoder/v2/" in line:
#here deal with the web api request.
try:
bson = geocoder.getJson(line)
print(bson)
if bson is None:
return
post_id = self.posts.insert_one(bson).inserted_id
except Exception as e:
self.__logger.logerror(str(e))
elif "qt=rgc" in line:
#here deal with the qt=rgc request.
try:
json = rgc.getJson(line)
if json is None:
return
print(json)
post_id = self.posts.insert_one(json).inserted_id
except Exception as e:
self.__logger.logerror(str(e))
else:
#other types such as qt=rgc_stand
print()
def analyzelogfolder(self, folder):
geocoder = Geocoder()
rgc = RGC()
for root, dirs, files in os.walk(folder):
for f in files:
filename = root + os.sep + f
file_read = open(filename, 'r')
# lines = file_read.readlines()
for line in file_read:
try:
#urls = self.htmlparser.parse_url(urlQueue.pop(), output_directory, target_url, crawl_interval, crawl_timeout)
#self.tp.add_job(self.__anslyseAndStoreSingle, line, geocoder, rgc)
self.__anslyseAndStoreSingle(line, geocoder, rgc)
except Exception as ex:
self.__logger.logerror(str(ex))
self.__logger.logerror(line)
# self.__anslyseAndStoreSingle(line, geocoder, rgc)
# if (count % self.threadsize) == 0:
# self.tp.wait_for_complete()
#self.__analyseAndStore(lines)
file_read.close()
lines = []
time.sleep(10)
def get_empty_db_da(_database_name, _json_schema_folder):
_client = MongoClient()
if _database_name in _client.database_names():
_client.drop_database(_client[_database_name])
_database = _client[_database_name]
return DatabaseAccess(_json_schema_folder=None, _database=_database)
