def __init__(self): ''' init the common db connection transactions. ''' self.client = mongo_client.MongoClient() self.db = self.client.testnginxlog self.data_user = None
def get_an_instance(): client = None try: client = mongo_client.MongoClient("mongodb://localhost:27017") except Exception: print("Database connection error.") return client
def main(): # Parse the arguments parser = argparse.ArgumentParser( prog='surveil-pack-upload', add_help=False, ) parser.add_argument('--mongo-uri', default='mongodb://localhost:27017', help='Defaults to localhost', type=str) parser.add_argument('pack', metavar='[Pack]', type=str, nargs=1, help='Pack directory') (options, args) = parser.parse_known_args(sys.argv[1:]) pack_dir = options.pack[0] pack_name = os.path.basename(os.path.normpath(pack_dir)) # Find the .cfg files cfg_files = [ os.path.join(dirpath, f) for dirpath, dirnames, files in os.walk(pack_dir) for f in fnmatch.filter(files, '*.cfg') ] # Load the config conf = config.Config() loaded_conf = conf.read_config(cfg_files) raw_objects = conf.read_config_buf(loaded_conf) # Remove the empty items non_empty_config = {k: v for k, v in raw_objects.items() if v} for config_type in non_empty_config: for config_item in non_empty_config[config_type]: # Tag the config objects config_item['SURVEIL_PACK_NAME'] = pack_name # Replace lists with csv items_to_modify = ([ i for i in config_item.items() if isinstance(i[1], list) ]) for i in items_to_modify: config_item[i[0]] = ','.join(i[1]) # Remove the existing pack from mongodb mongo = mongo_client.MongoClient(options.mongo_uri) mongo_shinken = mongo.shinken for collection in ([ c for c in mongo_shinken.collection_names() if not c.startswith("system.") ]): mongo_shinken[collection].remove({'SURVEIL_PACK_NAME': pack_name}) # Add the replacement pack for config_type in non_empty_config: mongo_shinken[config_type + 's'].insert(non_empty_config[config_type])
def connect(self, uri, database, collection=None): """ Positional Arguments: - uri (string) - Mongo-compliant connection uri string database (string) - name of the database you want to connect to Optional Arguments: - collection (string) - name of the collection you want to reach (Default: None) """ try: self._client = mongo_client.MongoClient(uri) self._database = self._client[database] # Reaching collection names to check whether the received exists print 'Database Status : %s' % self._database.last_status() if collection: self._collection = self._database[collection] return True except errors.OperationFailure as e: print 'Error Code : %s - Details: %s' % (e.code, e.details) return False
def __init__(self, client=None, disk_cache=None, db_name="video_cache", collection_name='default'): uri = "mongodb://%s:%s@%s" % ( quote_plus(configs.MONGO_USERNAME), quote_plus(configs.MONGO_PASSWORD), configs.MONGO_HOST) self.client = (mongo_client.MongoClient(uri) if not client else client ) self.db = self.client[db_name] self.collection = self.db[collection_name] self.disk_cache = disk_cache
def test_connection_and_basic_query(self): client = mongo_client.MongoClient( 'mongodb://ds031651.mongolab.com:31651') database = client['vppdev'] database.authenticate('vppdev', 'vpp2015', mechanism='SCRAM-SHA-1') collection = database['registro'] document = collection.find_one() client.close() print(type(document))
def get_an_instance(): client = None try: client = mongo_client.MongoClient( "mongodb://*****:*****@localhost:27017/lifewatcher?authMechanism=SCRAM-SHA-1" ) except Exception: print("Database connection error.") return client
def __init__(self): self.mongodb_host=settings.MONGODB_HOST self.mongodb_port = settings.MONGODB_PORT self.mongodb_db = settings.MONGODB_DB self.mongodb_password = settings.MONGODB_PASSWORD self.mongodb_username = settings.MONGODB_USERNAME self.client = mongo_client.MongoClient( "mongodb://" + self.mongodb_username + ":" + self.mongodb_password + "@" + self.mongodb_host + ":" + self.mongodb_port + "/" + self.mongodb_db)
def __init__(self, path='./policy.csv', host='127.0.0.1', port=27017, db_name='szkj', doc_name='policy'): self.to_path = path self.conn = mongo_client.MongoClient(host=host, port=port) self.db = self.conn[db_name] self.doc = self.db[doc_name] if not self.conn: raise Exception('Connection failure')
def populate_user_data(): """ Pre-populate user data for the app, including an admin account """ try: db = mongo_client.MongoClient(config.MONGO_URI).twitter db.user.insert_one({ 'username': '******', 'password': '******', }) print("Created an admin account") except Exception as e: print(e)
def test_insertion(self): client = mongo_client.MongoClient( 'mongodb://ds031651.mongolab.com:31651') database = client['vppdev'] database.authenticate('vppdev', 'vpp2015', mechanism='SCRAM-SHA-1') collection = database['registro'] new_document = { u'Port': 25, u'Server': u'localhost', u'api': u'custom' } print(type(new_document)) collection.insert(new_document) print(dumps(new_document))
def __init__(self, db_name, host_name=settings.hostname): """ init the common db connection transactions. """ self.client = mongo_client.MongoClient(host=host_name) self.db = self.client.__getattr__(db_name) #init all the collections #set attr self.colname = db.col self.col_name_dict = DBCol().__dict__ for i in range(len(self.col_name_dict)): self.__setattr__( self.col_name_dict.keys()[i], self.db.__getattr__(self.col_name_dict.values()[i]))
def set_config(host,port,db,user,password): global _coll_language global _db global _coll_error from pymongo import mongo_client cnn = mongo_client.MongoClient(host=host, port=port) _db = cnn.get_database(db) _db.authenticate(user, password) _coll_language = _db.get_collection("sys.export_import_language") _coll_error = _db.get_collection("sys.export_import_error") pass
def __init__(self, conn_cred): """ Spins up the MongoDB Cluster with the credentials needed @:conn_cred, an XML parsed object containing necessary attributes """ self.loggerManager = DBLoggerManager() self.issues_logger = self.loggerManager.getIssuesLogger() self.queries_logger = self.loggerManager.getQueriesLogger() try: self.client = mongo_client.MongoClient(conn_cred) self.queries_logger.log(logging.INFO, "Connected!") print("Connected") except mongo_client.InvalidURI as e: self.issues_logger.log( logging.CRITICAL, "Connection failed. Info {0}".format(e.message))
def getConn(self): rootLogger.debug("MongoDaoSupport getConn start") conn = mongo_client.MongoClient( host=MongoConfig.HOST, port=MongoConfig.PORT, document_class=MongoConfig.DOCUMENT_CLASS, tz_aware=MongoConfig.TZ_AWARE, connect=MongoConfig.CONNECT, maxPoolSize=MongoConfig.MAX_POOL_SIZE, minPoolSize=MongoConfig.MIN_POOL_SIZE ) # 建立和数据库系统的连接,创建Connection时,指定host及port参数 db_auth = conn.admin # admin 数据库有帐号,连接-认证-切换库 db_auth.authenticate(MongoConfig.USER, MongoConfig.PWD) try: conn.admin.command('ismaster') except ConnectionFailure: print("Server not available") rootLogger.debug("MongoDaoSupport getConn end") return conn
def populate_hateword_data(): """ Populate the `hateword` table in MongoDB with data from CSV file. """ with open("./data/hate-speech-lexicons/refined_ngram_dict.csv") as f: lst = [row.split(',', 1)[0] for row in f] lst = lst[1:] lst = [{ 'word': word, 'category': [], 'similar_to': [] } for word in lst] try: db = mongo_client.MongoClient(config.MONGO_URI).twitter db.hateword.delete_many({}) result = db.hateword.insert_many(lst) print("Completed populating", len(result.inserted_ids), "hate words") except pymongo.errors.BulkWriteError as e: print(e.details)
# and then get the location info from ip2location.com ,store them into the mongodb. from mongobase import Conn __author__ = 'fandongyun' from utils import IpLocation from utils import IpLocationFetch from mongobase.Settings import DBColNames from pymongo import mongo_client from requests import ConnectionError log_conn = Conn("NginxEventLog") log_db = log_conn.get_db() client = mongo_client.MongoClient(host="172.26.178.208") ip_db = client.__getattr__("IPLocationDB") ip_db_col_name = "IPLocationInfo" def find_ip_from_collection(collection_name): distinct_ip = log_db.__getattr__(collection_name).distinct("remote_addr") print len(distinct_ip) return distinct_ip def is_ip_resolved(ip): #TODO ip_count = ip_db.__getattr__(ip_db_col_name).find({"ip": ip}).count() if ip_count > 0: return True
import matplotlib.pyplot as plt from pymongo import mongo_client import pprint import time uri = "mongodb+srv://github_reader:[email protected]/admin?ssl=true&authSource=admin&retryWrites=true&w=majority" client = mongo_client.MongoClient( "mongodb+srv://github_reader:[email protected]/admin?ssl=true&authSource=admin&retryWrites=true&w=majority" ) db = client['cryptomood'] collection = db.get_collection('github') symbols = set() print("1") for document in collection.find(): if document["symbol"] not in symbols: symbols.add(document["symbol"]) sym = document["symbol"] print(sym)
def __init__(self): client = mongo_client.MongoClient(host=db_host, port=db_port) self.rows = client['logging']['rows'] self.rows.ensure_index('session_id', ASCENDING, unique=True) self.rows.ensure_index('leased_at', ASCENDING)
import gridfs from pymongo import mongo_client from PIL import Image c = mongo_client.MongoClient() db = c.botPhotos fs = gridfs.GridFS(db) def save_image(path, c_id, num): fs.put(path.encode('ascii'), chat=c_id, filename=f"{c_id}_{num}.jpg") from io import BytesIO import requests def load_image(c_id): photos = [] for grid_out in fs.find({"chat": c_id}): file = grid_out.read() a = requests.get(file.decode("utf-8", "ignore")) img = Image.open(BytesIO(a.content)) photos.append(img) return photos
# directory = r"H:\temp\Ebeam_RunLog" import re import os import traceback import sys from pprint import pprint import datetime from dateutil.parser import parse # open database connections from pymongo import mongo_client from pprint import pprint client = mongo_client.MongoClient('mongodb://tali-e7440z:27017/') db = client['result_database'] # or db = client.test_database collection = db['fileStor'] # or collection = db.posts import pickle # from os import walk from scandir import \ walk # use scandir.walk instead of os.walk - its faster (5-10x faster on win64; 3x on Linux, MacOS) deepscanflag = True # DEEP SCAN: All dir and subdirs needed for user folders K Drive etc. def listFilesWithPattern2( directory, extension): # working version for one type of extension files_set = set()
def main(): c = mongo_client.MongoClient(connectTimeoutMS=60 * 1000) # jack up timeout c.drop_database("benchmark") db = c.benchmark timed("insert (small, no index)", insert, [db, 'small_none', small], setup_insert) timed("insert (medium, no index)", insert, [db, 'medium_none', medium], setup_insert) timed("insert (large, no index)", insert, [db, 'large_none', large], setup_insert) db.small_index.create_index("x", ASCENDING) timed("insert (small, indexed)", insert, [db, 'small_index', small]) db.medium_index.create_index("x", ASCENDING) timed("insert (medium, indexed)", insert, [db, 'medium_index', medium]) db.large_index.create_index("x", ASCENDING) timed("insert (large, indexed)", insert, [db, 'large_index', large]) timed("batch insert (small, no index)", insert_batch, [db, 'small_bulk', small], setup_insert) timed("batch insert (medium, no index)", insert_batch, [db, 'medium_bulk', medium], setup_insert) timed("batch insert (large, no index)", insert_batch, [db, 'large_bulk', large], setup_insert) timed("find_one (small, no index)", find_one, [db, 'small_none', per_trial / 2]) timed("find_one (medium, no index)", find_one, [db, 'medium_none', per_trial / 2]) timed("find_one (large, no index)", find_one, [db, 'large_none', per_trial / 2]) timed("find_one (small, indexed)", find_one, [db, 'small_index', per_trial / 2]) timed("find_one (medium, indexed)", find_one, [db, 'medium_index', per_trial / 2]) timed("find_one (large, indexed)", find_one, [db, 'large_index', per_trial / 2]) timed("find (small, no index)", find, [db, 'small_none', per_trial / 2]) timed("find (medium, no index)", find, [db, 'medium_none', per_trial / 2]) timed("find (large, no index)", find, [db, 'large_none', per_trial / 2]) timed("find (small, indexed)", find, [db, 'small_index', per_trial / 2]) timed("find (medium, indexed)", find, [db, 'medium_index', per_trial / 2]) timed("find (large, indexed)", find, [db, 'large_index', per_trial / 2]) # timed("find range (small, no index)", find, # [db, 'small_none', # {"$gt": per_trial / 4, "$lt": 3 * per_trial / 4}]) # timed("find range (medium, no index)", find, # [db, 'medium_none', # {"$gt": per_trial / 4, "$lt": 3 * per_trial / 4}]) # timed("find range (large, no index)", find, # [db, 'large_none', # {"$gt": per_trial / 4, "$lt": 3 * per_trial / 4}]) timed("find range (small, indexed)", find, [ db, 'small_index', { "$gt": per_trial / 2, "$lt": per_trial / 2 + batch_size } ]) timed("find range (medium, indexed)", find, [ db, 'medium_index', { "$gt": per_trial / 2, "$lt": per_trial / 2 + batch_size } ]) timed("find range (large, indexed)", find, [ db, 'large_index', { "$gt": per_trial / 2, "$lt": per_trial / 2 + batch_size } ])
def configure_and_connect(collection_name): client = mongo_client.MongoClient('mongodb://ds031651.mongolab.com:31651') database = client['vppdev'] database.authenticate('vppdev', 'vpp2015', mechanism='SCRAM-SHA-1') collection = database[collection_name] return {'client': client, 'collection': collection}
def __init__(self): self.logger = getLogger("LotteryDatabase") self.client = mongo_client.MongoClient(environ["MONGO_URI"]) self.lottery = self.client['bepis_bot']['lottery']
def __init__(self): self.logger = getLogger("CodeDatabase") self.client = mongo_client.MongoClient(environ["MONGO_URI"]) self.codes = self.client['bepis_bot']['codes'] self.codes.create_index("hash", unique=True)
def __init__(self): self.logger = getLogger("InviteDatabase") self.client = mongo_client.MongoClient(environ["MONGO_URI"]) self.invites = self.client['bepis_bot']['invites'] self.profiles = self.client['bepis_bot']['profiles'] super().__init__()
"""Handles database connection, and all that fun stuff. Adds a wrapper to pymongo. @package ppbot """ from pymongo import mongo_client from settings import * client = mongo_client.MongoClient(MONGO_HOST, MONGO_PORT) db = client[MONGO_DB]
post = { "Keywords": categories } posts = db["category_words"] posts.insert_one(post) pprint.pprint(posts.find()) print('Success!') print('(Make sure mongod is running...)') time.sleep(1) labels = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sept', 'Oct', 'Nov', 'Dec'] year = '2018' # year = datetime.datetime.today().strftime('%Y') # month = datetime.datetime.today().strftime('%m') wb = px.load_workbook('{} Monthly Expenses.xlsx'.format(year)) month = input('what month?\n') month_data = wb.get_sheet_by_name(labels[int(month)-1]) month_max = month_data.max_row client = mongo_client.MongoClient('localhost', 27017) db = client.month_expenses # update_expenses() category_upload()
def __init__(self, name: str): self.logger = getLogger(name + "-database") self.client = mongo_client.MongoClient(environ["MONGO_URI"]) self.profiles = self.client['bepis_bot']['profiles'] self.profiles.create_index("user_id", unique=True) super().__init__()
from pymongo import mongo_client import binascii BITS_MTU = 1522 * 8 HEX_MTU = 1522 * 2 mc = mongo_client.MongoClient("localhost", 27017) collection = mc["Packet_Snoop"].collection.Raw_Packet def build_cardinal(cardinal, y): c = cardinal.get(y) if c: return c else: id = len(cardinal.keys()) + 1 cardinal[y] = id return id def write_row_to_result_file(fh, doc, cardinal, count, raw=False): y = doc["Y"] l = doc["Length"] x = binascii.hexlify(doc["packet"]).decode('utf-8') if raw: row = [str(int(i, 16)) for i in x] else: row = [str(round(int(i, 16) / 15, 2)) for i in x]