def test_alive(self): self.assertTrue(get_client().alive()) client = MongoClient('doesnt exist', _connect=False) self.assertFalse(client.alive())
def setUpClass(cls): cls.client = MongoClient(connect=False)
def __init__(self): self.mongo_client = MongoClient('localhost', 27017) self.mongo_db = self.mongo_client['taxi'] self.mongo_train = self.mongo_db['160_daytype_A_calltype_A']
def test_bad_uri(self): with self.assertRaises(InvalidURI): MongoClient("http://localhost")
from login import Login import pandas as pd from sklearn.ensemble import GradientBoostingClassifier from sklearn.model_selection import cross_val_score, GridSearchCV from pymongo.mongo_client import MongoClient from sklearn.externals import joblib import time # Conexión con la base de datos client = MongoClient('localhost', 27017) db = client['myDB'] collection = db["login"] # Rutina de entrenamiento def train(): records = [] for v in collection.find({}): login = Login() login.tiempo_presionado = v["tiempo_presionado"] login.tiempo_vuelo = v["tiempo_vuelo"] login.usuario = v["usuario"] records.append(login.toTuple()) features = ["tiempo_presionado", "tiempo_vuelo"] target = "usuario" labels = features + [target] df = pd.DataFrame.from_records(records, columns = labels) # filtrado de outliers
def test_get_default_database_error(self): # URI with no database. c = MongoClient("mongodb://%s:%d/" % (host, port), connect=False) self.assertRaises(ConfigurationError, c.get_default_database)
def test_init_disconnected_with_auth(self): uri = "mongodb://*****:*****@somedomainthatdoesntexist" c = MongoClient(uri, connectTimeoutMS=1, serverSelectionTimeoutMS=10) self.assertRaises(ConnectionFailure, c.pymongo_test.test.find_one)
def test_secondary_connection(self): self.c = MongoReplicaSetClient( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.assertTrue(bool(len(self.c.secondaries))) db = self.c.pymongo_test # Wait for replication... w = len(self.c.secondaries) + 1 db.test.remove({}, w=w) db.test.insert({'foo': 'bar'}, w=w) # Test direct connection to a primary or secondary primary_host, primary_port = ha_tools.get_primary().split(':') primary_port = int(primary_port) (secondary_host, secondary_port) = ha_tools.get_secondaries()[0].split(':') secondary_port = int(secondary_port) arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':') arbiter_port = int(arbiter_port) # MongoClient succeeds no matter the read preference for kwargs in [ {'read_preference': PRIMARY}, {'read_preference': PRIMARY_PREFERRED}, {'read_preference': SECONDARY}, {'read_preference': SECONDARY_PREFERRED}, {'read_preference': NEAREST}, {'slave_okay': True} ]: client = MongoClient(primary_host, primary_port, use_greenlets=use_greenlets, **kwargs) self.assertEqual(primary_host, client.host) self.assertEqual(primary_port, client.port) self.assertTrue(client.is_primary) # Direct connection to primary can be queried with any read pref self.assertTrue(client.pymongo_test.test.find_one()) client = MongoClient(secondary_host, secondary_port, use_greenlets=use_greenlets, **kwargs) self.assertEqual(secondary_host, client.host) self.assertEqual(secondary_port, client.port) self.assertFalse(client.is_primary) # Direct connection to secondary can be queried with any read pref # but PRIMARY if kwargs.get('read_preference') != PRIMARY: self.assertTrue(client.pymongo_test.test.find_one()) else: self.assertRaises( AutoReconnect, client.pymongo_test.test.find_one) # Since an attempt at an acknowledged write to a secondary from a # direct connection raises AutoReconnect('not master'), MongoClient # should do the same for unacknowledged writes. try: client.pymongo_test.test.insert({}, w=0) except AutoReconnect, e: self.assertEqual('not master', e.args[0]) else: self.fail( 'Unacknowledged insert into secondary client %s should' 'have raised exception' % (client,)) # Test direct connection to an arbiter client = MongoClient(arbiter_host, arbiter_port, **kwargs) self.assertEqual(arbiter_host, client.host) self.assertEqual(arbiter_port, client.port) self.assertFalse(client.is_primary) # See explanation above try: client.pymongo_test.test.insert({}, w=0) except AutoReconnect, e: self.assertEqual('not master', e.args[0])
def test_read_preference(self): # We pass through four states: # # 1. A primary and two secondaries # 2. Primary down # 3. Primary up, one secondary down # 4. Primary up, all secondaries down # # For each state, we verify the behavior of PRIMARY, # PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST c = MongoReplicaSetClient( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) def assertReadFrom(member, *args, **kwargs): utils.assertReadFrom(self, c, member, *args, **kwargs) def assertReadFromAll(members, *args, **kwargs): utils.assertReadFromAll(self, c, members, *args, **kwargs) def unpartition_node(node): host, port = node return '%s:%s' % (host, port) # To make the code terser, copy hosts into local scope primary = self.primary secondary = self.secondary other_secondary = self.other_secondary bad_tag = {'bad': 'tag'} # 1. THREE MEMBERS UP ------------------------------------------------- # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED # Trivial: mode and tags both match assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc) # Secondary matches but not primary, choose primary assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc) # Chooses primary, ignoring tag sets assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc) # Chooses primary, ignoring tag sets assertReadFrom(primary, PRIMARY_PREFERRED, bad_tag) assertReadFrom(primary, PRIMARY_PREFERRED, [bad_tag, {}]) # SECONDARY assertReadFromAll([secondary, other_secondary], SECONDARY) # SECONDARY_PREFERRED assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED) # Multiple tags assertReadFrom(secondary, SECONDARY_PREFERRED, self.secondary_tags) # Fall back to primary if it's the only one matching the tags assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'primary'}) # No matching secondaries assertReadFrom(primary, SECONDARY_PREFERRED, bad_tag) # Fall back from non-matching tag set to matching set assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED, [bad_tag, {}]) assertReadFrom(other_secondary, SECONDARY_PREFERRED, [bad_tag, {'dc': 'ny'}]) # NEAREST self.clear_ping_times() assertReadFromAll([primary, secondary, other_secondary], NEAREST) assertReadFromAll([primary, other_secondary], NEAREST, [bad_tag, {'dc': 'ny'}]) self.set_ping_time(primary, 0) self.set_ping_time(secondary, .03) # 30 ms self.set_ping_time(other_secondary, 10) # Nearest member, no tags assertReadFrom(primary, NEAREST) # Tags override nearness assertReadFrom(primary, NEAREST, {'name': 'primary'}) assertReadFrom(secondary, NEAREST, self.secondary_dc) # Make secondary fast self.set_ping_time(primary, .03) # 30 ms self.set_ping_time(secondary, 0) assertReadFrom(secondary, NEAREST) # Other secondary fast self.set_ping_time(secondary, 10) self.set_ping_time(other_secondary, 0) assertReadFrom(other_secondary, NEAREST) # High secondaryAcceptableLatencyMS, should read from all members assertReadFromAll( [primary, secondary, other_secondary], NEAREST, secondary_acceptable_latency_ms=1000*1000) self.clear_ping_times() assertReadFromAll([primary, other_secondary], NEAREST, [{'dc': 'ny'}]) # 2. PRIMARY DOWN ----------------------------------------------------- killed = ha_tools.kill_primary() # Let monitor notice primary's gone sleep(2 * MONITOR_INTERVAL) # PRIMARY assertReadFrom(None, PRIMARY) # PRIMARY_PREFERRED # No primary, choose matching secondary assertReadFromAll([secondary, other_secondary], PRIMARY_PREFERRED) assertReadFrom(secondary, PRIMARY_PREFERRED, {'name': 'secondary'}) # No primary or matching secondary assertReadFrom(None, PRIMARY_PREFERRED, bad_tag) # SECONDARY assertReadFromAll([secondary, other_secondary], SECONDARY) # Only primary matches assertReadFrom(None, SECONDARY, {'name': 'primary'}) # No matching secondaries assertReadFrom(None, SECONDARY, bad_tag) # SECONDARY_PREFERRED assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED) # Mode and tags both match assertReadFrom(secondary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST self.clear_ping_times() assertReadFromAll([secondary, other_secondary], NEAREST) # 3. PRIMARY UP, ONE SECONDARY DOWN ----------------------------------- ha_tools.restart_members([killed]) for _ in range(30): if ha_tools.get_primary(): break sleep(1) else: self.fail("Primary didn't come back up") ha_tools.kill_members([unpartition_node(secondary)], 2) self.assertTrue(MongoClient( unpartition_node(primary), use_greenlets=use_greenlets, read_preference=PRIMARY_PREFERRED ).admin.command('ismaster')['ismaster']) sleep(2 * MONITOR_INTERVAL) # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED assertReadFrom(primary, PRIMARY_PREFERRED) # SECONDARY assertReadFrom(other_secondary, SECONDARY) assertReadFrom(other_secondary, SECONDARY, self.other_secondary_dc) # Only the down secondary matches assertReadFrom(None, SECONDARY, {'name': 'secondary'}) # SECONDARY_PREFERRED assertReadFrom(other_secondary, SECONDARY_PREFERRED) assertReadFrom( other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc) # The secondary matching the tag is down, use primary assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST assertReadFromAll([primary, other_secondary], NEAREST) assertReadFrom(other_secondary, NEAREST, {'name': 'other_secondary'}) assertReadFrom(primary, NEAREST, {'name': 'primary'}) # 4. PRIMARY UP, ALL SECONDARIES DOWN --------------------------------- ha_tools.kill_members([unpartition_node(other_secondary)], 2) self.assertTrue(MongoClient( unpartition_node(primary), use_greenlets=use_greenlets, read_preference=PRIMARY_PREFERRED ).admin.command('ismaster')['ismaster']) # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED assertReadFrom(primary, PRIMARY_PREFERRED) assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc) # SECONDARY assertReadFrom(None, SECONDARY) assertReadFrom(None, SECONDARY, self.other_secondary_dc) assertReadFrom(None, SECONDARY, {'dc': 'ny'}) # SECONDARY_PREFERRED assertReadFrom(primary, SECONDARY_PREFERRED) assertReadFrom(primary, SECONDARY_PREFERRED, self.secondary_dc) assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) assertReadFrom(primary, SECONDARY_PREFERRED, {'dc': 'ny'}) # NEAREST assertReadFrom(primary, NEAREST) assertReadFrom(None, NEAREST, self.secondary_dc) assertReadFrom(None, NEAREST, {'name': 'secondary'}) # Even if primary's slow, still read from it self.set_ping_time(primary, 100) assertReadFrom(primary, NEAREST) assertReadFrom(None, NEAREST, self.secondary_dc) self.clear_ping_times()
# -*- coding: utf-8 -*- Dan_test = True import os.path if Dan_test: from pymongo.mongo_client import MongoClient C = MongoClient(port=int(37010)) C['admin'].authenticate('lmfdb','lmfdb') else: from lmfdb.lmfdb.base import getDBConnection C= getDBConnection() C['admin'].authenticate('lmfdb', 'lmfdb') # read-only if Dan_test: import sys from sage.all import preparse sys.path.append('/Users/d_yasaki/repos/lmfdb/lmfdb/scripts/hmf') else: import sage.misc.preparser from sage.misc.preparser import preparse from sage.interfaces.magma import magma from sage.all import ZZ, Rationals, PolynomialRing from check_conjugates import fix_one_label from sage.databases.cremona import class_to_int import yaml if Dan_test: pw_dict = yaml.load(open(os.path.join(os.getcwd(), "../../passwords.yaml")))
def do_index(self, job_manager, batch_size, ids, mode, **kwargs): client = MongoClient(**self.mongo_client_args) database = client[self.mongo_database_name] collection = database[self.mongo_collection_name] if ids: self.logger.info(("Indexing from '%s' with specific list of _ids, " "create indexer job with batch_size=%d."), self.mongo_collection_name, batch_size) # use user provided ids in batch id_provider = iter_n(ids, batch_size) else: self.logger.info(("Fetch _ids from '%s', and create " "indexer job with batch_size=%d."), self.mongo_collection_name, batch_size) # use ids from the target mongodb collection in batch id_provider = id_feeder(collection, batch_size, logger=self.logger) jobs = [] # asyncio.Future(s) error = None # the first Exception total = len(ids) if ids else collection.count() schedule = Schedule(total, batch_size) def batch_finished(future): nonlocal error try: schedule.finished += future.result() except Exception as exc: self.logger.warning(exc) error = exc for batch_num, ids in zip(schedule, id_provider): yield from asyncio.sleep(0.0) # when one batch failed, and job scheduling has not completed, # stop scheduling and cancel all on-going jobs, to fail quickly. if error: for job in jobs: if not job.done(): job.cancel() raise error self.logger.info(schedule) pinfo = self.pinfo.get_pinfo( schedule.suffix(self.mongo_collection_name)) job = yield from job_manager.defer_to_process( pinfo, dispatch, self.mongo_client_args, self.mongo_database_name, self.mongo_collection_name, self.es_client_args, self.es_blkidx_args, self.es_index_name, ids, mode, batch_num) job.add_done_callback(batch_finished) jobs.append(job) self.logger.info(schedule) yield from asyncio.gather(*jobs) schedule.completed() self.logger.notify(schedule) return {"count": total, "created_at": datetime.now().astimezone()}
def get_client(host, db, user, pwd): if user: return MongoClient(host, authSource=db, username=user, password=pwd) else: return MongoClient(host)
return "cardNumber: {} roomNumber: {} outcome: {} time: {}".format( self.cardNumber, self.roomNumber, self.outcome, self.time) def encode(accessRequest): return { "cardNumber": accessRequest.cardNumber, "outcome": accessRequest.outcome, "roomNumber": accessRequest.roomNumber, "timestamp": accessRequest.time } def decode(document): return AccessRequest(cardNumber=document["cardNumber"], roomNumber=document["roomNumber"], outcome=document["outcome"], time=document['timestamp']) client = MongoClient( 'mongodb://*****:*****@ds018258.mlab.com:18258/savage-security') db = client['savage-security'] ar = db.accessRequest for x in range(10): my_access = AccessRequest( random.randint(1, 999), 72, "granted" if x % 2 == 0 else "denied", datetime.datetime.utcnow()) # - datetime.timedelta(days=1)) ar.insert(encode(my_access))
from pymongo.mongo_client import MongoClient from pymongo.collection import Collection, ObjectId from pymongo.database import Database try: tasks_collection_name = 'tasks' defect_collection_name = 'defect' hsegap_collection_name = 'HSE_gaps_pbe_structure' db_info = { "aliases": {}, "database": "results_GY", "host": "marilyn.pcpm.ucl.ac.be", "port": 27017, "admin_user": "******", "admin_password": "******", "readonly_user": "******", "readonly_password": "******" } clt = MongoClient(host=db_info['host'], port=db_info['port']) db = Database(clt, db_info['database']) db.authenticate(db_info['admin_user'], db_info['admin_password']) except: print 'Can not connect database'
def test_max_pool_size_zero(self): with self.assertRaises(ValueError): MongoClient(maxPoolSize=0)
def setUp(self): seed_list = ha_tools.create_sharded_cluster() self.dbname = 'pymongo_mongos_ha' self.client = MongoClient(seed_list) self.client.drop_database(self.dbname)
def test_get_default_database(self): c = MongoClient("mongodb://%s:%d/foo" % (host, port), connect=False) self.assertEqual(Database(c, 'foo'), c.get_default_database())
def __init__(self, host, db): self.client = MongoClient(host) self.db = self.client.get_database(db) self.gifts = self.db.get_collection("gifts") initial()
def test_get_default_database_with_authsource(self): # Ensure we distinguish database name from authSource. uri = "mongodb://%s:%d/foo?authSource=src" % (host, port) c = MongoClient(uri, connect=False) self.assertEqual(Database(c, 'foo'), c.get_default_database())
# coding=utf-8 ''' duplication deletion ''' from pymongo.mongo_client import MongoClient from util_common import Constant import requests cli = MongoClient(Constant.MONGODB_URL) db = cli.spider attachs = db.component_original.find({"attachTask": Constant.DONE}, { "_id": True, "attachUrl": True, "attachUrl_uu": True }) fs_api_delete = "http://10.10.100.200:9999/file/delete?path=%s" attach_ids = set() for attach in attachs: if attach["_id"] not in attach_ids: try: oth_attachs = db.component_original.find( { "attachTask": Constant.DONE, "_id": { "$ne": attach["_id"] }, "attachUrl": attach["attachUrl"],
def test_host_w_port(self): with self.assertRaises(ValueError): connected( MongoClient("%s:1234567" % host, connectTimeoutMS=1, serverSelectionTimeoutMS=10))
def setUpClass(cls): cls.db = MongoClient(connect=False).pymongo_test
def setUpClass(cls): cls.client = MongoClient(host, port, connect=False, serverSelectionTimeoutMS=100)
def get_storage_instance(mongo) -> IDMappingStorage: client = MongoClient('localhost:' + str(mongo.port)) return IDMappingMongoStorage(client[DB_NAME])
import sys, time, os assert time import bson import re import hashlib import json import sage from sage.all import ZZ, QQ, PolynomialRing, prime_factors from zlib import gzip pw_filename = "../../../xyzzy" password = open(pw_filename, "r").readlines()[0].strip() from pymongo.mongo_client import MongoClient C = MongoClient(port=37010) C['numberfields'].authenticate('editor', password) fields = C.numberfields.fields saving = True PR = PolynomialRing(QQ, 'a') R = PolynomialRing(QQ, 'x') #def coeffs(s): # return [a for a in s[1:-1].split(',')] def sd(f): for k in f.keys(): print '%s ---> %s' % (k, f[k])
# # more details on how to customize your logging configuration. LOGGING = { 'version': 1, 'disable_existing_loggers': False, 'filters': { 'require_debug_false': { '()': 'django.utils.log.RequireDebugFalse' } }, 'handlers': { 'mail_admins': { 'level': 'ERROR', 'filters': ['require_debug_false'], 'class': 'django.utils.log.AdminEmailHandler' } }, 'loggers': { 'django.request': { 'handlers': ['mail_admins'], 'level': 'ERROR', 'propagate': True, }, } } from pymongo.mongo_client import MongoClient MONGODB = MongoClient("localhost", 27017)["xBadalona"] LANGUAGES_MONGO = MONGODB.text.distinct("lang") DEFAULT_LANGUAGE = 'ca'
import os.path import gzip import re import sys import time import sage.misc.preparser import subprocess from sage.interfaces.magma import magma from lmfdb.website import DEFAULT_DB_PORT as dbport from pymongo.mongo_client import MongoClient C= MongoClient(port=dbport) C['admin'].authenticate('lmfdb', 'lmfdb') # read-only import yaml pw_dict = yaml.load(open(os.path.join(os.getcwd(), os.extsep, os.extsep, os.extsep, "passwords.yaml"))) username = pw_dict['data']['username'] password = pw_dict['data']['password'] C['hmfs'].authenticate(username, password) hmf_forms = C.hmfs.forms hmf_fields = C.hmfs.fields C['admin'].authenticate('lmfdb', 'lmfdb') # read-only fields = C.numberfields.fields # hmf_forms.create_index('field_label') # hmf_forms.create_index('level_norm') # hmf_forms.create_index('level_ideal') # hmf_forms.create_index('dimension') magma.eval('nice_idealstr := function(F : Bound := 10000); idealsstr := []; ideals := IdealsUpTo(Bound, F); for I in ideals do bl, g := IsPrincipal(I); if bl then s := Sprintf("[%o, %o, %o]", Norm(I), Minimum(I), F!g); else zs := Generators(I); z := zs[#zs]; m := Minimum(I); z := F![(Integers()!c) mod m : c in Eltseq(F!z)]; assert ideal<Integers(F) | [m, z]> eq I; s := Sprintf("[%o, %o, %o]", Norm(I), m, z); end if; Append(~idealsstr, s); end for; return idealsstr; end function;')
from bs4 import BeautifulSoup import pandas as pd import numpy as np import json import string import spotipy import requests from spotipy.oauth2 import SpotifyClientCredentials import time import sys from unidecode import unidecode from pymongo.mongo_client import MongoClient URI = "mongodb://*****:*****@features-shard-00-00-edm1t.mongodb.net:27017,features-shard-00-01-edm1t.mongodb.net:27017,features-shard-00-02-edm1t.mongodb.net:27017/features?ssl=true&replicaSet=features-shard-0&authSource=admin" client = MongoClient(URI) db = client['MetaMind'] posts = db.posts #Genius base_url = "https://api.genius.com" headers = { 'Authorization': 'Bearer tQoh0aD9H5Od9EmoORVzKkki48MEG4K6Kyy8zCQvO8lq1Rjx1IVqEqUQMUgqJTHv' } #Spotify# SPOTIPY_CLIENT_ID = "3a883c6b1fc4405ba45608df5e60e09f" SPOTIPY_CLIENT_SECRET = "3168b907abf54925b8e482797f0eb718"
def open_spider(self, spider): """连接mongo数据库""" if spider.name == DoubanSpider.name: self.client = MongoClient() self.collection = self.client['douban']['top250']
def get_client(*args, **kwargs): return MongoClient(host, port, *args, **kwargs)