def test_alive(self):
self.assertTrue(get_client().alive())
client = MongoClient('doesnt exist', _connect=False)
self.assertFalse(client.alive())
def setUpClass(cls):
cls.client = MongoClient(connect=False)
def __init__(self):
self.mongo_client = MongoClient('localhost', 27017)
self.mongo_db = self.mongo_client['taxi']
self.mongo_train = self.mongo_db['160_daytype_A_calltype_A']
def test_bad_uri(self):
with self.assertRaises(InvalidURI):
MongoClient("http://localhost")
from login import Login
import pandas as pd
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import cross_val_score, GridSearchCV
from pymongo.mongo_client import MongoClient
from sklearn.externals import joblib
import time
# Conexión con la base de datos
client = MongoClient('localhost', 27017)
db = client['myDB']
collection = db["login"]
# Rutina de entrenamiento
def train():
records = []
for v in collection.find({}):
login = Login()
login.tiempo_presionado = v["tiempo_presionado"]
login.tiempo_vuelo = v["tiempo_vuelo"]
login.usuario = v["usuario"]
records.append(login.toTuple())
features = ["tiempo_presionado", "tiempo_vuelo"]
target = "usuario"
labels = features + [target]
df = pd.DataFrame.from_records(records, columns = labels)
# filtrado de outliers
def test_get_default_database_error(self):
# URI with no database.
c = MongoClient("mongodb://%s:%d/" % (host, port), connect=False)
self.assertRaises(ConfigurationError, c.get_default_database)
def test_init_disconnected_with_auth(self):
uri = "mongodb://*****:*****@somedomainthatdoesntexist"
c = MongoClient(uri, connectTimeoutMS=1, serverSelectionTimeoutMS=10)
self.assertRaises(ConnectionFailure, c.pymongo_test.test.find_one)
def test_secondary_connection(self):
self.c = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
self.assertTrue(bool(len(self.c.secondaries)))
db = self.c.pymongo_test
# Wait for replication...
w = len(self.c.secondaries) + 1
db.test.remove({}, w=w)
db.test.insert({'foo': 'bar'}, w=w)
# Test direct connection to a primary or secondary
primary_host, primary_port = ha_tools.get_primary().split(':')
primary_port = int(primary_port)
(secondary_host,
secondary_port) = ha_tools.get_secondaries()[0].split(':')
secondary_port = int(secondary_port)
arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':')
arbiter_port = int(arbiter_port)
# MongoClient succeeds no matter the read preference
for kwargs in [
{'read_preference': PRIMARY},
{'read_preference': PRIMARY_PREFERRED},
{'read_preference': SECONDARY},
{'read_preference': SECONDARY_PREFERRED},
{'read_preference': NEAREST},
{'slave_okay': True}
]:
client = MongoClient(primary_host,
primary_port,
use_greenlets=use_greenlets,
**kwargs)
self.assertEqual(primary_host, client.host)
self.assertEqual(primary_port, client.port)
self.assertTrue(client.is_primary)
# Direct connection to primary can be queried with any read pref
self.assertTrue(client.pymongo_test.test.find_one())
client = MongoClient(secondary_host,
secondary_port,
use_greenlets=use_greenlets,
**kwargs)
self.assertEqual(secondary_host, client.host)
self.assertEqual(secondary_port, client.port)
self.assertFalse(client.is_primary)
# Direct connection to secondary can be queried with any read pref
# but PRIMARY
if kwargs.get('read_preference') != PRIMARY:
self.assertTrue(client.pymongo_test.test.find_one())
else:
self.assertRaises(
AutoReconnect, client.pymongo_test.test.find_one)
# Since an attempt at an acknowledged write to a secondary from a
# direct connection raises AutoReconnect('not master'), MongoClient
# should do the same for unacknowledged writes.
try:
client.pymongo_test.test.insert({}, w=0)
except AutoReconnect, e:
self.assertEqual('not master', e.args[0])
else:
self.fail(
'Unacknowledged insert into secondary client %s should'
'have raised exception' % (client,))
# Test direct connection to an arbiter
client = MongoClient(arbiter_host, arbiter_port, **kwargs)
self.assertEqual(arbiter_host, client.host)
self.assertEqual(arbiter_port, client.port)
self.assertFalse(client.is_primary)
# See explanation above
try:
client.pymongo_test.test.insert({}, w=0)
except AutoReconnect, e:
self.assertEqual('not master', e.args[0])
def test_read_preference(self):
# We pass through four states:
#
# 1. A primary and two secondaries
# 2. Primary down
# 3. Primary up, one secondary down
# 4. Primary up, all secondaries down
#
# For each state, we verify the behavior of PRIMARY,
# PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST
c = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
def assertReadFrom(member, *args, **kwargs):
utils.assertReadFrom(self, c, member, *args, **kwargs)
def assertReadFromAll(members, *args, **kwargs):
utils.assertReadFromAll(self, c, members, *args, **kwargs)
def unpartition_node(node):
host, port = node
return '%s:%s' % (host, port)
# To make the code terser, copy hosts into local scope
primary = self.primary
secondary = self.secondary
other_secondary = self.other_secondary
bad_tag = {'bad': 'tag'}
# 1. THREE MEMBERS UP -------------------------------------------------
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
# Trivial: mode and tags both match
assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc)
# Secondary matches but not primary, choose primary
assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc)
# Chooses primary, ignoring tag sets
assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc)
# Chooses primary, ignoring tag sets
assertReadFrom(primary, PRIMARY_PREFERRED, bad_tag)
assertReadFrom(primary, PRIMARY_PREFERRED, [bad_tag, {}])
# SECONDARY
assertReadFromAll([secondary, other_secondary], SECONDARY)
# SECONDARY_PREFERRED
assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED)
# Multiple tags
assertReadFrom(secondary, SECONDARY_PREFERRED, self.secondary_tags)
# Fall back to primary if it's the only one matching the tags
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'primary'})
# No matching secondaries
assertReadFrom(primary, SECONDARY_PREFERRED, bad_tag)
# Fall back from non-matching tag set to matching set
assertReadFromAll([secondary, other_secondary],
SECONDARY_PREFERRED, [bad_tag, {}])
assertReadFrom(other_secondary,
SECONDARY_PREFERRED, [bad_tag, {'dc': 'ny'}])
# NEAREST
self.clear_ping_times()
assertReadFromAll([primary, secondary, other_secondary], NEAREST)
assertReadFromAll([primary, other_secondary],
NEAREST, [bad_tag, {'dc': 'ny'}])
self.set_ping_time(primary, 0)
self.set_ping_time(secondary, .03) # 30 ms
self.set_ping_time(other_secondary, 10)
# Nearest member, no tags
assertReadFrom(primary, NEAREST)
# Tags override nearness
assertReadFrom(primary, NEAREST, {'name': 'primary'})
assertReadFrom(secondary, NEAREST, self.secondary_dc)
# Make secondary fast
self.set_ping_time(primary, .03) # 30 ms
self.set_ping_time(secondary, 0)
assertReadFrom(secondary, NEAREST)
# Other secondary fast
self.set_ping_time(secondary, 10)
self.set_ping_time(other_secondary, 0)
assertReadFrom(other_secondary, NEAREST)
# High secondaryAcceptableLatencyMS, should read from all members
assertReadFromAll(
[primary, secondary, other_secondary],
NEAREST, secondary_acceptable_latency_ms=1000*1000)
self.clear_ping_times()
assertReadFromAll([primary, other_secondary], NEAREST, [{'dc': 'ny'}])
# 2. PRIMARY DOWN -----------------------------------------------------
killed = ha_tools.kill_primary()
# Let monitor notice primary's gone
sleep(2 * MONITOR_INTERVAL)
# PRIMARY
assertReadFrom(None, PRIMARY)
# PRIMARY_PREFERRED
# No primary, choose matching secondary
assertReadFromAll([secondary, other_secondary], PRIMARY_PREFERRED)
assertReadFrom(secondary, PRIMARY_PREFERRED, {'name': 'secondary'})
# No primary or matching secondary
assertReadFrom(None, PRIMARY_PREFERRED, bad_tag)
# SECONDARY
assertReadFromAll([secondary, other_secondary], SECONDARY)
# Only primary matches
assertReadFrom(None, SECONDARY, {'name': 'primary'})
# No matching secondaries
assertReadFrom(None, SECONDARY, bad_tag)
# SECONDARY_PREFERRED
assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED)
# Mode and tags both match
assertReadFrom(secondary, SECONDARY_PREFERRED, {'name': 'secondary'})
# NEAREST
self.clear_ping_times()
assertReadFromAll([secondary, other_secondary], NEAREST)
# 3. PRIMARY UP, ONE SECONDARY DOWN -----------------------------------
ha_tools.restart_members([killed])
for _ in range(30):
if ha_tools.get_primary():
break
sleep(1)
else:
self.fail("Primary didn't come back up")
ha_tools.kill_members([unpartition_node(secondary)], 2)
self.assertTrue(MongoClient(
unpartition_node(primary), use_greenlets=use_greenlets,
read_preference=PRIMARY_PREFERRED
).admin.command('ismaster')['ismaster'])
sleep(2 * MONITOR_INTERVAL)
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
assertReadFrom(primary, PRIMARY_PREFERRED)
# SECONDARY
assertReadFrom(other_secondary, SECONDARY)
assertReadFrom(other_secondary, SECONDARY, self.other_secondary_dc)
# Only the down secondary matches
assertReadFrom(None, SECONDARY, {'name': 'secondary'})
# SECONDARY_PREFERRED
assertReadFrom(other_secondary, SECONDARY_PREFERRED)
assertReadFrom(
other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc)
# The secondary matching the tag is down, use primary
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'})
# NEAREST
assertReadFromAll([primary, other_secondary], NEAREST)
assertReadFrom(other_secondary, NEAREST, {'name': 'other_secondary'})
assertReadFrom(primary, NEAREST, {'name': 'primary'})
# 4. PRIMARY UP, ALL SECONDARIES DOWN ---------------------------------
ha_tools.kill_members([unpartition_node(other_secondary)], 2)
self.assertTrue(MongoClient(
unpartition_node(primary), use_greenlets=use_greenlets,
read_preference=PRIMARY_PREFERRED
).admin.command('ismaster')['ismaster'])
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
assertReadFrom(primary, PRIMARY_PREFERRED)
assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc)
# SECONDARY
assertReadFrom(None, SECONDARY)
assertReadFrom(None, SECONDARY, self.other_secondary_dc)
assertReadFrom(None, SECONDARY, {'dc': 'ny'})
# SECONDARY_PREFERRED
assertReadFrom(primary, SECONDARY_PREFERRED)
assertReadFrom(primary, SECONDARY_PREFERRED, self.secondary_dc)
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'})
assertReadFrom(primary, SECONDARY_PREFERRED, {'dc': 'ny'})
# NEAREST
assertReadFrom(primary, NEAREST)
assertReadFrom(None, NEAREST, self.secondary_dc)
assertReadFrom(None, NEAREST, {'name': 'secondary'})
# Even if primary's slow, still read from it
self.set_ping_time(primary, 100)
assertReadFrom(primary, NEAREST)
assertReadFrom(None, NEAREST, self.secondary_dc)
self.clear_ping_times()
# -*- coding: utf-8 -*-
Dan_test = True
import os.path
if Dan_test:
from pymongo.mongo_client import MongoClient
C = MongoClient(port=int(37010))
C['admin'].authenticate('lmfdb','lmfdb')
else:
from lmfdb.lmfdb.base import getDBConnection
C= getDBConnection()
C['admin'].authenticate('lmfdb', 'lmfdb') # read-only
if Dan_test:
import sys
from sage.all import preparse
sys.path.append('/Users/d_yasaki/repos/lmfdb/lmfdb/scripts/hmf')
else:
import sage.misc.preparser
from sage.misc.preparser import preparse
from sage.interfaces.magma import magma
from sage.all import ZZ, Rationals, PolynomialRing
from check_conjugates import fix_one_label
from sage.databases.cremona import class_to_int
import yaml
if Dan_test:
pw_dict = yaml.load(open(os.path.join(os.getcwd(), "../../passwords.yaml")))
def do_index(self, job_manager, batch_size, ids, mode, **kwargs):
client = MongoClient(**self.mongo_client_args)
database = client[self.mongo_database_name]
collection = database[self.mongo_collection_name]
if ids:
self.logger.info(("Indexing from '%s' with specific list of _ids, "
"create indexer job with batch_size=%d."),
self.mongo_collection_name, batch_size)
# use user provided ids in batch
id_provider = iter_n(ids, batch_size)
else:
self.logger.info(("Fetch _ids from '%s', and create "
"indexer job with batch_size=%d."),
self.mongo_collection_name, batch_size)
# use ids from the target mongodb collection in batch
id_provider = id_feeder(collection, batch_size, logger=self.logger)
jobs = [] # asyncio.Future(s)
error = None # the first Exception
total = len(ids) if ids else collection.count()
schedule = Schedule(total, batch_size)
def batch_finished(future):
nonlocal error
try:
schedule.finished += future.result()
except Exception as exc:
self.logger.warning(exc)
error = exc
for batch_num, ids in zip(schedule, id_provider):
yield from asyncio.sleep(0.0)
# when one batch failed, and job scheduling has not completed,
# stop scheduling and cancel all on-going jobs, to fail quickly.
if error:
for job in jobs:
if not job.done():
job.cancel()
raise error
self.logger.info(schedule)
pinfo = self.pinfo.get_pinfo(
schedule.suffix(self.mongo_collection_name))
job = yield from job_manager.defer_to_process(
pinfo, dispatch, self.mongo_client_args,
self.mongo_database_name, self.mongo_collection_name,
self.es_client_args, self.es_blkidx_args, self.es_index_name,
ids, mode, batch_num)
job.add_done_callback(batch_finished)
jobs.append(job)
self.logger.info(schedule)
yield from asyncio.gather(*jobs)
schedule.completed()
self.logger.notify(schedule)
return {"count": total, "created_at": datetime.now().astimezone()}
def get_client(host, db, user, pwd):
if user:
return MongoClient(host, authSource=db, username=user, password=pwd)
else:
return MongoClient(host)
return "cardNumber: {} roomNumber: {} outcome: {} time: {}".format(
self.cardNumber, self.roomNumber, self.outcome, self.time)
def encode(accessRequest):
return {
"cardNumber": accessRequest.cardNumber,
"outcome": accessRequest.outcome,
"roomNumber": accessRequest.roomNumber,
"timestamp": accessRequest.time
}
def decode(document):
return AccessRequest(cardNumber=document["cardNumber"],
roomNumber=document["roomNumber"],
outcome=document["outcome"],
time=document['timestamp'])
client = MongoClient(
'mongodb://*****:*****@ds018258.mlab.com:18258/savage-security')
db = client['savage-security']
ar = db.accessRequest
for x in range(10):
my_access = AccessRequest(
random.randint(1, 999), 72, "granted" if x % 2 == 0 else "denied",
datetime.datetime.utcnow()) # - datetime.timedelta(days=1))
ar.insert(encode(my_access))
from pymongo.mongo_client import MongoClient
from pymongo.collection import Collection, ObjectId
from pymongo.database import Database
try:
tasks_collection_name = 'tasks'
defect_collection_name = 'defect'
hsegap_collection_name = 'HSE_gaps_pbe_structure'
db_info = {
"aliases": {},
"database": "results_GY",
"host": "marilyn.pcpm.ucl.ac.be",
"port": 27017,
"admin_user": "******",
"admin_password": "******",
"readonly_user": "******",
"readonly_password": "******"
}
clt = MongoClient(host=db_info['host'], port=db_info['port'])
db = Database(clt, db_info['database'])
db.authenticate(db_info['admin_user'], db_info['admin_password'])
except:
print 'Can not connect database'
def test_max_pool_size_zero(self):
with self.assertRaises(ValueError):
MongoClient(maxPoolSize=0)
def setUp(self):
seed_list = ha_tools.create_sharded_cluster()
self.dbname = 'pymongo_mongos_ha'
self.client = MongoClient(seed_list)
self.client.drop_database(self.dbname)
def test_get_default_database(self):
c = MongoClient("mongodb://%s:%d/foo" % (host, port), connect=False)
self.assertEqual(Database(c, 'foo'), c.get_default_database())
def __init__(self, host, db):
self.client = MongoClient(host)
self.db = self.client.get_database(db)
self.gifts = self.db.get_collection("gifts")
initial()
def test_get_default_database_with_authsource(self):
# Ensure we distinguish database name from authSource.
uri = "mongodb://%s:%d/foo?authSource=src" % (host, port)
c = MongoClient(uri, connect=False)
self.assertEqual(Database(c, 'foo'), c.get_default_database())
# coding=utf-8
'''
duplication deletion
'''
from pymongo.mongo_client import MongoClient
from util_common import Constant
import requests
cli = MongoClient(Constant.MONGODB_URL)
db = cli.spider
attachs = db.component_original.find({"attachTask": Constant.DONE}, {
"_id": True,
"attachUrl": True,
"attachUrl_uu": True
})
fs_api_delete = "http://10.10.100.200:9999/file/delete?path=%s"
attach_ids = set()
for attach in attachs:
if attach["_id"] not in attach_ids:
try:
oth_attachs = db.component_original.find(
{
"attachTask": Constant.DONE,
"_id": {
"$ne": attach["_id"]
},
"attachUrl": attach["attachUrl"],
def test_host_w_port(self):
with self.assertRaises(ValueError):
connected(
MongoClient("%s:1234567" % host,
connectTimeoutMS=1,
serverSelectionTimeoutMS=10))
def setUpClass(cls):
cls.db = MongoClient(connect=False).pymongo_test
def setUpClass(cls):
cls.client = MongoClient(host,
port,
connect=False,
serverSelectionTimeoutMS=100)
def get_storage_instance(mongo) -> IDMappingStorage:
client = MongoClient('localhost:' + str(mongo.port))
return IDMappingMongoStorage(client[DB_NAME])
import sys, time, os
assert time
import bson
import re
import hashlib
import json
import sage
from sage.all import ZZ, QQ, PolynomialRing, prime_factors
from zlib import gzip
pw_filename = "../../../xyzzy"
password = open(pw_filename, "r").readlines()[0].strip()
from pymongo.mongo_client import MongoClient
C = MongoClient(port=37010)
C['numberfields'].authenticate('editor', password)
fields = C.numberfields.fields
saving = True
PR = PolynomialRing(QQ, 'a')
R = PolynomialRing(QQ, 'x')
#def coeffs(s):
# return [a for a in s[1:-1].split(',')]
def sd(f):
for k in f.keys():
print '%s ---> %s' % (k, f[k])
# # more details on how to customize your logging configuration.
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'filters': {
'require_debug_false': {
'()': 'django.utils.log.RequireDebugFalse'
}
},
'handlers': {
'mail_admins': {
'level': 'ERROR',
'filters': ['require_debug_false'],
'class': 'django.utils.log.AdminEmailHandler'
}
},
'loggers': {
'django.request': {
'handlers': ['mail_admins'],
'level': 'ERROR',
'propagate': True,
},
}
}
from pymongo.mongo_client import MongoClient
MONGODB = MongoClient("localhost", 27017)["xBadalona"]
LANGUAGES_MONGO = MONGODB.text.distinct("lang")
DEFAULT_LANGUAGE = 'ca'
import os.path
import gzip
import re
import sys
import time
import sage.misc.preparser
import subprocess
from sage.interfaces.magma import magma
from lmfdb.website import DEFAULT_DB_PORT as dbport
from pymongo.mongo_client import MongoClient
C= MongoClient(port=dbport)
C['admin'].authenticate('lmfdb', 'lmfdb') # read-only
import yaml
pw_dict = yaml.load(open(os.path.join(os.getcwd(), os.extsep, os.extsep, os.extsep, "passwords.yaml")))
username = pw_dict['data']['username']
password = pw_dict['data']['password']
C['hmfs'].authenticate(username, password)
hmf_forms = C.hmfs.forms
hmf_fields = C.hmfs.fields
C['admin'].authenticate('lmfdb', 'lmfdb') # read-only
fields = C.numberfields.fields
# hmf_forms.create_index('field_label')
# hmf_forms.create_index('level_norm')
# hmf_forms.create_index('level_ideal')
# hmf_forms.create_index('dimension')
magma.eval('nice_idealstr := function(F : Bound := 10000); idealsstr := []; ideals := IdealsUpTo(Bound, F); for I in ideals do bl, g := IsPrincipal(I); if bl then s := Sprintf("[%o, %o, %o]", Norm(I), Minimum(I), F!g); else zs := Generators(I); z := zs[#zs]; m := Minimum(I); z := F![(Integers()!c) mod m : c in Eltseq(F!z)]; assert ideal<Integers(F) | [m, z]> eq I; s := Sprintf("[%o, %o, %o]", Norm(I), m, z); end if; Append(~idealsstr, s); end for; return idealsstr; end function;')
from bs4 import BeautifulSoup
import pandas as pd
import numpy as np
import json
import string
import spotipy
import requests
from spotipy.oauth2 import SpotifyClientCredentials
import time
import sys
from unidecode import unidecode
from pymongo.mongo_client import MongoClient
URI = "mongodb://*****:*****@features-shard-00-00-edm1t.mongodb.net:27017,features-shard-00-01-edm1t.mongodb.net:27017,features-shard-00-02-edm1t.mongodb.net:27017/features?ssl=true&replicaSet=features-shard-0&authSource=admin"
client = MongoClient(URI)
db = client['MetaMind']
posts = db.posts
#Genius
base_url = "https://api.genius.com"
headers = {
'Authorization':
'Bearer tQoh0aD9H5Od9EmoORVzKkki48MEG4K6Kyy8zCQvO8lq1Rjx1IVqEqUQMUgqJTHv'
}
#Spotify#
SPOTIPY_CLIENT_ID = "3a883c6b1fc4405ba45608df5e60e09f"
SPOTIPY_CLIENT_SECRET = "3168b907abf54925b8e482797f0eb718"
def open_spider(self, spider):
"""连接mongo数据库"""
if spider.name == DoubanSpider.name:
self.client = MongoClient()
self.collection = self.client['douban']['top250']
def get_client(*args, **kwargs):
return MongoClient(host, port, *args, **kwargs)
