def check_mongo_python_bindings(hostname, username, trd_party_dir) :
'''
TBD
'''
try:
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
_msg = "Checking MongoDB python bindings version....."
import pymongo
if pymongo.has_c() is False:
msg = "WARNING: You do not have the pymongo C extensions " + \
"installed. Mongodb performance will be extermely slow. " + \
"To resolve this, please make sure you have the " + \
"'python-dev' or 'python-devel' development " + \
"headers installed and then *remove* and *reinstall* " + \
"the mongodb python bindings."
print(msg)
_version = str(pymongo.version).strip().replace('+','')
del pymongo
_msg += compare_versions('2.1.1', _version)
_status = 0
except ImportError, e:
_status = 7282
def _show_config_mongoengine(app):
#print "----------------MONGODB_SETTINGS-----------------------"
try:
from mongoengine.connection import _connection_settings, DEFAULT_CONNECTION_NAME
from mongoengine import get_version as mongoengine_version
import pymongo
if 'MONGODB_SETTINGS' in app.config:
print "---------------------------------"
print "Your MONGODB_SETTINGS :"
print "---------------------------------"
for key, val in app.config['MONGODB_SETTINGS'].items():
print "%s = %s" % (key, val)
print "---------------------------------"
print "Reals Settings in mongoengine :"
print "---------------------------------"
for key, val in _connection_settings.get(DEFAULT_CONNECTION_NAME).items():
print "%s = %s" % (key, val)
print "-------------------------------------------------------"
print "Pymongo (has_c:%s).... : %s" % (pymongo.has_c(), pymongo.version)
print "Mongoengine .......... : %s" % mongoengine_version()
#print "-------------------------------------------------------"
except ImportError:
print "!!! PAS DE MONGODB_SETTINGS !!!"
def init(fast_db_url, big_db_url):
"""Initialize FDB/BDB factory singletons.
As a useful side effect, creates all (missing) indices.
@type fast_db_url: basestring
@type big_db_url: basestring
@raises MongoDBInitializationException: if cannot complete connection.
"""
global FDB, BDB
assert FDB is None and BDB is None, (FDB, BDB)
try:
FDB = DocStoreWrapperFactory(fast_db_url)
BDB = GFSEnabledDocStoreWrapperFactory(big_db_url)
except MongoDBInitializationException:
raise
if not bson.has_c():
logger.warning('python-bson-ext is not installed, '
'performance may be lower than expected!')
if not pymongo.has_c():
logger.warning('python-pymongo-ext is not installed, '
'performance may be lower than expected!')
logger.debug('Creating indices on Fast DataBase')
with FDB() as fdbw:
__make_indices(fdbqueries.FDB_INDICES_PER_COLLECTION.iteritems(),
dsw=fdbw)
logger.debug('Creating indices on Big DataBase')
with BDB() as bdbw:
__make_indices(bdbqueries.GFS_INDICES_PER_COLLECTION.iteritems(),
dsw=bdbw)
def init(fast_db_url, big_db_url):
"""Initialize FDB/BDB factory singletons.
As a useful side effect, creates all (missing) indices.
@type fast_db_url: basestring
@type big_db_url: basestring
@raises MongoDBInitializationException: if cannot complete connection.
"""
global FDB, BDB
assert FDB is None and BDB is None, (FDB, BDB)
try:
FDB = DocStoreWrapperFactory(fast_db_url)
BDB = GFSEnabledDocStoreWrapperFactory(big_db_url)
except MongoDBInitializationException:
raise
if not bson.has_c():
logger.warning('python-bson-ext is not installed, '
'performance may be lower than expected!')
if not pymongo.has_c():
logger.warning('python-pymongo-ext is not installed, '
'performance may be lower than expected!')
logger.debug('Creating indices on Fast DataBase')
with FDB() as fdbw:
__make_indices(fdbqueries.FDB_INDICES_PER_COLLECTION.iteritems(),
dsw=fdbw)
logger.debug('Creating indices on Big DataBase')
with BDB() as bdbw:
__make_indices(bdbqueries.GFS_INDICES_PER_COLLECTION.iteritems(),
dsw=bdbw)
def _show_config_mongoengine(app):
#print "----------------MONGODB_SETTINGS-----------------------"
try:
from mongoengine.connection import _connection_settings, DEFAULT_CONNECTION_NAME
from mongoengine import get_version as mongoengine_version
import pymongo
if 'MONGODB_SETTINGS' in app.config:
print "---------------------------------"
print "Your MONGODB_SETTINGS :"
print "---------------------------------"
for key, val in app.config['MONGODB_SETTINGS'].items():
print "%s = %s" % (key, val)
print "---------------------------------"
print "Reals Settings in mongoengine :"
print "---------------------------------"
for key, val in _connection_settings.get(
DEFAULT_CONNECTION_NAME).items():
print "%s = %s" % (key, val)
print "-------------------------------------------------------"
print "Pymongo (has_c:%s).... : %s" % (pymongo.has_c(),
pymongo.version)
print "Mongoengine .......... : %s" % mongoengine_version()
#print "-------------------------------------------------------"
except ImportError:
print "!!! PAS DE MONGODB_SETTINGS !!!"
def check_mongo_python_bindings(hostname, username, trd_party_dir):
'''
TBD
'''
try:
_status = 100
_fmsg = "An error has occurred, but no error message was captured"
_msg = "Checking MongoDB python bindings version....."
import pymongo
if pymongo.has_c() is False:
msg = "WARNING: You do not have the pymongo C extensions " + \
"installed. Mongodb performance will be extermely slow. " + \
"To resolve this, please make sure you have the " + \
"'python-dev' or 'python-devel' development " + \
"headers installed and then *remove* and *reinstall* " + \
"the mongodb python bindings."
print(msg)
_version = str(pymongo.version).strip().replace('+', '')
del pymongo
_msg += compare_versions('2.1.1', _version)
_status = 0
except ImportError, e:
_status = 7282
def setup_app():
if not pymongo.has_c():
logger.warning('Failed to load pymongo c bindings')
if not bson.has_c():
logger.warning('Failed to load bson c bindings')
if settings.conf.debug and settings.conf.ssl:
settings.conf.ssl = False
def setup_app():
if not pymongo.has_c():
logger.warning('Failed to load pymongo c bindings')
if not bson.has_c():
logger.warning('Failed to load bson c bindings')
if settings.conf.debug and settings.conf.ssl:
settings.conf.ssl = False
def test_dates(self):
doc = {"early": datetime.datetime(1686, 5, 5), "late": datetime.datetime(2086, 5, 5)}
try:
self.assertEqual(doc, BSON.from_dict(doc).to_dict())
except ValueError:
# Ignore ValueError when no C ext, since it's probably
# a problem w/ 32-bit Python - we work around this in the
# C ext, though.
if pymongo.has_c():
raise
def __init__(self, parameters) :
'''
TBD
'''
set_my_parameters(self, parameters)
self.pid = "TEST_" + getpwuid(os.getuid())[0]
self.mongodb_conn = False
if pymongo.has_c() is False:
msg = "WARNING: You do not have the pymongo C extensions installed. Data retrieval performance will be slow"
cberr(msg)
print(msg)
def __init__(self, parameters):
'''
TBD
'''
set_my_parameters(self, parameters)
self.pid = "TEST_" + getpwuid(os.getuid())[0]
self.mongodb_conn = False
if pymongo.has_c() is False:
msg = "WARNING: You do not have the pymongo C extensions installed. Data retrieval performance will be slow"
cberr(msg)
print(msg)
def __init__(self, parameters) :
MetricStoreMgdConn.__init__(self, parameters)
self.username = str(self.mongodb_username)
self.port = self.mongodb_port
self.mongodb_conn = False
if pymongo.has_c() is False:
msg = "WARNING: You do not have the pymongo C extensions installed. Data retrieval performance will be slow"
cberr(msg)
print(msg)
self.version = pymongo.version.split('.')[0]
def __init__(self, host='localhost'):
"""Connect to MongoDB database, get logaar collections
Create and populate them if needed"""
if not has_c():
log.warning("Pymongo C module not available. Consider installing it to increase performances.")
c = Connection()
self._connection = c
if 'logaar' not in c.database_names():
log.info("Creating logaar database")
db = c.logaar
if 'logs' not in db.collection_names():
log.info("Creating collection 'logs'")
self.logs = db.logs
# Create index if needed
db.logs.ensure_index(
[
('date', 1),
('level', 1),
('message', 1),
('program', 1),
('pid', 1),
],
unique = True,
dropDups = True,
)
if 'incoming' not in db.collection_names():
log.info("Creating collection 'incoming'")
db.create_collection("incoming", capped=True,size="100000")
self.incoming = db.incoming
# Create index if needed
db.incoming.ensure_index(
[
("date", DESCENDING)
]
)
if 'rules' not in db.collection_names():
log.info("Creating collection 'rules'")
#TODO: rewrite this using proper collection dump/restore
import setup_rules
for rulevals in setup_rules.rules_tuple:
d = dict(zip(setup_rules.keys, rulevals))
db.rules.insert(d)
self.rules = db.rules
log.info('connected')
def log_startup_info():
"""Log info about the current environment."""
LOG.always('Starting mongo-connector version: %s', __version__)
if 'dev' in __version__:
LOG.warning('This is a development version (%s) of mongo-connector',
__version__)
LOG.always('Python version: %s', sys.version)
LOG.always('Platform: %s', platform.platform())
LOG.always('pymongo version: %s', pymongo.__version__)
if not pymongo.has_c():
LOG.warning(
'pymongo version %s was installed without the C extensions. '
'"InvalidBSON: Date value out of range" errors may occur if '
'there are documents with BSON Datetimes that represent times '
'outside of Python\'s datetime limit.', pymongo.__version__)
def log_startup_info():
"""Log info about the current environment."""
LOG.always('Starting mongo-connector version: %s', __version__)
if 'dev' in __version__:
LOG.warning('This is a development version (%s) of mongo-connector',
__version__)
LOG.always('Python version: %s', sys.version)
LOG.always('Platform: %s', platform.platform())
LOG.always('pymongo version: %s', pymongo.__version__)
if not pymongo.has_c():
LOG.warning(
'pymongo version %s was installed without the C extensions. '
'"InvalidBSON: Date value out of range" errors may occur if '
'there are documents with BSON Datetimes that represent times '
'outside of Python\'s datetime limit.', pymongo.__version__)
def removenodedataolderthan(collection, period, test):
print('removenodedataolderthan starting for', period,
'hours. C extentions in use:', pymongo.has_c())
empty_results = {'results': '0'}
start = getstart(period)
total_records = 0
qry = {'time': {'$lte': start}}
print('query is %s ' % qry)
if test == 'true':
print('starting TEST query at ', dt.datetime.now())
cursor = collection.find(qry).batch_size(1000)
print('cursor returned at ', dt.datetime.now()) #this takes .2 millis
for row in cursor:
total_records = total_records + 1
print('total_records is ', total_records, ' at ', dt.datetime.now())
return total_records
elif test == 'false':
print('starting REMOVE query at ', dt.datetime.now())
results = collection.remove(qry)
print('REMOVE query results ', results)
print('REMOVE query finished at ', dt.datetime.now())
def setup_mongo():
if not pymongo.has_c():
logger.warning('Failed to load pymongo c bindings')
if not bson.has_c():
logger.warning('Failed to load bson c bindings')
prefix = settings.conf.mongodb_collection_prefix or ''
last_error = time.time() - 24
while True:
try:
client = pymongo.MongoClient(settings.conf.mongodb_url,
connectTimeoutMS=2000)
break
except pymongo.errors.ConnectionFailure:
time.sleep(0.5)
if time.time() - last_error > 30:
last_error = time.time()
logger.exception('Error connecting to mongodb server')
database = client.get_default_database()
cur_collections = database.collection_names()
if prefix + 'messages' not in cur_collections:
database.create_collection(prefix + 'messages',
capped=True,
size=100000)
mongo.collections.update({
'transaction':
getattr(database, prefix + 'transaction'),
'queue':
getattr(database, prefix + 'queue'),
'task':
getattr(database, prefix + 'task'),
'system':
getattr(database, prefix + 'system'),
'messages':
getattr(database, prefix + 'messages'),
'administrators':
getattr(database, prefix + 'administrators'),
'users':
getattr(database, prefix + 'users'),
'users_key_link':
getattr(database, prefix + 'users_key_link'),
'organizations':
getattr(database, prefix + 'organizations'),
'hosts':
getattr(database, prefix + 'hosts'),
'hosts_usage':
getattr(database, prefix + 'hosts_usage'),
'servers':
getattr(database, prefix + 'servers'),
'servers_output':
getattr(database, prefix + 'servers_output'),
'servers_bandwidth':
getattr(database, prefix + 'servers_bandwidth'),
'servers_ip_pool':
getattr(database, prefix + 'servers_ip_pool'),
'dh_params':
getattr(database, prefix + 'dh_params'),
'auth_nonces':
getattr(database, prefix + 'auth_nonces'),
'auth_limiter':
getattr(database, prefix + 'auth_limiter'),
'otp':
getattr(database, prefix + 'otp'),
'otp_cache':
getattr(database, prefix + 'otp_cache'),
})
if prefix + 'log_entries' not in cur_collections:
log_limit = settings.app.log_entry_limit
database.create_collection(prefix + 'log_entries',
capped=True,
size=log_limit * 256 * 2,
max=log_limit)
mongo.collections.update({
'log_entries':
getattr(database, prefix + 'log_entries'),
})
for collection_name, collection in mongo.collections.items():
collection.name_str = collection_name
settings.init()
mongo.collections['transaction'].ensure_index('lock_id', unique=True)
mongo.collections['transaction'].ensure_index([
('ttl_timestamp', pymongo.ASCENDING),
('state', pymongo.ASCENDING),
('priority', pymongo.DESCENDING),
])
mongo.collections['queue'].ensure_index('runner_id')
mongo.collections['queue'].ensure_index('ttl_timestamp')
mongo.collections['task'].ensure_index('type', unique=True)
mongo.collections['task'].ensure_index('ttl_timestamp')
mongo.collections['log_entries'].ensure_index([
('timestamp', pymongo.DESCENDING),
])
mongo.collections['messages'].ensure_index('channel')
mongo.collections['administrators'].ensure_index('username', unique=True)
mongo.collections['users'].ensure_index([
('type', pymongo.ASCENDING),
('org_id', pymongo.ASCENDING),
])
mongo.collections['users'].ensure_index([
('org_id', pymongo.ASCENDING),
('name', pymongo.ASCENDING),
])
mongo.collections['users_key_link'].ensure_index('key_id')
mongo.collections['users_key_link'].ensure_index('short_id', unique=True)
mongo.collections['organizations'].ensure_index('type')
mongo.collections['hosts'].ensure_index('name')
mongo.collections['hosts_usage'].ensure_index([
('host_id', pymongo.ASCENDING),
('timestamp', pymongo.ASCENDING),
])
mongo.collections['servers'].ensure_index('name')
mongo.collections['servers'].ensure_index('ping_timestamp')
mongo.collections['servers_output'].ensure_index([
('server_id', pymongo.ASCENDING),
('timestamp', pymongo.ASCENDING),
])
mongo.collections['servers_bandwidth'].ensure_index([
('server_id', pymongo.ASCENDING),
('period', pymongo.ASCENDING),
('timestamp', pymongo.ASCENDING),
])
mongo.collections['servers_ip_pool'].ensure_index([
('server_id', pymongo.ASCENDING),
('user_id', pymongo.ASCENDING),
])
mongo.collections['servers_ip_pool'].ensure_index('user_id')
mongo.collections['dh_params'].ensure_index('dh_param_bits')
mongo.collections['auth_nonces'].ensure_index([
('token', pymongo.ASCENDING),
('nonce', pymongo.ASCENDING),
],
unique=True)
mongo.collections['users_key_link'].ensure_index(
'timestamp', expireAfterSeconds=settings.app.key_link_timeout)
mongo.collections['auth_nonces'].ensure_index(
'timestamp', expireAfterSeconds=settings.app.auth_time_window * 2.1)
mongo.collections['auth_limiter'].ensure_index(
'timestamp', expireAfterSeconds=settings.app.auth_limiter_ttl)
mongo.collections['otp'].ensure_index('timestamp', expireAfterSeconds=120)
mongo.collections['otp_cache'].ensure_index(
'timestamp', expireAfterSeconds=settings.user.otp_cache_ttl)
if not auth.Administrator.collection.find_one():
auth.Administrator(
username=DEFAULT_USERNAME,
password=DEFAULT_PASSWORD,
default=True,
).commit()
secret_key = settings.app.cookie_secret
if not secret_key:
secret_key = re.sub(r'[\W_]+', '',
base64.b64encode(os.urandom(128)))[:64]
settings.app.cookie_secret = secret_key
settings.commit()
app.app.secret_key = secret_key.encode()
server_api_key = settings.app.server_api_key
if not server_api_key:
server_api_key = re.sub(r'[\W_]+', '',
base64.b64encode(os.urandom(128)))[:64]
settings.app.server_api_key = server_api_key
settings.commit()
from confPull import ConfPullThread
import logConfig
# Python
import os, sys, platform, time, threading, socket, traceback, random, hashlib, tempfile
socket.setdefaulttimeout( _settings.socket_timeout )
_agentVersion = "1.6.8"
_pymongoVersion = pymongo.version
_pymongoHasC = False
try:
_pymongoHasC = pymongo.has_c()
except:
pass
# Try and reduce the stack size.
try:
threading.stack_size(409600)
except:
pass
class AgentProcess( threading.Thread ):
""" The parent process - monitors agent process and checks for updates etc """
def __init__( self, loggerObj, agentDir, existingSessionKey ):
""" Construct the object """
self.logger = loggerObj
# hackery to make sure to use the bson parser without the C extensions
# since those crash on some of the input.
# https://jira.mongodb.org/browse/PYTHON-571
#a="/usr/lib/python2.7/site-packages/pymongo-2.6_-py2.7.egg/"
import sys
a="/usr/local/lib/python2.7/dist-packages/pymongo-2.6_-py2.7.egg"
sys.path.insert(0,a)
import pymongo
assert not pymongo.has_c(), "you must compile pymongo with --no-ext"
import bson
import datetime
import collections
import mmap,os,struct,pdb,json,time,sys
schema = json.load(open('schema'))
def run(filename):
fd=open(filename,"r")
# we used mmap before, but converted to stdio while chasing down the
# bson parser bug. It's probably safe to change it back. The slowdown
# wasn't that bad though, so I didn't bother.
mm = 0# mmap.mmap(fd,0,access=mmap.ACCESS_READ)
# pdb.set_trace()
scan(filename,fd,mm)
fd.close()
print ('done',dt(),filename)
t0 = time.time()
def dt(): return time.time()-t0
def __init__(self, mongo_address, doc_managers=None, **kwargs):
super(Connector, self).__init__()
# can_run is set to false when we join the thread
self.can_run = True
# main address - either mongos for sharded setups or a primary otherwise
self.address = mongo_address
# connection to the main address
self.main_conn = None
# List of DocManager instances
if doc_managers:
self.doc_managers = doc_managers
else:
LOG.warning('No doc managers specified, using simulator.')
self.doc_managers = (simulator.DocManager(),)
if not pymongo.has_c():
warning = ('pymongo version %s was installed without the C '
'extensions. "InvalidBSON: Date value out of '
'range" errors may occur if there are documents '
'with BSON Datetimes that represent times outside of '
'Python\'s datetime.datetime limit.') % (
pymongo.__version__,)
# Print and warn to make it extra noticeable
print(warning)
LOG.warning(warning)
# Password for authentication
self.auth_key = kwargs.pop('auth_key', None)
# Username for authentication
self.auth_username = kwargs.pop('auth_username', None)
# The name of the file that stores the progress of the OplogThreads
self.oplog_checkpoint = kwargs.pop('oplog_checkpoint',
'oplog.timestamp')
# The set of OplogThreads created
self.shard_set = {}
# Dict of OplogThread/timestamp pairs to record progress
self.oplog_progress = LockingDict()
# Timezone awareness
self.tz_aware = kwargs.get('tz_aware', False)
# SSL keyword arguments to MongoClient.
ssl_certfile = kwargs.pop('ssl_certfile', None)
ssl_ca_certs = kwargs.pop('ssl_ca_certs', None)
ssl_keyfile = kwargs.pop('ssl_keyfile', None)
ssl_cert_reqs = kwargs.pop('ssl_cert_reqs', None)
self.ssl_kwargs = {}
if ssl_certfile:
self.ssl_kwargs['ssl_certfile'] = ssl_certfile
if ssl_ca_certs:
self.ssl_kwargs['ssl_ca_certs'] = ssl_ca_certs
if ssl_keyfile:
self.ssl_kwargs['ssl_keyfile'] = ssl_keyfile
if ssl_cert_reqs:
self.ssl_kwargs['ssl_cert_reqs'] = ssl_cert_reqs
# Save the rest of kwargs.
self.kwargs = kwargs
# Replace the origin dest_mapping
self.dest_mapping = DestMapping(kwargs.get('ns_set', []),
kwargs.get('ex_ns_set', []),
kwargs.get('dest_mapping', {}))
# Initialize and set the command helper
command_helper = CommandHelper(self.dest_mapping)
for dm in self.doc_managers:
dm.command_helper = command_helper
if self.oplog_checkpoint is not None:
if not os.path.exists(self.oplog_checkpoint):
info_str = ("MongoConnector: Can't find %s, "
"attempting to create an empty progress log" %
self.oplog_checkpoint)
LOG.warning(info_str)
try:
# Create oplog progress file
open(self.oplog_checkpoint, "w").close()
except IOError as e:
LOG.critical("MongoConnector: Could not "
"create a progress log: %s" %
str(e))
sys.exit(2)
else:
if (not os.access(self.oplog_checkpoint, os.W_OK)
and not os.access(self.oplog_checkpoint, os.R_OK)):
LOG.critical("Invalid permissions on %s! Exiting" %
(self.oplog_checkpoint))
sys.exit(2)
# Copyright 2009-2014 MongoDB, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Fail if the C extension module doesn't exist.
Only really intended to be used by internal build scripts.
"""
import sys
sys.path[0:0] = [""]
import bson
import pymongo
if not pymongo.has_c() or not bson.has_c():
sys.exit("could not load C extensions")
def setup_check():
if not pymongo.has_c():
logger.warning('Failed to load pymongo c bindings')
if not bson.has_c():
logger.warning('Failed to load bson c bindings')
class PopulateDatabase:
baseUrl = "https://www.imdb.com"
headers = {"Accept-Language": "en-US"}
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
logHandler = logging.FileHandler("Logs/populate_database.log")
logHandler.setFormatter(
logging.Formatter(
"%(asctime)s (%(filename)s)/(%(funcName)s): %(message)s"))
logger.addHandler(logHandler)
logger.info("-----------NEW SESSION-----------")
pymongo.has_c() == True
def __init__(self, client):
self.db = client.sugsn
def parseToMovieItem(self, object):
primaryInfo = object.findNext("h3", class_="lister-item-header")
secondaryInfo = object.findNext("p", class_="text-muted")
ratingsInfo = object.findNext("div", class_="ratings-bar")
posterData = object.findNext(
"div", class_="lister-item-image float-left").findNext(
"a").findNext("img")
return MovieItem(
primaryInfo.findNext(
"span", class_="lister-item-index unbold text-primary").text,
posterData["data-tconst"],
primaryInfo.findNext("a").text,
primaryInfo.findNext(
"span", class_="lister-item-year text-muted unbold").text,
secondaryInfo.findNext("span", class_="certificate").text,
secondaryInfo.findNext("span", class_="runtime").text,
secondaryInfo.findNext("span", class_="genre").text,
ratingsInfo.findNext("strong").text,
ratingsInfo.findNext(
"div",
class_="inline-block ratings-metascore").findNext("span").text,
ratingsInfo.findNext("p", class_="text-muted").text,
posterData["loadlate"])
# Imdb Parser
def getTopRatedMovies(self):
html = requests.get(
self.baseUrl +
"/search/title/?sort=user_rating&title_type=feature&num_votes=250000,",
headers=self.headers)
soup = BeautifulSoup(html.content, "lxml")
nextTag = soup.findAll("a", class_="lister-page-next next-page")
# Traverse pages
while (len(nextTag) != 0):
# Populate database
resultList = soup.find_all("div",
class_="lister-item mode-advanced")
for item in resultList:
movie = self.parseToMovieItem(item)
self.logger.info("Adding " + str(movie.listNum) +
" - %s" % movie.name)
self.db.topRatedMovies.update_one({"listNum": movie.listNum},
{"$set": movie.__dict__},
True)
# Load next page
html = requests.get(self.baseUrl + nextTag[0]["href"],
headers=self.headers)
soup = BeautifulSoup(html.content, "lxml")
nextTag = soup.findAll("a", class_="lister-page-next next-page")
# @proyect inKoutPi
# history dao
__author__ = 'rbioque'
import pymongo
pymongo.has_c()
from . import measure
class HistoryDao:
def __init__(self):
# Making connection with MongoGlient
from pymongo import MongoClient
client = MongoClient('localhost', 27017)
self.db = client.inKoutPiDB
def persist(self, measure):
self.db.HISTORY.insert(measure.toDBCollection())
def findLastMeasure(self):
measure = self.db.HISTORY.find().sort("date", -1).limit(1)
return measure
def findLastsMeasure(self, rows):
measures = self.db.HISTORY.find().sort("date", -1).limit(rows)
return measures
def findLastsOnlyMeasure(self, rows):
measures = self.db.HISTORY.find({
"msg": {
"$exists": False
# h t t p s : / / github .com /mkleehammer/pyodbc/wiki/Getting-started
servername = 'HIPERDRIVE2\MSSQLSERVER01'
port = '1433'
database = 'pymssql'
username = '******'
password = '******'
driver = '{ODBC Driver 17 for SQL Server}'
connSql = pyodbc.connect(';DRIVER=' + driver + ';SERVER=' + servername +
';PORT=' + port + ';DATABASE=' + database + ';UID=' +
username + ';PWD=' + password +
';Trusted_Connection=yes')
cursorSql = connSql.cursor()
print(pymongo.version)
print(pymongo.has_c())
connMongo = pymongo.MongoClient(
"mongodb://localhost:27017") # connect to the db on standard port
print(connMongo)
dbMongo = connMongo.pymongo # attach to db m101
print(dbMongo)
collMongo = dbMongo.ColecaoUm # specify the collection funnynumbers
print(collMongo)
cursorSql.execute("""
SELECT * FROM dbo.Table_1 WHERE name=?
""", 'pateta')
rowSql = cursorSql.fetchone()
descriptionSql = cursorSql.description
print(descriptionSql)
columnNamesSQL = [column[0] for column in cursorSql.description
def getdatausinggw(gw, node, start, mytype, timezone):
print('getdatausinggw starting. C extentions in use:', pymongo.has_c())
docs = []
resultsarray = []
empty_results = {'results': '0'}
try:
toZone = gettz(timezone)
fromZone = tzutc()
except ValueError as err:
print('Invalid timezone parameter %s. Defaulting to 0' % tz)
tz = 'None'
qry = {'gateway_id': gw, 'node_id': str(node), 'time': {'$gte': start}}
sortparam = [('time', 1)]
if mytype:
qry['type'] = mytype
print('query is %s and sort is ' % qry, sortparam)
print('starting query at ', dt.datetime.now())
cursor = sensors.find(qry).sort(sortparam).batch_size(100000)
print('query returned at ', dt.datetime.now()) #this takes .2 millis
for row in cursor:
resultsarray.append(row)
print('Arrayified at ', dt.datetime.now())
count = len(resultsarray)
print('%i records returned' % count, dt.datetime.now())
#this takes 1.86 sec
ct = 0
total = 0
skip = 0
if count == 0:
return empty_results
if count > 300:
skip = int(count / 300 + .49)
print('Since more than 300 records were returned, skip is set to %i' %
skip, dt.datetime.now()) #this take .1 milli
#insert initial doc as first 'goalpost' with time same as start
newdoc = {'value': 0, 'human_time': '', 'time': 0}
# datetimes in DB are utc, convert to local timezone
newdoc['human_time'] = dt.datetime.fromtimestamp(start).replace(
tzinfo=fromZone).astimezone(toZone).strftime(timefmt)
newdoc['time'] = start
newvalue = cleanvalue(resultsarray[skip + 1]['value'])
newdoc['value'] = newvalue
docs.append(newdoc)
for doc in resultsarray:
total += 1
ct += 1
newdoc = {'value': 0, 'human_time': '', 'time': 0}
# skip if needed
if ct > skip:
#newdoc['value'] = float(doc['value'].replace('b', '').replace('v', '').replace("'", ""))
newvalue = cleanvalue(doc['value'])
newdoc['value'] = newvalue
latestvalue = newvalue
# datetimes in DB are utc, convert to local timezone
newdoc['human_time'] = dt.datetime.fromtimestamp(
doc['time']).replace(
tzinfo=fromZone).astimezone(toZone).strftime(timefmt)
newdoc['time'] = doc['time']
docs.append(newdoc)
ct = 0
if ct != 0:
newdoc['value'] = float(doc['value'].replace('b', '').replace(
'v', '').replace("'", ""))
# datetimes in DB are utc, convert to local timezone
newdoc['human_time'] = dt.datetime.fromtimestamp(doc['time']).replace(
tzinfo=fromZone).astimezone(toZone).strftime(timefmt)
newdoc['time'] = doc['time']
docs.append(newdoc)
#insert last doc as end 'goalpost' with timestamp of now
newdoc = {'value': 0, 'human_time': '', 'time': 0}
# datetimes in DB are utc, convert to local timezone
newdoc['human_time'] = dt.datetime.timestamp(dt.datetime.now())
newdoc['time'] = dt.datetime.timestamp(dt.datetime.now())
newdoc['value'] = latestvalue
docs.append(newdoc)
print('total docs found:', total, ' and returning:', len(docs))
print('document returned at ', dt.datetime.now()) #this takes 2.28 sec
return docs
import numpy as np
from bson import BSON, CodecOptions, Int64, ObjectId
from bson.raw_bson import RawBSONDocument
try:
import bsonnumpy
except (ImportError, OSError) as exc:
print(exc)
bsonnumpy = None
try:
import monary
except (ImportError, OSError) as exc:
monary = None
assert pymongo.has_c()
# Use large document in tests? If SMALL, no, if LARGE, then yes.
SMALL = False
LARGE = True
db = None
raw_bson = None
large_doc_keys = None
collection_names = {LARGE: "large", SMALL: "small"}
dtypes = {}
raw_bsons = {}
def _setup():
global db
global raw_bson
def setup_check():
if not pymongo.has_c():
logger.warning('Failed to load pymongo c bindings')
if not bson.has_c():
logger.warning('Failed to load bson c bindings')
del sys.modules[name]
for module, attr in ((bson, '_cbson'), (pymongo, '_cmessage'),
(pymongo.message, '_cmessage')):
try:
delattr(module, attr)
except AttributeError:
pass
del attr, name, module, needs_replacing
importlib.reload(bson)
importlib.reload(pymongo.message)
assert not bson.has_c()
assert not pymongo.has_c()
import types, functools
import bson, bson.son, contextlib, struct
from bson.binary import Binary, OLD_UUID_SUBTYPE
from gridfs import GridFS
from collections import OrderedDict
from bson import _elements_to_dict, DBRef
from pytransact import spickle
max_bson_element_size = 2 ** 18
Extension = spickle.ExtensionType()
def register(typeobj, code=spickle.ExtensionType._empty):
from confPull import ConfPullThread
import logConfig
# Python
import os, sys, platform, time, threading, socket, traceback, random, hashlib, tempfile
socket.setdefaulttimeout(_settings.socket_timeout)
_agentVersion = "1.5.7"
_pymongoVersion = pymongo.version
_pymongoHasC = False
try:
_pymongoHasC = pymongo.has_c()
except:
pass
# Try and reduce the stack size.
try:
threading.stack_size(409600)
except:
pass
class AgentProcess(threading.Thread):
""" The parent process - monitors agent process and checks for updates etc """
def __init__(self, loggerObj, agentDir, existingSessionKey):
""" Construct the object """
self.logger = loggerObj
def setup_mongo():
if not pymongo.has_c():
logger.warning('Failed to load pymongo c bindings')
if not bson.has_c():
logger.warning('Failed to load bson c bindings')
prefix = settings.conf.mongodb_collection_prefix or ''
last_error = time.time() - 24
while True:
try:
client = pymongo.MongoClient(settings.conf.mongodb_url,
connectTimeoutMS=2000)
break
except pymongo.errors.ConnectionFailure:
time.sleep(0.5)
if time.time() - last_error > 30:
last_error = time.time()
logger.exception('Error connecting to mongodb server')
database = client.get_default_database()
cur_collections = database.collection_names()
if prefix + 'messages' not in cur_collections:
database.create_collection(prefix + 'messages', capped=True,
size=100000)
mongo.collections.update({
'transaction': getattr(database, prefix + 'transaction'),
'queue': getattr(database, prefix + 'queue'),
'task': getattr(database, prefix + 'task'),
'system': getattr(database, prefix + 'system'),
'messages': getattr(database, prefix + 'messages'),
'administrators': getattr(database, prefix + 'administrators'),
'users': getattr(database, prefix + 'users'),
'users_key_link': getattr(database, prefix + 'users_key_link'),
'organizations': getattr(database, prefix + 'organizations'),
'hosts': getattr(database, prefix + 'hosts'),
'hosts_usage': getattr(database, prefix + 'hosts_usage'),
'servers': getattr(database, prefix + 'servers'),
'servers_output': getattr(database, prefix + 'servers_output'),
'servers_bandwidth': getattr(database, prefix + 'servers_bandwidth'),
'servers_ip_pool': getattr(database, prefix + 'servers_ip_pool'),
'dh_params': getattr(database, prefix + 'dh_params'),
'auth_nonces': getattr(database, prefix + 'auth_nonces'),
'auth_limiter': getattr(database, prefix + 'auth_limiter'),
'otp': getattr(database, prefix + 'otp'),
'otp_cache': getattr(database, prefix + 'otp_cache'),
})
if prefix + 'log_entries' not in cur_collections:
log_limit = settings.app.log_entry_limit
database.create_collection(prefix + 'log_entries', capped=True,
size=log_limit * 256 * 2, max=log_limit)
mongo.collections.update({
'log_entries': getattr(database, prefix + 'log_entries'),
})
for collection_name, collection in mongo.collections.items():
collection.name_str = collection_name
settings.init()
mongo.collections['transaction'].ensure_index('lock_id', unique=True)
mongo.collections['transaction'].ensure_index([
('ttl_timestamp', pymongo.ASCENDING),
('state', pymongo.ASCENDING),
('priority', pymongo.DESCENDING),
])
mongo.collections['queue'].ensure_index('runner_id')
mongo.collections['queue'].ensure_index('ttl_timestamp')
mongo.collections['task'].ensure_index('type', unique=True)
mongo.collections['task'].ensure_index('ttl_timestamp')
mongo.collections['log_entries'].ensure_index([
('timestamp', pymongo.DESCENDING),
])
mongo.collections['messages'].ensure_index('channel')
mongo.collections['administrators'].ensure_index('username', unique=True)
mongo.collections['users'].ensure_index([
('type', pymongo.ASCENDING),
('org_id', pymongo.ASCENDING),
])
mongo.collections['users'].ensure_index([
('org_id', pymongo.ASCENDING),
('name', pymongo.ASCENDING),
])
mongo.collections['users_key_link'].ensure_index('key_id')
mongo.collections['users_key_link'].ensure_index('short_id', unique=True)
mongo.collections['organizations'].ensure_index('type')
mongo.collections['hosts'].ensure_index('name')
mongo.collections['hosts_usage'].ensure_index([
('host_id', pymongo.ASCENDING),
('timestamp', pymongo.ASCENDING),
])
mongo.collections['servers'].ensure_index('name')
mongo.collections['servers'].ensure_index('ping_timestamp')
mongo.collections['servers_output'].ensure_index([
('server_id', pymongo.ASCENDING),
('timestamp', pymongo.ASCENDING),
])
mongo.collections['servers_bandwidth'].ensure_index([
('server_id', pymongo.ASCENDING),
('period', pymongo.ASCENDING),
('timestamp', pymongo.ASCENDING),
])
mongo.collections['servers_ip_pool'].ensure_index([
('server_id', pymongo.ASCENDING),
('user_id', pymongo.ASCENDING),
])
mongo.collections['servers_ip_pool'].ensure_index('user_id')
mongo.collections['dh_params'].ensure_index('dh_param_bits')
mongo.collections['auth_nonces'].ensure_index([
('token', pymongo.ASCENDING),
('nonce', pymongo.ASCENDING),
], unique=True)
mongo.collections['users_key_link'].ensure_index('timestamp',
expireAfterSeconds=settings.app.key_link_timeout)
mongo.collections['auth_nonces'].ensure_index('timestamp',
expireAfterSeconds=settings.app.auth_time_window * 2.1)
mongo.collections['auth_limiter'].ensure_index('timestamp',
expireAfterSeconds=settings.app.auth_limiter_ttl)
mongo.collections['otp'].ensure_index('timestamp',
expireAfterSeconds=120)
mongo.collections['otp_cache'].ensure_index('timestamp',
expireAfterSeconds=settings.user.otp_cache_ttl)
if not auth.Administrator.collection.find_one():
auth.Administrator(
username=DEFAULT_USERNAME,
password=DEFAULT_PASSWORD,
default=True,
).commit()
secret_key = settings.app.cookie_secret
if not secret_key:
secret_key = re.sub(r'[\W_]+', '',
base64.b64encode(os.urandom(128)))[:64]
settings.app.cookie_secret = secret_key
settings.commit()
app.app.secret_key = secret_key.encode()
server_api_key = settings.app.server_api_key
if not server_api_key:
server_api_key = re.sub(r'[\W_]+', '',
base64.b64encode(os.urandom(128)))[:64]
settings.app.server_api_key = server_api_key
settings.commit()
def __init__(self, mongo_address, doc_managers=None, **kwargs):
super(Connector, self).__init__()
# can_run is set to false when we join the thread
self.can_run = True
# main address - either mongos for sharded setups or a primary otherwise
self.address = mongo_address
# List of DocManager instances
if doc_managers:
self.doc_managers = doc_managers
else:
LOG.warning('No doc managers specified, using simulator.')
self.doc_managers = (simulator.DocManager(),)
# Warning when pymongo does not have C extensions regarding possible "Date out of range" errors
if not pymongo.has_c():
LOG.warning('pymongo was installed without C extensions. Possible "Date out of range" errors may occur.')
# Password for authentication
self.auth_key = kwargs.pop('auth_key', None)
# Username for authentication
self.auth_username = kwargs.pop('auth_username', None)
# The name of the file that stores the progress of the OplogThreads
self.oplog_checkpoint = kwargs.pop('oplog_checkpoint',
'oplog.timestamp')
# The set of OplogThreads created
self.shard_set = {}
# Dict of OplogThread/timestamp pairs to record progress
self.oplog_progress = LockingDict()
# Timezone awareness
self.tz_aware = kwargs.get('tz_aware', False)
# SSL keyword arguments to MongoClient.
ssl_certfile = kwargs.pop('ssl_certfile', None)
ssl_ca_certs = kwargs.pop('ssl_ca_certs', None)
ssl_keyfile = kwargs.pop('ssl_keyfile', None)
ssl_cert_reqs = kwargs.pop('ssl_cert_reqs', None)
self.ssl_kwargs = {}
if ssl_certfile:
self.ssl_kwargs['ssl_certfile'] = ssl_certfile
if ssl_ca_certs:
self.ssl_kwargs['ssl_ca_certs'] = ssl_ca_certs
if ssl_keyfile:
self.ssl_kwargs['ssl_keyfile'] = ssl_keyfile
if ssl_cert_reqs:
self.ssl_kwargs['ssl_cert_reqs'] = ssl_cert_reqs
# Save the rest of kwargs.
self.kwargs = kwargs
# Initialize and set the command helper
command_helper = CommandHelper(kwargs.get('ns_set', []),
kwargs.get('dest_mapping', {}))
for dm in self.doc_managers:
dm.command_helper = command_helper
if self.oplog_checkpoint is not None:
if not os.path.exists(self.oplog_checkpoint):
info_str = ("MongoConnector: Can't find %s, "
"attempting to create an empty progress log" %
self.oplog_checkpoint)
LOG.warning(info_str)
try:
# Create oplog progress file
open(self.oplog_checkpoint, "w").close()
except IOError as e:
LOG.critical("MongoConnector: Could not "
"create a progress log: %s" %
str(e))
sys.exit(2)
else:
if (not os.access(self.oplog_checkpoint, os.W_OK)
and not os.access(self.oplog_checkpoint, os.R_OK)):
LOG.critical("Invalid permissions on %s! Exiting" %
(self.oplog_checkpoint))
sys.exit(2)
# hackery to make sure to use the bson parser without the C extensions
# since those crash on some of the input.
# https://jira.mongodb.org/browse/PYTHON-571
#a="/usr/lib/python2.7/site-packages/pymongo-2.6_-py2.7.egg/"
import sys
a = "/usr/local/lib/python2.7/dist-packages/pymongo-2.6_-py2.7.egg"
sys.path.insert(0, a)
import pymongo
assert not pymongo.has_c(), "you must compile pymongo with --no-ext"
import bson
import datetime
import collections
import mmap, os, struct, pdb, json, time, sys
schema = json.load(open('schema'))
def run(filename):
fd = open(filename, "r")
# we used mmap before, but converted to stdio while chasing down the
# bson parser bug. It's probably safe to change it back. The slowdown
# wasn't that bad though, so I didn't bother.
mm = 0 # mmap.mmap(fd,0,access=mmap.ACCESS_READ)
# pdb.set_trace()
scan(filename, fd, mm)
fd.close()
print('done', dt(), filename)
t0 = time.time()
# Copyright 2009-2010 10gen, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Fail if the C extension module doesn't exist.
Only really intended to be used by internal build scripts.
"""
import sys
sys.path[0:0] = [""]
import pymongo
if not pymongo.has_c():
sys.exit("could not import _cbson")
# limitations under the License.
"""MongoDB benchmarking suite."""
import argparse
import time
import threading
import math
import sys
from tornado import gen
from tornado.ioloop import IOLoop
import pymongo
import bson
assert pymongo.has_c()
assert bson._use_c
def parse_args():
parser = argparse.ArgumentParser(description='Benchmark a script')
parser.add_argument(
'load', type=int, help='Number of requests to begin per second')
return parser.parse_args()
def async_trial(log, c, fn, load, duration, warmup):
# An object so it can be modified from inner functions
class State(object):
pass
st = State()