Example #1
0
def check_mongo_python_bindings(hostname, username, trd_party_dir) :
    '''
    TBD
    '''
    try:
        _status = 100
        _fmsg = "An error has occurred, but no error message was captured"
        
        _msg = "Checking MongoDB python bindings version....."
        import pymongo

        if pymongo.has_c() is False:
            msg = "WARNING: You do not have the pymongo C extensions " + \
                    "installed. Mongodb performance will be extermely slow. " + \
                     "To resolve this, please make sure you have the " + \
                     "'python-dev' or 'python-devel' development " + \
                     "headers installed and then *remove* and *reinstall* " + \
                     "the mongodb python bindings."
            print(msg)
        _version = str(pymongo.version).strip().replace('+','')
        del pymongo

        _msg += compare_versions('2.1.1', _version)
        _status = 0
        
    except ImportError, e:
        _status = 7282
def _show_config_mongoengine(app):
    #print "----------------MONGODB_SETTINGS-----------------------"
    try:
        from mongoengine.connection import _connection_settings, DEFAULT_CONNECTION_NAME
        from mongoengine import get_version as mongoengine_version
        import pymongo
        if 'MONGODB_SETTINGS' in app.config:
            print "---------------------------------"
            print "Your MONGODB_SETTINGS :"
            print "---------------------------------"        
            for key, val in app.config['MONGODB_SETTINGS'].items():
                print "%s = %s" % (key, val)
            
            print "---------------------------------"
            print "Reals Settings in mongoengine :"
            print "---------------------------------"        
            for key, val in _connection_settings.get(DEFAULT_CONNECTION_NAME).items():
                print "%s = %s" % (key, val)
                
        print "-------------------------------------------------------"
        print "Pymongo (has_c:%s).... : %s" % (pymongo.has_c(), pymongo.version)
        print "Mongoengine .......... : %s" % mongoengine_version()
        #print "-------------------------------------------------------"
                    
    except ImportError:
        print "!!! PAS DE MONGODB_SETTINGS !!!"
Example #3
0
def init(fast_db_url, big_db_url):
    """Initialize FDB/BDB factory singletons.

    As a useful side effect, creates all (missing) indices.

    @type fast_db_url: basestring
    @type big_db_url: basestring

    @raises MongoDBInitializationException: if cannot complete connection.
    """
    global FDB, BDB
    assert FDB is None and BDB is None, (FDB, BDB)

    try:
        FDB = DocStoreWrapperFactory(fast_db_url)
        BDB = GFSEnabledDocStoreWrapperFactory(big_db_url)
    except MongoDBInitializationException:
        raise

    if not bson.has_c():
        logger.warning('python-bson-ext is not installed, '
                       'performance may be lower than expected!')
    if not pymongo.has_c():
        logger.warning('python-pymongo-ext is not installed, '
                       'performance may be lower than expected!')

    logger.debug('Creating indices on Fast DataBase')
    with FDB() as fdbw:
        __make_indices(fdbqueries.FDB_INDICES_PER_COLLECTION.iteritems(),
                       dsw=fdbw)
    logger.debug('Creating indices on Big DataBase')
    with BDB() as bdbw:
        __make_indices(bdbqueries.GFS_INDICES_PER_COLLECTION.iteritems(),
                       dsw=bdbw)
Example #4
0
def init(fast_db_url, big_db_url):
    """Initialize FDB/BDB factory singletons.

    As a useful side effect, creates all (missing) indices.

    @type fast_db_url: basestring
    @type big_db_url: basestring

    @raises MongoDBInitializationException: if cannot complete connection.
    """
    global FDB, BDB
    assert FDB is None and BDB is None, (FDB, BDB)

    try:
        FDB = DocStoreWrapperFactory(fast_db_url)
        BDB = GFSEnabledDocStoreWrapperFactory(big_db_url)
    except MongoDBInitializationException:
        raise

    if not bson.has_c():
        logger.warning('python-bson-ext is not installed, '
                           'performance may be lower than expected!')
    if not pymongo.has_c():
        logger.warning('python-pymongo-ext is not installed, '
                           'performance may be lower than expected!')

    logger.debug('Creating indices on Fast DataBase')
    with FDB() as fdbw:
        __make_indices(fdbqueries.FDB_INDICES_PER_COLLECTION.iteritems(),
                       dsw=fdbw)
    logger.debug('Creating indices on Big DataBase')
    with BDB() as bdbw:
        __make_indices(bdbqueries.GFS_INDICES_PER_COLLECTION.iteritems(),
                       dsw=bdbw)
Example #5
0
def _show_config_mongoengine(app):
    #print "----------------MONGODB_SETTINGS-----------------------"
    try:
        from mongoengine.connection import _connection_settings, DEFAULT_CONNECTION_NAME
        from mongoengine import get_version as mongoengine_version
        import pymongo
        if 'MONGODB_SETTINGS' in app.config:
            print "---------------------------------"
            print "Your MONGODB_SETTINGS :"
            print "---------------------------------"
            for key, val in app.config['MONGODB_SETTINGS'].items():
                print "%s = %s" % (key, val)

            print "---------------------------------"
            print "Reals Settings in mongoengine :"
            print "---------------------------------"
            for key, val in _connection_settings.get(
                    DEFAULT_CONNECTION_NAME).items():
                print "%s = %s" % (key, val)

        print "-------------------------------------------------------"
        print "Pymongo (has_c:%s).... : %s" % (pymongo.has_c(),
                                               pymongo.version)
        print "Mongoengine .......... : %s" % mongoengine_version()
        #print "-------------------------------------------------------"

    except ImportError:
        print "!!! PAS DE MONGODB_SETTINGS !!!"
Example #6
0
def check_mongo_python_bindings(hostname, username, trd_party_dir):
    '''
    TBD
    '''
    try:
        _status = 100
        _fmsg = "An error has occurred, but no error message was captured"

        _msg = "Checking MongoDB python bindings version....."
        import pymongo

        if pymongo.has_c() is False:
            msg = "WARNING: You do not have the pymongo C extensions " + \
                    "installed. Mongodb performance will be extermely slow. " + \
                     "To resolve this, please make sure you have the " + \
                     "'python-dev' or 'python-devel' development " + \
                     "headers installed and then *remove* and *reinstall* " + \
                     "the mongodb python bindings."
            print(msg)
        _version = str(pymongo.version).strip().replace('+', '')
        del pymongo

        _msg += compare_versions('2.1.1', _version)
        _status = 0

    except ImportError, e:
        _status = 7282
Example #7
0
def setup_app():
    if not pymongo.has_c():
        logger.warning('Failed to load pymongo c bindings')

    if not bson.has_c():
        logger.warning('Failed to load bson c bindings')

    if settings.conf.debug and settings.conf.ssl:
        settings.conf.ssl = False
Example #8
0
def setup_app():
    if not pymongo.has_c():
        logger.warning('Failed to load pymongo c bindings')

    if not bson.has_c():
        logger.warning('Failed to load bson c bindings')

    if settings.conf.debug and settings.conf.ssl:
        settings.conf.ssl = False
Example #9
0
 def test_dates(self):
     doc = {"early": datetime.datetime(1686, 5, 5), "late": datetime.datetime(2086, 5, 5)}
     try:
         self.assertEqual(doc, BSON.from_dict(doc).to_dict())
     except ValueError:
         # Ignore ValueError when no C ext, since it's probably
         # a problem w/ 32-bit Python - we work around this in the
         # C ext, though.
         if pymongo.has_c():
             raise
 def __init__(self, parameters) :
     '''
     TBD
     '''
     set_my_parameters(self, parameters)
     self.pid = "TEST_" + getpwuid(os.getuid())[0]
     self.mongodb_conn = False
     if pymongo.has_c() is False:
         msg = "WARNING: You do not have the pymongo C extensions installed. Data retrieval performance will be slow"
         cberr(msg)
         print(msg)
 def __init__(self, parameters):
     '''
     TBD
     '''
     set_my_parameters(self, parameters)
     self.pid = "TEST_" + getpwuid(os.getuid())[0]
     self.mongodb_conn = False
     if pymongo.has_c() is False:
         msg = "WARNING: You do not have the pymongo C extensions installed. Data retrieval performance will be slow"
         cberr(msg)
         print(msg)
    def __init__(self, parameters) :
        MetricStoreMgdConn.__init__(self, parameters)
        self.username = str(self.mongodb_username)
        self.port = self.mongodb_port
        self.mongodb_conn = False
        
        if pymongo.has_c() is False:
            msg = "WARNING: You do not have the pymongo C extensions installed. Data retrieval performance will be slow"
            cberr(msg)
            print(msg)

        self.version = pymongo.version.split('.')[0]
Example #13
0
    def __init__(self, host='localhost'):
        """Connect to MongoDB database, get logaar collections
        Create and populate them if needed"""
        if not has_c():
            log.warning("Pymongo C module not available. Consider installing it to increase performances.")

        c = Connection()
        self._connection = c

        if 'logaar' not in c.database_names():
            log.info("Creating logaar database")
        db = c.logaar

        if 'logs' not in db.collection_names():
            log.info("Creating collection 'logs'")
        self.logs = db.logs

        # Create index if needed
        db.logs.ensure_index(
            [
                ('date', 1),
                ('level', 1),
                ('message', 1),
                ('program', 1),
                ('pid', 1),
            ],
            unique = True,
            dropDups = True,
        )


        if 'incoming' not in db.collection_names():
            log.info("Creating collection 'incoming'")
            db.create_collection("incoming", capped=True,size="100000")
        self.incoming = db.incoming

        # Create index if needed
        db.incoming.ensure_index(
            [
                ("date", DESCENDING)
            ]
        )

        if 'rules' not in db.collection_names():
            log.info("Creating collection 'rules'")
            #TODO: rewrite this using proper collection dump/restore
            import setup_rules
            for rulevals in setup_rules.rules_tuple:
                d = dict(zip(setup_rules.keys, rulevals))
                db.rules.insert(d)
        self.rules = db.rules
        log.info('connected')
Example #14
0
def log_startup_info():
    """Log info about the current environment."""
    LOG.always('Starting mongo-connector version: %s', __version__)
    if 'dev' in __version__:
        LOG.warning('This is a development version (%s) of mongo-connector',
                    __version__)
    LOG.always('Python version: %s', sys.version)
    LOG.always('Platform: %s', platform.platform())
    LOG.always('pymongo version: %s', pymongo.__version__)
    if not pymongo.has_c():
        LOG.warning(
            'pymongo version %s was installed without the C extensions. '
            '"InvalidBSON: Date value out of range" errors may occur if '
            'there are documents with BSON Datetimes that represent times '
            'outside of Python\'s datetime limit.', pymongo.__version__)
Example #15
0
def log_startup_info():
    """Log info about the current environment."""
    LOG.always('Starting mongo-connector version: %s', __version__)
    if 'dev' in __version__:
        LOG.warning('This is a development version (%s) of mongo-connector',
                    __version__)
    LOG.always('Python version: %s', sys.version)
    LOG.always('Platform: %s', platform.platform())
    LOG.always('pymongo version: %s', pymongo.__version__)
    if not pymongo.has_c():
        LOG.warning(
            'pymongo version %s was installed without the C extensions. '
            '"InvalidBSON: Date value out of range" errors may occur if '
            'there are documents with BSON Datetimes that represent times '
            'outside of Python\'s datetime limit.', pymongo.__version__)
Example #16
0
def removenodedataolderthan(collection, period, test):
    print('removenodedataolderthan starting for', period,
          'hours. C extentions in use:', pymongo.has_c())
    empty_results = {'results': '0'}
    start = getstart(period)
    total_records = 0

    qry = {'time': {'$lte': start}}
    print('query is %s ' % qry)

    if test == 'true':
        print('starting TEST query at ', dt.datetime.now())
        cursor = collection.find(qry).batch_size(1000)
        print('cursor returned at ', dt.datetime.now())  #this takes .2 millis
        for row in cursor:
            total_records = total_records + 1
        print('total_records is ', total_records, ' at ', dt.datetime.now())
        return total_records
    elif test == 'false':
        print('starting REMOVE query at ', dt.datetime.now())
        results = collection.remove(qry)
        print('REMOVE query results ', results)
        print('REMOVE query finished at ', dt.datetime.now())
Example #17
0
def setup_mongo():
    if not pymongo.has_c():
        logger.warning('Failed to load pymongo c bindings')

    if not bson.has_c():
        logger.warning('Failed to load bson c bindings')

    prefix = settings.conf.mongodb_collection_prefix or ''
    last_error = time.time() - 24
    while True:
        try:
            client = pymongo.MongoClient(settings.conf.mongodb_url,
                                         connectTimeoutMS=2000)
            break
        except pymongo.errors.ConnectionFailure:
            time.sleep(0.5)
            if time.time() - last_error > 30:
                last_error = time.time()
                logger.exception('Error connecting to mongodb server')

    database = client.get_default_database()
    cur_collections = database.collection_names()

    if prefix + 'messages' not in cur_collections:
        database.create_collection(prefix + 'messages',
                                   capped=True,
                                   size=100000)

    mongo.collections.update({
        'transaction':
        getattr(database, prefix + 'transaction'),
        'queue':
        getattr(database, prefix + 'queue'),
        'task':
        getattr(database, prefix + 'task'),
        'system':
        getattr(database, prefix + 'system'),
        'messages':
        getattr(database, prefix + 'messages'),
        'administrators':
        getattr(database, prefix + 'administrators'),
        'users':
        getattr(database, prefix + 'users'),
        'users_key_link':
        getattr(database, prefix + 'users_key_link'),
        'organizations':
        getattr(database, prefix + 'organizations'),
        'hosts':
        getattr(database, prefix + 'hosts'),
        'hosts_usage':
        getattr(database, prefix + 'hosts_usage'),
        'servers':
        getattr(database, prefix + 'servers'),
        'servers_output':
        getattr(database, prefix + 'servers_output'),
        'servers_bandwidth':
        getattr(database, prefix + 'servers_bandwidth'),
        'servers_ip_pool':
        getattr(database, prefix + 'servers_ip_pool'),
        'dh_params':
        getattr(database, prefix + 'dh_params'),
        'auth_nonces':
        getattr(database, prefix + 'auth_nonces'),
        'auth_limiter':
        getattr(database, prefix + 'auth_limiter'),
        'otp':
        getattr(database, prefix + 'otp'),
        'otp_cache':
        getattr(database, prefix + 'otp_cache'),
    })

    if prefix + 'log_entries' not in cur_collections:
        log_limit = settings.app.log_entry_limit
        database.create_collection(prefix + 'log_entries',
                                   capped=True,
                                   size=log_limit * 256 * 2,
                                   max=log_limit)

    mongo.collections.update({
        'log_entries':
        getattr(database, prefix + 'log_entries'),
    })

    for collection_name, collection in mongo.collections.items():
        collection.name_str = collection_name

    settings.init()

    mongo.collections['transaction'].ensure_index('lock_id', unique=True)
    mongo.collections['transaction'].ensure_index([
        ('ttl_timestamp', pymongo.ASCENDING),
        ('state', pymongo.ASCENDING),
        ('priority', pymongo.DESCENDING),
    ])
    mongo.collections['queue'].ensure_index('runner_id')
    mongo.collections['queue'].ensure_index('ttl_timestamp')
    mongo.collections['task'].ensure_index('type', unique=True)
    mongo.collections['task'].ensure_index('ttl_timestamp')
    mongo.collections['log_entries'].ensure_index([
        ('timestamp', pymongo.DESCENDING),
    ])
    mongo.collections['messages'].ensure_index('channel')
    mongo.collections['administrators'].ensure_index('username', unique=True)
    mongo.collections['users'].ensure_index([
        ('type', pymongo.ASCENDING),
        ('org_id', pymongo.ASCENDING),
    ])
    mongo.collections['users'].ensure_index([
        ('org_id', pymongo.ASCENDING),
        ('name', pymongo.ASCENDING),
    ])
    mongo.collections['users_key_link'].ensure_index('key_id')
    mongo.collections['users_key_link'].ensure_index('short_id', unique=True)
    mongo.collections['organizations'].ensure_index('type')
    mongo.collections['hosts'].ensure_index('name')
    mongo.collections['hosts_usage'].ensure_index([
        ('host_id', pymongo.ASCENDING),
        ('timestamp', pymongo.ASCENDING),
    ])
    mongo.collections['servers'].ensure_index('name')
    mongo.collections['servers'].ensure_index('ping_timestamp')
    mongo.collections['servers_output'].ensure_index([
        ('server_id', pymongo.ASCENDING),
        ('timestamp', pymongo.ASCENDING),
    ])
    mongo.collections['servers_bandwidth'].ensure_index([
        ('server_id', pymongo.ASCENDING),
        ('period', pymongo.ASCENDING),
        ('timestamp', pymongo.ASCENDING),
    ])
    mongo.collections['servers_ip_pool'].ensure_index([
        ('server_id', pymongo.ASCENDING),
        ('user_id', pymongo.ASCENDING),
    ])
    mongo.collections['servers_ip_pool'].ensure_index('user_id')
    mongo.collections['dh_params'].ensure_index('dh_param_bits')
    mongo.collections['auth_nonces'].ensure_index([
        ('token', pymongo.ASCENDING),
        ('nonce', pymongo.ASCENDING),
    ],
                                                  unique=True)

    mongo.collections['users_key_link'].ensure_index(
        'timestamp', expireAfterSeconds=settings.app.key_link_timeout)
    mongo.collections['auth_nonces'].ensure_index(
        'timestamp', expireAfterSeconds=settings.app.auth_time_window * 2.1)
    mongo.collections['auth_limiter'].ensure_index(
        'timestamp', expireAfterSeconds=settings.app.auth_limiter_ttl)
    mongo.collections['otp'].ensure_index('timestamp', expireAfterSeconds=120)
    mongo.collections['otp_cache'].ensure_index(
        'timestamp', expireAfterSeconds=settings.user.otp_cache_ttl)

    if not auth.Administrator.collection.find_one():
        auth.Administrator(
            username=DEFAULT_USERNAME,
            password=DEFAULT_PASSWORD,
            default=True,
        ).commit()

    secret_key = settings.app.cookie_secret
    if not secret_key:
        secret_key = re.sub(r'[\W_]+', '',
                            base64.b64encode(os.urandom(128)))[:64]
        settings.app.cookie_secret = secret_key
        settings.commit()
    app.app.secret_key = secret_key.encode()

    server_api_key = settings.app.server_api_key
    if not server_api_key:
        server_api_key = re.sub(r'[\W_]+', '',
                                base64.b64encode(os.urandom(128)))[:64]
        settings.app.server_api_key = server_api_key
        settings.commit()
from confPull import ConfPullThread
import logConfig

# Python
import os, sys, platform, time, threading, socket, traceback, random, hashlib, tempfile

socket.setdefaulttimeout( _settings.socket_timeout )

_agentVersion = "1.6.8"

_pymongoVersion = pymongo.version

_pymongoHasC = False

try:
    _pymongoHasC = pymongo.has_c()
except:
    pass

# Try and reduce the stack size.
try:
    threading.stack_size(409600)
except:
    pass

class AgentProcess( threading.Thread ):
    """ The parent process - monitors agent process and checks for updates etc """

    def __init__( self, loggerObj, agentDir, existingSessionKey ):
        """ Construct the object """
        self.logger = loggerObj
Example #19
0
# hackery to make sure to use the bson parser without the C extensions
# since those crash on some of the input.  
#    https://jira.mongodb.org/browse/PYTHON-571
#a="/usr/lib/python2.7/site-packages/pymongo-2.6_-py2.7.egg/"
import sys
a="/usr/local/lib/python2.7/dist-packages/pymongo-2.6_-py2.7.egg"
sys.path.insert(0,a)
import pymongo
assert not pymongo.has_c(), "you must compile pymongo with --no-ext"
import bson
import datetime
import collections

import mmap,os,struct,pdb,json,time,sys

schema = json.load(open('schema'))

def run(filename):
    fd=open(filename,"r")
    # we used mmap before, but converted to stdio while chasing down the
    # bson parser bug.  It's probably safe to change it back.  The slowdown
    # wasn't that bad though, so I didn't bother.
    mm = 0# mmap.mmap(fd,0,access=mmap.ACCESS_READ)
    # pdb.set_trace()
    scan(filename,fd,mm)
    fd.close()
    print ('done',dt(),filename)

t0 = time.time()
def dt(): return time.time()-t0
Example #20
0
    def __init__(self, mongo_address, doc_managers=None, **kwargs):
        super(Connector, self).__init__()

        # can_run is set to false when we join the thread
        self.can_run = True

        # main address - either mongos for sharded setups or a primary otherwise
        self.address = mongo_address

        # connection to the main address
        self.main_conn = None

        # List of DocManager instances
        if doc_managers:
            self.doc_managers = doc_managers
        else:
            LOG.warning('No doc managers specified, using simulator.')
            self.doc_managers = (simulator.DocManager(),)

        if not pymongo.has_c():
            warning = ('pymongo version %s was installed without the C '
                       'extensions. "InvalidBSON: Date value out of '
                       'range" errors may occur if there are documents '
                       'with BSON Datetimes that represent times outside of '
                       'Python\'s datetime.datetime limit.') % (
                pymongo.__version__,)
            # Print and warn to make it extra noticeable
            print(warning)
            LOG.warning(warning)

        # Password for authentication
        self.auth_key = kwargs.pop('auth_key', None)

        # Username for authentication
        self.auth_username = kwargs.pop('auth_username', None)

        # The name of the file that stores the progress of the OplogThreads
        self.oplog_checkpoint = kwargs.pop('oplog_checkpoint',
                                           'oplog.timestamp')

        # The set of OplogThreads created
        self.shard_set = {}

        # Dict of OplogThread/timestamp pairs to record progress
        self.oplog_progress = LockingDict()

        # Timezone awareness
        self.tz_aware = kwargs.get('tz_aware', False)

        # SSL keyword arguments to MongoClient.
        ssl_certfile = kwargs.pop('ssl_certfile', None)
        ssl_ca_certs = kwargs.pop('ssl_ca_certs', None)
        ssl_keyfile = kwargs.pop('ssl_keyfile', None)
        ssl_cert_reqs = kwargs.pop('ssl_cert_reqs', None)
        self.ssl_kwargs = {}
        if ssl_certfile:
            self.ssl_kwargs['ssl_certfile'] = ssl_certfile
        if ssl_ca_certs:
            self.ssl_kwargs['ssl_ca_certs'] = ssl_ca_certs
        if ssl_keyfile:
            self.ssl_kwargs['ssl_keyfile'] = ssl_keyfile
        if ssl_cert_reqs:
            self.ssl_kwargs['ssl_cert_reqs'] = ssl_cert_reqs

        # Save the rest of kwargs.
        self.kwargs = kwargs

        # Replace the origin dest_mapping
        self.dest_mapping = DestMapping(kwargs.get('ns_set', []),
                                        kwargs.get('ex_ns_set', []),
                                        kwargs.get('dest_mapping', {}))

        # Initialize and set the command helper
        command_helper = CommandHelper(self.dest_mapping)
        for dm in self.doc_managers:
            dm.command_helper = command_helper

        if self.oplog_checkpoint is not None:
            if not os.path.exists(self.oplog_checkpoint):
                info_str = ("MongoConnector: Can't find %s, "
                            "attempting to create an empty progress log" %
                            self.oplog_checkpoint)
                LOG.warning(info_str)
                try:
                    # Create oplog progress file
                    open(self.oplog_checkpoint, "w").close()
                except IOError as e:
                    LOG.critical("MongoConnector: Could not "
                                 "create a progress log: %s" %
                                 str(e))
                    sys.exit(2)
            else:
                if (not os.access(self.oplog_checkpoint, os.W_OK)
                        and not os.access(self.oplog_checkpoint, os.R_OK)):
                    LOG.critical("Invalid permissions on %s! Exiting" %
                                 (self.oplog_checkpoint))
                    sys.exit(2)
# Copyright 2009-2014 MongoDB, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Fail if the C extension module doesn't exist.

Only really intended to be used by internal build scripts.
"""

import sys
sys.path[0:0] = [""]

import bson
import pymongo

if not pymongo.has_c() or not bson.has_c():
    sys.exit("could not load C extensions")
Example #22
0
def setup_check():
    if not pymongo.has_c():
        logger.warning('Failed to load pymongo c bindings')

    if not bson.has_c():
        logger.warning('Failed to load bson c bindings')
Example #23
0
class PopulateDatabase:

    baseUrl = "https://www.imdb.com"
    headers = {"Accept-Language": "en-US"}
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.INFO)
    logHandler = logging.FileHandler("Logs/populate_database.log")
    logHandler.setFormatter(
        logging.Formatter(
            "%(asctime)s (%(filename)s)/(%(funcName)s): %(message)s"))
    logger.addHandler(logHandler)

    logger.info("-----------NEW SESSION-----------")

    pymongo.has_c() == True

    def __init__(self, client):
        self.db = client.sugsn

    def parseToMovieItem(self, object):
        primaryInfo = object.findNext("h3", class_="lister-item-header")
        secondaryInfo = object.findNext("p", class_="text-muted")
        ratingsInfo = object.findNext("div", class_="ratings-bar")
        posterData = object.findNext(
            "div", class_="lister-item-image float-left").findNext(
                "a").findNext("img")

        return MovieItem(
            primaryInfo.findNext(
                "span", class_="lister-item-index unbold text-primary").text,
            posterData["data-tconst"],
            primaryInfo.findNext("a").text,
            primaryInfo.findNext(
                "span", class_="lister-item-year text-muted unbold").text,
            secondaryInfo.findNext("span", class_="certificate").text,
            secondaryInfo.findNext("span", class_="runtime").text,
            secondaryInfo.findNext("span", class_="genre").text,
            ratingsInfo.findNext("strong").text,
            ratingsInfo.findNext(
                "div",
                class_="inline-block ratings-metascore").findNext("span").text,
            ratingsInfo.findNext("p", class_="text-muted").text,
            posterData["loadlate"])

    # Imdb Parser

    def getTopRatedMovies(self):
        html = requests.get(
            self.baseUrl +
            "/search/title/?sort=user_rating&title_type=feature&num_votes=250000,",
            headers=self.headers)
        soup = BeautifulSoup(html.content, "lxml")
        nextTag = soup.findAll("a", class_="lister-page-next next-page")

        # Traverse pages
        while (len(nextTag) != 0):
            # Populate database
            resultList = soup.find_all("div",
                                       class_="lister-item mode-advanced")

            for item in resultList:
                movie = self.parseToMovieItem(item)
                self.logger.info("Adding " + str(movie.listNum) +
                                 " - %s" % movie.name)
                self.db.topRatedMovies.update_one({"listNum": movie.listNum},
                                                  {"$set": movie.__dict__},
                                                  True)

            # Load next page
            html = requests.get(self.baseUrl + nextTag[0]["href"],
                                headers=self.headers)
            soup = BeautifulSoup(html.content, "lxml")
            nextTag = soup.findAll("a", class_="lister-page-next next-page")
Example #24
0
# @proyect inKoutPi
# history dao
__author__ = 'rbioque'

import pymongo
pymongo.has_c()
from . import measure


class HistoryDao:
    def __init__(self):
        # Making connection with MongoGlient
        from pymongo import MongoClient
        client = MongoClient('localhost', 27017)
        self.db = client.inKoutPiDB

    def persist(self, measure):
        self.db.HISTORY.insert(measure.toDBCollection())

    def findLastMeasure(self):
        measure = self.db.HISTORY.find().sort("date", -1).limit(1)
        return measure

    def findLastsMeasure(self, rows):
        measures = self.db.HISTORY.find().sort("date", -1).limit(rows)
        return measures

    def findLastsOnlyMeasure(self, rows):
        measures = self.db.HISTORY.find({
            "msg": {
                "$exists": False
Example #25
0
# h t t p s : / / github .com /mkleehammer/pyodbc/wiki/Getting-started
servername = 'HIPERDRIVE2\MSSQLSERVER01'
port = '1433'
database = 'pymssql'
username = '******'
password = '******'
driver = '{ODBC Driver 17 for SQL Server}'
connSql = pyodbc.connect(';DRIVER=' + driver + ';SERVER=' + servername +
                         ';PORT=' + port + ';DATABASE=' + database + ';UID=' +
                         username + ';PWD=' + password +
                         ';Trusted_Connection=yes')
cursorSql = connSql.cursor()

print(pymongo.version)
print(pymongo.has_c())
connMongo = pymongo.MongoClient(
    "mongodb://localhost:27017")  # connect to the db on standard port
print(connMongo)
dbMongo = connMongo.pymongo  # attach to db m101
print(dbMongo)
collMongo = dbMongo.ColecaoUm  # specify the collection funnynumbers
print(collMongo)

cursorSql.execute("""
SELECT * FROM dbo.Table_1 WHERE name=?
""", 'pateta')
rowSql = cursorSql.fetchone()
descriptionSql = cursorSql.description
print(descriptionSql)
columnNamesSQL = [column[0] for column in cursorSql.description
Example #26
0
def getdatausinggw(gw, node, start, mytype, timezone):
    print('getdatausinggw starting. C extentions in use:', pymongo.has_c())
    docs = []
    resultsarray = []
    empty_results = {'results': '0'}

    try:
        toZone = gettz(timezone)
        fromZone = tzutc()
    except ValueError as err:
        print('Invalid timezone parameter %s. Defaulting to 0' % tz)
        tz = 'None'

    qry = {'gateway_id': gw, 'node_id': str(node), 'time': {'$gte': start}}
    sortparam = [('time', 1)]
    if mytype:
        qry['type'] = mytype
    print('query is %s and sort is ' % qry, sortparam)
    print('starting query at ', dt.datetime.now())
    cursor = sensors.find(qry).sort(sortparam).batch_size(100000)
    print('query returned at ', dt.datetime.now())  #this takes .2 millis
    for row in cursor:
        resultsarray.append(row)
    print('Arrayified at ', dt.datetime.now())
    count = len(resultsarray)
    print('%i records returned' % count, dt.datetime.now())
    #this takes 1.86 sec
    ct = 0
    total = 0
    skip = 0
    if count == 0:
        return empty_results
    if count > 300:
        skip = int(count / 300 + .49)
        print('Since more than 300 records were returned, skip is set to %i' %
              skip, dt.datetime.now())  #this take .1 milli

    #insert initial doc as first 'goalpost' with time same as start
    newdoc = {'value': 0, 'human_time': '', 'time': 0}
    # datetimes in DB are utc, convert to local timezone
    newdoc['human_time'] = dt.datetime.fromtimestamp(start).replace(
        tzinfo=fromZone).astimezone(toZone).strftime(timefmt)
    newdoc['time'] = start
    newvalue = cleanvalue(resultsarray[skip + 1]['value'])
    newdoc['value'] = newvalue
    docs.append(newdoc)

    for doc in resultsarray:
        total += 1
        ct += 1
        newdoc = {'value': 0, 'human_time': '', 'time': 0}
        # skip if needed
        if ct > skip:
            #newdoc['value'] = float(doc['value'].replace('b', '').replace('v', '').replace("'", ""))
            newvalue = cleanvalue(doc['value'])
            newdoc['value'] = newvalue
            latestvalue = newvalue
            # datetimes in DB are utc, convert to local timezone
            newdoc['human_time'] = dt.datetime.fromtimestamp(
                doc['time']).replace(
                    tzinfo=fromZone).astimezone(toZone).strftime(timefmt)
            newdoc['time'] = doc['time']
            docs.append(newdoc)
            ct = 0
    if ct != 0:
        newdoc['value'] = float(doc['value'].replace('b', '').replace(
            'v', '').replace("'", ""))
        # datetimes in DB are utc, convert to local timezone
        newdoc['human_time'] = dt.datetime.fromtimestamp(doc['time']).replace(
            tzinfo=fromZone).astimezone(toZone).strftime(timefmt)
        newdoc['time'] = doc['time']
        docs.append(newdoc)

    #insert last doc as end 'goalpost' with timestamp of now
    newdoc = {'value': 0, 'human_time': '', 'time': 0}
    # datetimes in DB are utc, convert to local timezone
    newdoc['human_time'] = dt.datetime.timestamp(dt.datetime.now())
    newdoc['time'] = dt.datetime.timestamp(dt.datetime.now())
    newdoc['value'] = latestvalue
    docs.append(newdoc)

    print('total docs found:', total, ' and returning:', len(docs))
    print('document returned at ', dt.datetime.now())  #this takes 2.28 sec
    return docs
Example #27
0
import numpy as np
from bson import BSON, CodecOptions, Int64, ObjectId
from bson.raw_bson import RawBSONDocument

try:
    import bsonnumpy
except (ImportError, OSError) as exc:
    print(exc)
    bsonnumpy = None

try:
    import monary
except (ImportError, OSError) as exc:
    monary = None

assert pymongo.has_c()

# Use large document in tests? If SMALL, no, if LARGE, then yes.
SMALL = False
LARGE = True
db = None
raw_bson = None
large_doc_keys = None
collection_names = {LARGE: "large", SMALL: "small"}
dtypes = {}
raw_bsons = {}


def _setup():
    global db
    global raw_bson
Example #28
0
def setup_check():
    if not pymongo.has_c():
        logger.warning('Failed to load pymongo c bindings')

    if not bson.has_c():
        logger.warning('Failed to load bson c bindings')
Example #29
0
            del sys.modules[name]

    for module, attr in ((bson, '_cbson'), (pymongo, '_cmessage'),
                         (pymongo.message, '_cmessage')):
        try:
            delattr(module, attr)
        except AttributeError:
            pass

    del attr, name, module, needs_replacing

    importlib.reload(bson)
    importlib.reload(pymongo.message)

    assert not bson.has_c()
    assert not pymongo.has_c()


import types, functools
import bson, bson.son, contextlib, struct
from bson.binary import Binary, OLD_UUID_SUBTYPE
from gridfs import GridFS
from collections import OrderedDict
from bson import _elements_to_dict, DBRef
from pytransact import spickle

max_bson_element_size = 2 ** 18

Extension = spickle.ExtensionType()

def register(typeobj, code=spickle.ExtensionType._empty):
from confPull import ConfPullThread
import logConfig

# Python
import os, sys, platform, time, threading, socket, traceback, random, hashlib, tempfile

socket.setdefaulttimeout(_settings.socket_timeout)

_agentVersion = "1.5.7"

_pymongoVersion = pymongo.version

_pymongoHasC = False

try:
    _pymongoHasC = pymongo.has_c()
except:
    pass

# Try and reduce the stack size.
try:
    threading.stack_size(409600)
except:
    pass


class AgentProcess(threading.Thread):
    """ The parent process - monitors agent process and checks for updates etc """
    def __init__(self, loggerObj, agentDir, existingSessionKey):
        """ Construct the object """
        self.logger = loggerObj
Example #31
0
def setup_mongo():
    if not pymongo.has_c():
        logger.warning('Failed to load pymongo c bindings')

    if not bson.has_c():
        logger.warning('Failed to load bson c bindings')

    prefix = settings.conf.mongodb_collection_prefix or ''
    last_error = time.time() - 24
    while True:
        try:
            client = pymongo.MongoClient(settings.conf.mongodb_url,
                connectTimeoutMS=2000)
            break
        except pymongo.errors.ConnectionFailure:
            time.sleep(0.5)
            if time.time() - last_error > 30:
                last_error = time.time()
                logger.exception('Error connecting to mongodb server')

    database = client.get_default_database()
    cur_collections = database.collection_names()

    if prefix + 'messages' not in cur_collections:
        database.create_collection(prefix + 'messages', capped=True,
            size=100000)

    mongo.collections.update({
        'transaction': getattr(database, prefix + 'transaction'),
        'queue': getattr(database, prefix + 'queue'),
        'task': getattr(database, prefix + 'task'),
        'system': getattr(database, prefix + 'system'),
        'messages': getattr(database, prefix + 'messages'),
        'administrators': getattr(database, prefix + 'administrators'),
        'users': getattr(database, prefix + 'users'),
        'users_key_link': getattr(database, prefix + 'users_key_link'),
        'organizations': getattr(database, prefix + 'organizations'),
        'hosts': getattr(database, prefix + 'hosts'),
        'hosts_usage': getattr(database, prefix + 'hosts_usage'),
        'servers': getattr(database, prefix + 'servers'),
        'servers_output': getattr(database, prefix + 'servers_output'),
        'servers_bandwidth': getattr(database, prefix + 'servers_bandwidth'),
        'servers_ip_pool': getattr(database, prefix + 'servers_ip_pool'),
        'dh_params': getattr(database, prefix + 'dh_params'),
        'auth_nonces': getattr(database, prefix + 'auth_nonces'),
        'auth_limiter': getattr(database, prefix + 'auth_limiter'),
        'otp': getattr(database, prefix + 'otp'),
        'otp_cache': getattr(database, prefix + 'otp_cache'),
    })

    if prefix + 'log_entries' not in cur_collections:
        log_limit = settings.app.log_entry_limit
        database.create_collection(prefix + 'log_entries', capped=True,
            size=log_limit * 256 * 2, max=log_limit)

    mongo.collections.update({
        'log_entries': getattr(database, prefix + 'log_entries'),
    })

    for collection_name, collection in mongo.collections.items():
        collection.name_str = collection_name

    settings.init()

    mongo.collections['transaction'].ensure_index('lock_id', unique=True)
    mongo.collections['transaction'].ensure_index([
        ('ttl_timestamp', pymongo.ASCENDING),
        ('state', pymongo.ASCENDING),
        ('priority', pymongo.DESCENDING),
    ])
    mongo.collections['queue'].ensure_index('runner_id')
    mongo.collections['queue'].ensure_index('ttl_timestamp')
    mongo.collections['task'].ensure_index('type', unique=True)
    mongo.collections['task'].ensure_index('ttl_timestamp')
    mongo.collections['log_entries'].ensure_index([
        ('timestamp', pymongo.DESCENDING),
    ])
    mongo.collections['messages'].ensure_index('channel')
    mongo.collections['administrators'].ensure_index('username', unique=True)
    mongo.collections['users'].ensure_index([
        ('type', pymongo.ASCENDING),
        ('org_id', pymongo.ASCENDING),
    ])
    mongo.collections['users'].ensure_index([
        ('org_id', pymongo.ASCENDING),
        ('name', pymongo.ASCENDING),
    ])
    mongo.collections['users_key_link'].ensure_index('key_id')
    mongo.collections['users_key_link'].ensure_index('short_id', unique=True)
    mongo.collections['organizations'].ensure_index('type')
    mongo.collections['hosts'].ensure_index('name')
    mongo.collections['hosts_usage'].ensure_index([
        ('host_id', pymongo.ASCENDING),
        ('timestamp', pymongo.ASCENDING),
    ])
    mongo.collections['servers'].ensure_index('name')
    mongo.collections['servers'].ensure_index('ping_timestamp')
    mongo.collections['servers_output'].ensure_index([
        ('server_id', pymongo.ASCENDING),
        ('timestamp', pymongo.ASCENDING),
    ])
    mongo.collections['servers_bandwidth'].ensure_index([
        ('server_id', pymongo.ASCENDING),
        ('period', pymongo.ASCENDING),
        ('timestamp', pymongo.ASCENDING),
    ])
    mongo.collections['servers_ip_pool'].ensure_index([
        ('server_id', pymongo.ASCENDING),
        ('user_id', pymongo.ASCENDING),
    ])
    mongo.collections['servers_ip_pool'].ensure_index('user_id')
    mongo.collections['dh_params'].ensure_index('dh_param_bits')
    mongo.collections['auth_nonces'].ensure_index([
        ('token', pymongo.ASCENDING),
        ('nonce', pymongo.ASCENDING),
    ], unique=True)

    mongo.collections['users_key_link'].ensure_index('timestamp',
        expireAfterSeconds=settings.app.key_link_timeout)
    mongo.collections['auth_nonces'].ensure_index('timestamp',
        expireAfterSeconds=settings.app.auth_time_window * 2.1)
    mongo.collections['auth_limiter'].ensure_index('timestamp',
        expireAfterSeconds=settings.app.auth_limiter_ttl)
    mongo.collections['otp'].ensure_index('timestamp',
        expireAfterSeconds=120)
    mongo.collections['otp_cache'].ensure_index('timestamp',
        expireAfterSeconds=settings.user.otp_cache_ttl)

    if not auth.Administrator.collection.find_one():
        auth.Administrator(
            username=DEFAULT_USERNAME,
            password=DEFAULT_PASSWORD,
            default=True,
        ).commit()

    secret_key = settings.app.cookie_secret
    if not secret_key:
        secret_key = re.sub(r'[\W_]+', '',
            base64.b64encode(os.urandom(128)))[:64]
        settings.app.cookie_secret = secret_key
        settings.commit()
    app.app.secret_key = secret_key.encode()

    server_api_key = settings.app.server_api_key
    if not server_api_key:
        server_api_key = re.sub(r'[\W_]+', '',
            base64.b64encode(os.urandom(128)))[:64]
        settings.app.server_api_key = server_api_key
        settings.commit()
Example #32
0
    def __init__(self, mongo_address, doc_managers=None, **kwargs):
        super(Connector, self).__init__()

        # can_run is set to false when we join the thread
        self.can_run = True

        # main address - either mongos for sharded setups or a primary otherwise
        self.address = mongo_address

        # List of DocManager instances
        if doc_managers:
            self.doc_managers = doc_managers
        else:
            LOG.warning('No doc managers specified, using simulator.')
            self.doc_managers = (simulator.DocManager(),)

        # Warning when pymongo does not have C extensions regarding possible "Date out of range" errors
        if not pymongo.has_c():
            LOG.warning('pymongo was installed without C extensions. Possible "Date out of range" errors may occur.')

        # Password for authentication
        self.auth_key = kwargs.pop('auth_key', None)

        # Username for authentication
        self.auth_username = kwargs.pop('auth_username', None)

        # The name of the file that stores the progress of the OplogThreads
        self.oplog_checkpoint = kwargs.pop('oplog_checkpoint',
                                           'oplog.timestamp')

        # The set of OplogThreads created
        self.shard_set = {}

        # Dict of OplogThread/timestamp pairs to record progress
        self.oplog_progress = LockingDict()

        # Timezone awareness
        self.tz_aware = kwargs.get('tz_aware', False)

        # SSL keyword arguments to MongoClient.
        ssl_certfile = kwargs.pop('ssl_certfile', None)
        ssl_ca_certs = kwargs.pop('ssl_ca_certs', None)
        ssl_keyfile = kwargs.pop('ssl_keyfile', None)
        ssl_cert_reqs = kwargs.pop('ssl_cert_reqs', None)
        self.ssl_kwargs = {}
        if ssl_certfile:
            self.ssl_kwargs['ssl_certfile'] = ssl_certfile
        if ssl_ca_certs:
            self.ssl_kwargs['ssl_ca_certs'] = ssl_ca_certs
        if ssl_keyfile:
            self.ssl_kwargs['ssl_keyfile'] = ssl_keyfile
        if ssl_cert_reqs:
            self.ssl_kwargs['ssl_cert_reqs'] = ssl_cert_reqs

        # Save the rest of kwargs.
        self.kwargs = kwargs

        # Initialize and set the command helper
        command_helper = CommandHelper(kwargs.get('ns_set', []),
                                       kwargs.get('dest_mapping', {}))
        for dm in self.doc_managers:
            dm.command_helper = command_helper

        if self.oplog_checkpoint is not None:
            if not os.path.exists(self.oplog_checkpoint):
                info_str = ("MongoConnector: Can't find %s, "
                            "attempting to create an empty progress log" %
                            self.oplog_checkpoint)
                LOG.warning(info_str)
                try:
                    # Create oplog progress file
                    open(self.oplog_checkpoint, "w").close()
                except IOError as e:
                    LOG.critical("MongoConnector: Could not "
                                 "create a progress log: %s" %
                                 str(e))
                    sys.exit(2)
            else:
                if (not os.access(self.oplog_checkpoint, os.W_OK)
                        and not os.access(self.oplog_checkpoint, os.R_OK)):
                    LOG.critical("Invalid permissions on %s! Exiting" %
                                 (self.oplog_checkpoint))
                    sys.exit(2)
Example #33
0
# hackery to make sure to use the bson parser without the C extensions
# since those crash on some of the input.
#    https://jira.mongodb.org/browse/PYTHON-571
#a="/usr/lib/python2.7/site-packages/pymongo-2.6_-py2.7.egg/"
import sys
a = "/usr/local/lib/python2.7/dist-packages/pymongo-2.6_-py2.7.egg"
sys.path.insert(0, a)
import pymongo
assert not pymongo.has_c(), "you must compile pymongo with --no-ext"
import bson
import datetime
import collections

import mmap, os, struct, pdb, json, time, sys

schema = json.load(open('schema'))


def run(filename):
    fd = open(filename, "r")
    # we used mmap before, but converted to stdio while chasing down the
    # bson parser bug.  It's probably safe to change it back.  The slowdown
    # wasn't that bad though, so I didn't bother.
    mm = 0  # mmap.mmap(fd,0,access=mmap.ACCESS_READ)
    # pdb.set_trace()
    scan(filename, fd, mm)
    fd.close()
    print('done', dt(), filename)


t0 = time.time()
Example #34
0
# Copyright 2009-2010 10gen, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Fail if the C extension module doesn't exist.

Only really intended to be used by internal build scripts.
"""

import sys
sys.path[0:0] = [""]

import pymongo

if not pymongo.has_c():
    sys.exit("could not import _cbson")
Example #35
0
# limitations under the License.

"""MongoDB benchmarking suite."""

import argparse
import time
import threading
import math
import sys

from tornado import gen
from tornado.ioloop import IOLoop
import pymongo
import bson

assert pymongo.has_c()
assert bson._use_c

def parse_args():
    parser = argparse.ArgumentParser(description='Benchmark a script')
    parser.add_argument(
        'load', type=int, help='Number of requests to begin per second')
    return parser.parse_args()


def async_trial(log, c, fn, load, duration, warmup):
    # An object so it can be modified from inner functions
    class State(object):
        pass

    st = State()