def setUp(self):
disconnect()
# try docker mongo
host = settings._MONGODB_DATABASE_STR \
% (settings._MONGODB_HOST, self.mongodb_name)
connect(self.mongodb_name, host=host)
self.__test_id=0
def email_last_scraped_listing():
# mongodb params (using configparser)
config = configparser.ConfigParser()
config.read(
os.path.join(os.path.abspath(os.path.dirname(__file__)),
'settings.cfg'))
mlab_uri = config.get('MongoDB', 'mlab_uri')
#MONGO_URI = 'mongodb://localhost:27017'
# connect to db
MONGO_URI = mlab_uri
connect('tingi-sandbox', host=MONGO_URI)
## get last date of scraper run
for listing in StockListing.objects().fields(
date_str=1).order_by('-date_str').limit(1):
day_to_pull = listing.date_str
## pass vars, render template, and send
context = {
'day_to_pull': day_to_pull,
'Listing': StockListing,
}
html = render("template.html", context)
send_email(html)
def __init__(self, ext):
"""
Constructor.
:param ext: instance of :class:`EveMongoengine`.
"""
# get authentication info
username = ext.app.config['MONGO_USERNAME']
password = ext.app.config['MONGO_PASSWORD']
auth = (username, password)
if any(auth) and not all(auth):
raise ConfigException('Must set both USERNAME and PASSWORD '
'or neither')
# try to connect to db
if ext.app.config['MONGO_URI']: # use MONGO_URI if available
self.conn = connect(ext.app.config['MONGO_URI'])
else: # Or just use the usual connection settings
self.conn = connect(ext.app.config['MONGO_DBNAME'],
host=ext.app.config['MONGO_HOST'],
port=ext.app.config['MONGO_PORT'])
self.models = ext.models
self.app = ext.app
# create dummy driver instead of PyMongo, which causes errors
# when instantiating after config was initialized
self.driver = type('Driver', (), {})()
self.driver.db = get_db()
# authenticate
if any(auth):
self.driver.db.authenticate(username, password)
def setup_databases(self, *args, **kwargs):
for alias, params in self._iter_test_databases():
#disconnect(alias)
print("Connecting test database for alias '%s': %s" %
(alias, params['name']))
connect(params.pop('name'), alias=alias, **params)
return super(TestRunner, self).setup_databases(*args, **kwargs)
def setup_databases(self, **kwargs):
db_name = 'newsblur_test'
disconnect()
connect(db_name)
print('Creating test-database: ' + db_name)
return setup_databases(self.verbosity, self.interactive, **kwargs)
def connectToMongo():
mongoConfig = Config(app_var.CONFIG_PATH)
url = mongoConfig.getField("mongo", "url")
port = int(mongoConfig.getField("mongo", "port"))
username = mongoConfig.getField("mongo", "username")
password = mongoConfig.getField("mongo", "password")
db = mongoConfig.getField("mongo", "db")
print("\n===== MONGO =====")
print("\tConnecting to mongo: " + url + ":" + str(port) +
"\n\tuser/pass: "******" " + password + "\n\tdb: " + db)
connect(db=db, host=url, port=port, username=username, password=password)
conn = mongoengine.connection.get_connection(
) # type: mongoengine.connection.MongoClient
try:
print(
"\tConnecting to db... (if it take more than 20sec, you can stop the script, connection failed"
)
conn.get_database("mike")
except Exception as e:
print(
"ERROR: Can't connect to DB or DB name invalid. Full error is:\n" +
str(e))
sys.exit(1)
def setUp(self):
if os.environ.get('KARDBOARD_SETTINGS'):
os.environ['KARDBOARD_SETTINGS'] = ''
from kardboard import default_settings
default_settings.TEMPLATE_DEBUG = True
from kardboard.views import app
from flask.ext.mongoengine import MongoEngine
from kardboard.util import now
delattr(app, 'db')
from mongoengine.connection import connect, disconnect
disconnect()
app.config.from_object('kardboard.default_settings')
app.config['MONGODB_DB'] = 'kardboard_unittest'
app.config['TESTING'] = True
app.config['CELERY_ALWAYS_EAGER'] = True
connect(app.config['MONGODB_DB'])
app.db = MongoEngine(app)
self.config = app.config
self.app = app.test_client()
self.flask_app = app
self.used_keys = []
self._setup_logging()
self.now = now
super(KardboardTestCase, self).setUp()
def setup_databases(self, **kwargs):
from mongoengine.connection import connect, disconnect
disconnect()
connect(self.mongodb_name, tz_aware=True)
print 'Creating MongoDB test database, name: {0}'.format(
self.mongodb_name)
return super(VosaeTestRunner, self).setup_databases(**kwargs)
def setup():
sys.path.append("/home/ross/CytoPy")
os.mkdir(f"{os.getcwd()}/test_data")
connect("test", host="mongomock://localhost", alias="core")
yield
shutil.rmtree(f"{os.getcwd()}/test_data", ignore_errors=True)
disconnect(alias="core")
def init_db(app):
connect('swa_route',
host=app.config['MONGO_HOST'],
port=app.config['MONGO_PORT'],
username=app.config['MONGO_USERNAME'],
password=app.config['MONGO_PASSWORD']
)
def setUp(self):
if os.environ.get('KARDBOARD_SETTINGS'):
os.environ['KARDBOARD_SETTINGS'] = ''
from kardboard import default_settings
default_settings.TEMPLATE_DEBUG = True
from kardboard.views import app
from flask.ext.mongoengine import MongoEngine
from kardboard.util import now
delattr(app, 'db')
from mongoengine.connection import connect, disconnect
disconnect()
app.config.from_object('kardboard.default_settings')
app.config['MONGODB_DB'] = 'kardboard_unittest'
app.config['DEBUG'] = True
app.config['TESTING'] = True
app.config['CELERY_ALWAYS_EAGER'] = True
connect(app.config['MONGODB_DB'])
app.db = MongoEngine(app)
self.config = app.config
self.app = app.test_client()
self.flask_app = app
self.used_keys = []
self._setup_logging()
self.now = now
super(KardboardTestCase, self).setUp()
def __enter__(self):
connection.connect(db=self.database,
host=MongodbConfig.host,
port=MongodbConfig.port,
password=MongodbConfig.password,
username=MongodbConfig.username,
retryWrites=False)
print("AmazingQuant Database Connect")
def setup_databases(self, **kwargs):
#db_name = super(MongoTestRunner,self).setup_databases(**kwargs)
disconnect()
connect(self.mongo_name)
print 'MongoTestRunner: setup %s' % (self.mongo_name)
#return db_name
return
def setUp(self):
"""
Initialize client to send HTTP requests
and run migrations on the test db
"""
self.test_client = app.test_client()
connection.disconnect()
connection.connect('hangman_test')
def get_calendar(self, market):
connection.connect(db=self.database, host=MongodbConfig.host, port=MongodbConfig.port,
password=MongodbConfig.password, username=MongodbConfig.username, retryWrites=False)
data = AShareCalendar.objects(market=market).as_pymongo()
trade_days = data[0]['trade_days']
trade_days = [i for i in trade_days if i < datetime.now()]
connection.disconnect()
return trade_days
def setUp(self):
connect()
self.db = _get_db()
class Person(Document):
name = StringField()
age = IntField()
uid = ObjectIdField()
self.Person = Person
def _pre_setup(self):
from mongoengine.connection import connect, disconnect
disconnect()
import urllib, os
pwd = urllib.quote_plus(os.environ['MONGODB_PWD'])
uri = "mongodb://" + os.environ['MONGODB_USR'] + ":" + pwd + "@127.0.0.1/" + self.mongodb_name + "?authMechanism=SCRAM-SHA-1"
connect(self.mongodb_name, host=uri)
super(SearchappTest, self)._pre_setup()
def setup_databases(self, **kwangs):
global _running_test
_running_test = True
disconnect()
connect(MONGO_DB)
print 'Creating test-database: ' + MONGO_DB
print 'restoring default data'
call(["mongorestore", "--db", MONGO_DB, DUMP_DIR])
return MONGO_DB
def run_server():
mongo_connection.connect(host=options.mongo_uri)
server = Server()
server.listen(options.port, options.host)
server.sentry_client = AsyncSentryClient(
'https://*****:*****@sentry.team.ktsstudio.ru/6'
)
IOLoop.instance().start()
def __init__(self, path, database, username, password, host=None, port=None, auth=None):
if host is None:
host = 'localhost'
if port is None:
port = 27017
self.path = path
connect(database, host=host, port=port, username=username, password=password, authentication_source=auth)
self.db = get_db()
def __init__(self):
try:
host = get_config_value("MONGO", "host")
port = int(get_config_value("MONGO", "port"))
name = get_config_value("MONGO", "name")
disconnect()
connect(db=name, host=host, port=port)
self.conn = get_connection()
except:
logger.error('Connection to MongoDB could not be established.')
def __init__(self):
try:
host = get_config_value("MONGO", "host")
port = int(get_config_value("MONGO", "port"))
name = get_config_value("MONGO", "name")
disconnect()
connect(db=name, host=host, port=port)
self.conn = get_connection()
except:
logger.error('Connection to MongoDB could not be established.')
def random_close():
random.seed(time.time())
for _ in xrange(200):
time.sleep(0.1)
if random.random() < 0.8:
# print 'Clear connection...'
clear_all()
else:
# print 'Establish connection...'
connect(db_names=['test'])
def test_get_alias(self):
try:
alias = 'bla'
connect(alias=alias)
db = get_db(alias)
returned_alias = utils.get_alias_for_db(db)
self.assertEqual(returned_alias, alias)
finally:
del connection._connections['bla']
del connection._connection_settings['bla']
def migrate():
connect(yeti_config.mongodb.database,
host=yeti_config.mongodb.host,
port=yeti_config.mongodb.port,
username=yeti_config.mongodb.username,
password=yeti_config.mongodb.password,
connect=True)
db = get_db()
for i in list(db.observable.list_indexes()):
if i.to_dict()['name'] == "value_1":
db.observable.drop_index("value_1")
def _connect_db():
global __run_mode, __connected
if __connected:
return
db_config = 'db_%s' % __run_mode
if db_config in config.keys():
connect(**config[db_config])
else:
connect(**config['db'])
__connected = True
_importer.install_meta()
def setUp(self):
connect()
self.db = _get_db()
class Person(Document):
name = StringField()
age = IntField()
uid = ObjectIdField()
friends = ListField(StringField())
self.Person = Person
async def startup_event():
if settings.MONGO_DB_URI:
connect(host=settings.MONGO_DB_URI)
else:
connect(settings.MONGO_DB,
host=settings.MONGO_HOST,
port=settings.MONGO_PORT)
if settings.CLOUDINARY_CLOUD_NAME:
cloudinary.config(cloud_name=settings.CLOUDINARY_CLOUD_NAME,
api_key=settings.CLOUDINARY_API_KEY,
api_secret=settings.CLOUDINARY_API_SECRET)
def migrate():
connect(
yeti_config.mongodb.database,
host=yeti_config.mongodb.host,
port=yeti_config.mongodb.port,
username=yeti_config.mongodb.username,
password=yeti_config.mongodb.password,
connect=True)
db = get_db()
for i in list(db.observable.list_indexes()):
if i.to_dict()['name'] == "value_1":
db.observable.drop_index("value_1")
def setup_database(self, *args, **kwargs):
# Disconnect from the default mongo db, and use a test db instead.
self.disconnect_dbs()
connection.connect(MONGO_TEST_DATABASE_NAME,
alias=settings.MONGO_DATABASE_NAME, tz_aware=True)
for collection in ['account', 'user', 'rhic', 'fs.chunks',
'fs.files']:
print 'importing %s collection' % collection
call(['mongoimport', '--db', MONGO_TEST_DATABASE_NAME,
'-c', collection, '--file',
'%s.json' % os.path.join(settings.DUMP_DIR, collection)])
def _fixture_setup(self):
from mongoengine.connection import connect, disconnect
disconnect()
connect(self.mongodb_name)
from mongoengine.django.mongo_auth.models import MongoUser
if MongoUser.objects.filter(username="******").count() == 0:
MongoUser.objects.create_superuser("admin", "*****@*****.**", "admin")
if MongoUser.objects.filter(username="******").count() == 0:
MongoUser.objects.create_user("library_user", "*****@*****.**", "secret")
setup_test_environment()
def _fixture_setup(self):
from mongoengine.connection import connect, disconnect
disconnect()
connect(self.mongodb_name, host=settings.MONGODB_HOST)
from mongoengine.django.mongo_auth.models import MongoUser
if MongoUser.objects.filter(username="******").count() == 0:
MongoUser.objects.create_superuser("admin", "*****@*****.**", "admin")
if MongoUser.objects.filter(username="******").count() == 0:
MongoUser.objects.create_user("library_user", "*****@*****.**", "secret")
setup_test_environment()
def get_index_members(self, index_code):
database = DatabaseName.STOCK_BASE_DATA.value
connection.connect(db=database, host=MongodbConfig.host, port=MongodbConfig.port,
password=MongodbConfig.password, username=MongodbConfig.username, retryWrites=False)
index_members_data = AShareIndexMembers.objects(index_code=index_code).as_pymongo()
connection.disconnect()
field_list = ['index_code', 'security_code', 'in_date', 'out_date']
self.index_members_df = pd.DataFrame(list(index_members_data)).reindex(columns=field_list)
self.index_members_all = list(set(self.index_members_df["security_code"]))
self.index_members_df = self.index_members_df.fillna(datetime.now())
return self.index_members_df, self.index_members_all
def _pre_setup(self):
"""
Function: _pre_setup
--------------------
Automatically called before any test for connecting to MongoDB.
"""
from mongoengine.connection import connect, disconnect
disconnect()
import urllib, os
pwd = urllib.quote_plus(os.environ['MONGODB_PWD'])
uri = "mongodb://" + os.environ['MONGODB_USR'] + ":" + pwd + "@127.0.0.1/" + self.mongodb_name + "?authMechanism=SCRAM-SHA-1"
connect(self.mongodb_name, host=uri)
super(B2noteappTest, self)._pre_setup()
def _pre_setup(self):
"""
Function: _pre_setup
--------------------
Automatically called before any test for connecting to MongoDB.
"""
from mongoengine.connection import connect, disconnect
disconnect()
import urllib, os
pwd = urllib.quote_plus(os.environ['MONGODB_PWD'])
uri = "mongodb://" + os.environ[
'MONGODB_USR'] + ":" + pwd + "@127.0.0.1/" + self.mongodb_name + "?authMechanism=SCRAM-SHA-1"
connect(self.mongodb_name, host=uri)
super(B2noteappTest, self)._pre_setup()
def __init__(self, ext):
"""
Constructor.
:param ext: instance of :class:`EveMongoengine`.
"""
# get authentication info
username = ext.app.config['MONGO_USERNAME']
password = ext.app.config['MONGO_PASSWORD']
auth = (username, password)
if any(auth) and not all(auth):
raise ConfigException('Must set both USERNAME and PASSWORD '
'or neither')
# try to connect to db
self.conn = connect(ext.app.config['MONGO_DBNAME'],
host=ext.app.config['MONGO_HOST'],
port=ext.app.config['MONGO_PORT'])
self.models = ext.models
self.app = ext.app
# create dummy driver instead of PyMongo, which causes errors
# when instantiating after config was initialized
self.driver = type('Driver', (), {})()
self.driver.db = get_db()
# authenticate
if any(auth):
self.driver.db.authenticate(username, password)
def __init__(self, method_name='runtest'):
disconnect()
db_name = 'test_%s' % settings.MONGOENGINE['db']
self.database = connect(db_name)[db_name]
super(MongoTestCase, self).__init__(method_name)
def _setup_db(self, db_section='mongodb'):
# get all config values about DB
db_config = self.config[db_section] # map
db_name = db_config.get('name')
attr_map = {
'host': 'str',
'port': 'int',
'username': '******',
'password': '******'
}
kwargs = {}
for key, typ in attr_map.iteritems():
typecast_fn = getattr(__builtin__, typ)
# cast the value from db_config accordingly if key-value pair exists
kwargs[key] = typecast_fn(
db_config.get(key)) if db_config.get(key) else None
connection.disconnect(
'default') # disconnect previous default connection if any
self.db = connection.connect(db_name, **kwargs)
logging.info('connected to Database: {}'.format(self.db))
def migrate():
connect(yeti_config.mongodb.database,
host=yeti_config.mongodb.host,
port=yeti_config.mongodb.port,
username=yeti_config.mongodb.username,
password=yeti_config.mongodb.password,
connect=True)
db = get_db()
# Drop these indexes as they have changed
db.schedule_entry.drop_indexes()
db.one_shot_entry.drop_indexes()
db.inline_analytics.drop_indexes()
db.attached_file.drop_indexes()
change_oneshot_entries(db)
change_feed_name(db)
correct_feed_objects()
def test_data_conn():
"""
A simple CLI command to test the Mongo and Redis connections when in a
production environment.
"""
if os.environ.get('FLASK_ENV', 'development') == 'production':
disconnect_all()
_db_name = str(os.environ.get('MONGO_APP_DB'))
_host = os.environ.get('MONGO_HOST')
_port = os.environ.get('MONGO_PORT')
_username = os.environ.get('MONGO_APP_USER')
_password = str(os.environ.get('MONGO_APP_USER_PASSWORD'))
mongo_uri = (f'mongodb://{_username}:{_password}@{_host}:{_port}'
f'/?authSource=admin')
client = connect(db=_db_name,
host=mongo_uri,
alias='default',
authentication_mechanism='SCRAM-SHA-1')
db = get_db('default')
redis_client = redis.Redis(host=os.environ.get('REDIS_HOST'),
port=os.environ.get('REDIS_PORT'),
password=os.environ.get('REDIS_PASSWORD'))
print(f'MongoDB Client: \n{client}\n', file=sys.stderr)
print(f'MongoDB Database: \n{db}\n', file=sys.stderr)
print(f'Redis Datastore Client: \n{redis_client}\n', file=sys.stderr)
else:
print(
f'This CLI command is only used for Production\nCurrent Flask ENV: '
f'{os.environ.get("FLASK_ENV")}',
file=sys.stderr)
def get_mongodb_connection():
env = os.environ.get('UGGIPUGGI_BACKEND_ENV', 'docker_compose')
config = get_config(env)
# get all config values about DB
db_config = config['mongodb'] # map
db_name = db_config.get('name')
attr_map = {
'host': 'str',
'port': 'int',
'username': '******',
'password': '******'
}
kwargs = {}
for key, typ in attr_map.items():
typecast_fn = getattr(builtins, typ)
# cast the value from db_config accordingly if key-value pair exists
kwargs[key] = typecast_fn(
db_config.get(key)) if db_config.get(key) else None
#kwargs['alias'] = 'default_celery'
#connection.disconnect('mongo_celery') # disconnect previous default connection if any
connection.disconnect(
'default') # disconnect previous default connection if any
return connection.connect(db_name, **kwargs)
def __init__(self, ext):
"""
Constructor.
:param ext: instance of :class:`EveMongoengine`.
"""
# get authentication info
username = ext.app.config['MONGO_USERNAME']
password = ext.app.config['MONGO_PASSWORD']
auth = (username, password)
if any(auth) and not all(auth):
raise ConfigException('Must set both USERNAME and PASSWORD '
'or neither')
# try to connect to db
self.conn = connect(ext.app.config['MONGO_DBNAME'],
host=ext.app.config['MONGO_HOST'],
port=ext.app.config['MONGO_PORT'])
self.models = ext.models
self.app = ext.app
# create dummy driver instead of PyMongo, which causes errors
# when instantiating after config was initialized
self.driver = type('Driver', (), {})()
self.driver.db = get_db()
# authenticate
if any(auth):
self.driver.db.authenticate(username, password)
# helper object for managing PATCHes, which are a bit dirty
self.updater = MongoengineUpdater(self)
# map resource -> Mongoengine class
self.cls_map = ResourceClassMap(self)
def __init__(self, ext):
"""
Constructor.
:param ext: instance of :class:`EveMongoengine`.
"""
# get authentication info
username = ext.app.config.get("MONGO_USERNAME", None)
password = ext.app.config.get("MONGO_PASSWORD", None)
auth = (username, password)
if any(auth) and not all(auth):
raise ConfigException("Must set both USERNAME and PASSWORD "
"or neither")
# try to connect to db
self.conn = connect(
ext.app.config["MONGO_DBNAME"],
host=ext.app.config["MONGO_HOST"],
port=ext.app.config["MONGO_PORT"],
)
self.models = ext.models
self.app = ext.app
# create dummy driver instead of PyMongo, which causes errors
# when instantiating after config was initialized
self.driver = type("Driver", (), {})()
self.driver.db = get_db()
# authenticate
if any(auth):
self.driver.db.authenticate(username, password)
# helper object for managing PATCHes, which are a bit dirty
self.updater = MongoengineUpdater(self)
# map resource -> Mongoengine class
self.cls_map = ResourceClassMap(self)
def setUp(self):
db_name = settings.MONGO_DATABASE_NAME_RESULTS
self.db = connect(db_name)
ReportData.drop_collection()
rhel_product = TestData.create_products()
rhel_entry = TestData.create_entry(RHEL, memhigh=True)
rhel_entry.save()
def setUp(self):
db_name = settings.MONGO_DATABASE_NAME_RESULTS
self.db = connect(db_name)
ReportData.drop_collection()
rhel_product = TestData.create_products()
rhel_entry = TestData.create_entry(RHEL, memhigh=True)
rhel_entry.save()
def initDb():
dbServer = {"dbName":"telugubeats",
"ip":"104.155.234.161",# "db.quizapp.appsandlabs.com",
"port": 27017,
"username": "******",
"password":"******"
}
dbConnection = connect(dbServer["dbName"], host=dbServer["ip"], port=dbServer["port"], username=dbServer["username"],password=dbServer["password"])
def connect_db(db_alias):
disconnect('default')
# Hack to avoid mongoengine caching connection
User._collection = None
Handin._collection = None
return connect(test_db_name(db_alias))
def migrate():
connect(
yeti_config.mongodb.database,
host=yeti_config.mongodb.host,
port=yeti_config.mongodb.port,
username=yeti_config.mongodb.username,
password=yeti_config.mongodb.password,
connect=True)
db = get_db()
# Drop these indexes as they have changed
db.schedule_entry.drop_indexes()
db.one_shot_entry.drop_indexes()
db.inline_analytics.drop_indexes()
db.attached_file.drop_indexes()
change_oneshot_entries(db)
change_feed_name(db)
correct_feed_objects()
def drop_database_and_reconnect(self, reconnect=True):
disconnect()
self.db.drop_database(self.db_name)
# Mongoengine sometimes doesn't recreate unique indexes
# in between test runs, adding the below 'reset' to fix this
# https://github.com/hmarr/mongoengine/issues/422
QuerySet._reset_already_indexed()
if reconnect:
self.db = connect(self.db_name)
def _pre_setup(self):
super(MongoTestCase, self)._pre_setup()
disconnect()
db_name = 'test_%s' % settings.MONGOENGINE['db']
self._database = connect( # pylint: disable=attribute-defined-outside-init
db_name, tz_aware=settings.USE_TZ
)[db_name]
def __init__(self, *args, **kwargs):
super(Mongoengine, self).__init__(*args, **kwargs)
self.connection = connect(
db=self.settings.database,
alias=self.settings.alias,
host=self.settings.host,
port=self.settings.port,
max_pool_size=self.settings.max_pool_size,
tz_aware=self.settings.tz_aware)
self.db = get_db(alias=self.settings.alias)
def setUp(self):
setlocale(LC_ALL, "en_US.UTF_8")
connect(
db=environ.get("TEST_DB", "test"),
host=environ.get("TEST_HOST", "localhost"),
port=environ.get("TEST_PORT", 27017),
)
Country(
code="ru", name=mls(ru=u("Россия"), en="Russia", cs="Rusko")
).save()
Country(
code="cz", name=mls(
ru=u("Чехия"),
en="Czech Republic",
cs=u("Česko")
)
).save()
def setUp(self):
super(TestQuerysetPickable, self).setUp()
connection = connect(db="test") # type: pymongo.mongo_client.MongoClient
connection.drop_database("test")
self.john = Person.objects.create(
name="John",
age=21
)
def __init__(self):
# Build necessary args from config
kwargs = {
'db': get_config('data', 'dbname'),
'host': get_config('data', 'host'),
'port': get_config('data', 'port', is_int=True)
}
# Optional argument to MongoDB
user = get_config('data', 'user')
if user:
kwargs['username'] = user
# Optional argument to MongoDB
password = get_config('data', 'password')
if password:
kwargs['password'] = password
disconnect()
connect(**kwargs)
self.conn = get_connection()
def getPopulation(date,hour):
connection=connect(config["MONGODB"]["DB_NAME"])
#dfPopulation=pd.read_csv("data/telefonica/cleaned_mobile_data/"+str(date)+".csv",header=False,names=["date","cell_id","hour","n_people"],index_col=["date","hour"])
#dfPopulation=pd.read_csv("data/telefonica/cleaned_mobile_data/"+str(date)+".csv",header=False,names=["date","cell_id","hour","n_people"],parse_dates={"date_time" : [0,2]}, date_parser=parse,index_col=0)
#cursor=collection.find({"date": str(date)})
#for item in cursor:
# items.append(item)
qs=TelefonicaPopulation.objects.filter(date=str(date),hour=hour)
popn=qs[:1].get()
#disconnect()
#dfPopulation.index=pd.to_datetime(dfPopulation["date"]+)
return popn
def init_mongo_connection(settings_databases_conf):
'''
Initialize MongoDB connectivity
'''
for db_name, kwargs in settings_databases_conf.items():
if not db_name:
logger.info('Skipping empty mongodb name')
continue
if kwargs.get('disabled'):
logger.info('Skipping disabled MongoDB configuration for "%s"' % db_name)
continue
if not kwargs.get('alias') or not kwargs.get('host'):
raise RuntimeError('MongoDB configuration for "%s" needs to specify alias and host' % db_name)
connection.connect(db_name, **kwargs)
db_alias = kwargs['alias']
# wrap the connections with the proxy which handles AutoReconnect
mongo_conn = connection._connections[db_alias]
mongo_conn_proxied = mongo_proxy.MongoProxy(mongo_conn)
connection._connections[db_alias] = mongo_conn_proxied
logger.debug('Connected to %s', db_name)
def setUp(self):
disconnect()
settings.MONGODB = connect('test_newsblur')
settings.REDIS_STORY_HASH_POOL = redis.ConnectionPool(host=settings.REDIS_STORY['host'], port=6379, db=10)
settings.REDIS_FEED_READ_POOL = redis.ConnectionPool(host=settings.SESSION_REDIS_HOST, port=6379, db=10)
r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
r.delete('RS:1')
r.delete('lRS:1')
r.delete('RS:1:766')
r.delete('zF:766')
r.delete('F:766')
self.client = Client()
def setup_database(self):
# Disconnect from the default mongo db, and use a test db instead.
self.disconnect_dbs()
connection.connect(rhic_serve,
alias='default', tz_aware=True)
register_connection(rhic_serve, rhic_serve)
register_connection(checkin_service, checkin_service)
register_connection(report, report)
register_connection('default', rhic_serve)
for collection in ['rhic', 'account', 'user', 'fs.chunks']:
#print 'importing %s collection' % collection
call(['mongoimport', '--db', rhic_serve,
'-c', collection, '--file',
'%s.json' % os.path.join(settings.DUMP_DIR, collection)],
stdout=PIPE, stderr=PIPE)
for collection in ['splice_server', 'product']:
#print 'importing %s collection' % collection
call(['mongoimport', '--db', checkin_service,
'-c', collection, '--file',
'%s.json' % os.path.join(settings.DUMP_DIR, collection)],
stdout=PIPE, stderr=PIPE)
def build_base_dbs():
for test_set in test_sets:
db_name = test_db_base_name(test_set.db_alias)
db = connect(db_name)
db.drop_database(db_name)
print User._get_db()
print "==== STARTING {} ====".format(test_set.db_alias)
print "CREATING USERS"
users = [User(name='Joe', age=25)
for _ in range(test_set.num_of_users)]
User.objects.insert(users)
print "BUILDING HANDINS"
handins = [Handin(score=10)
for _ in range(test_set.num_of_handins)]
print "APPENDING USERS TO HANDINS"
handins = []
users = User.objects()
test_indices = range(0,
test_set.num_of_users,
test_set.num_of_users_per_handin)
for i in test_indices:
handins.append(
Handin(
score=i,
users=users[i:i+test_set.num_of_users_per_handin]
)
)
print "INSERTING HANDINS"
Handin.objects.insert(handins)
print "==== FINISHED {} ====".format(test_set.db_alias)
disconnect('default')
# Hack to avoid mongoengine caching connection
User._collection = None
Handin._collection = None
