def connect_to_db(self):
connect_to_db()
register_connection(DB_BACKTEST,
db=DB_BACKTEST,
name=DB_BACKTEST,
host=MONGO_HOST,
port=MONGO_PORT)
def db_conn_init():
config = get_config()
name = config['DATABASE']['Name']
host = config['DATABASE']['Host']
port = int(config['DATABASE']['Port'])
username = config['DATABASE'].get('Username')
username = os.environ.get('DB_USERNAME') or username
username = username.strip() if username else None
password = config['DATABASE'].get('Password')
password = os.environ.get('DB_PASSWORD') or password
password = password.strip() if password else None
tls = config['DATABASE'].getboolean('TlsEnabled', fallback=False)
tls_ca_file = config['DATABASE'].get('TlsCaCertificate') if tls else None
args = {
'alias': 'global',
'name': name,
'host': host,
'port': port,
'username': username if username and password else None,
'password': password if username and password else None,
'tls': tls,
'tlsCAFile': tls_ca_file
}
# Register connection
mongoengine.register_connection(**args)
def setup_pauli(flask_app, url_prefix="/pauli"):
# setup mongodb
for alias, attrs in conf.MONGO.items():
mongoengine.register_connection(alias, **attrs)
# setup session
session_db = mongoengine.connection.get_db(alias="pauli")
config_dict = {
"SESSION_COOKIE_NAME": conf.SESSION_COOKIE_NAME,
"SESSION_TYPE": "mongodb",
"SESSION_MONGODB": session_db.client,
"SESSION_MONGODB_DB": session_db.name,
"SESSION_MONGODB_COLLECT": "sessions",
"PERMANENT_SESSION_LIFETIME": conf.PAULI_SESSION_LIFETIME
}
flask_app.config.update(config_dict)
flask_session.Session(flask_app)
# import sub level path routings
from .user.views import api
from .auth.views import api
from .auth.views import web
from .perm.views import api
# bind pauli blueprint
flask_app.register_blueprint(pauli_root, url_prefix=url_prefix)
def main(global_config, **settings):
""" This function returns a Pyramid WSGI application.
"""
# authn_policy = AuthTktAuthenticationPolicy(
# 'gabbleblotchits')
# authz_policy = SNMAuthorizationPolicy()
config = Configurator(
# authentication_policy=authn_policy,
# authorization_policy=authz_policy,
settings=settings
)
config.include("pyramid_jinja2")
config.commit()
config.get_jinja2_environment().filters['epoch2readable'] = epoch2readable
config.get_jinja2_environment().filters['duration'] = duration
def addSettings(event):
event["sci_platform"] = settings["sci_platform"]
event["google_tracking_code"] = settings["google_tracking_code"]
config.add_subscriber(addSettings, BeforeRender)
# Configuring static assets
config.add_static_view(name="static", path="static")
config.add_settings({"sci_platform": settings["sci_platform"],
"google_tracking_code": settings["google_tracking_code"]})
config.add_route("home", "/")
config.add_route("dsm", "/dsm")
config.add_route("app_used_with", "/application/{name}/used_with")
config.add_route("app_usage", "/application/{name}/usage")
config.add_route("app_users", "/application/{name}/users")
config.add_route("app_dashboard", "/application/{name}/dashboard")
config.add_route("application", "/application/{name}")
config.add_route("app_pubs", "/application/{name}/publications")
config.add_route("app_gitprojects", "/application/{name}/gitprojects")
config.add_route("compare", "/compare")
config.add_route("about", "/about")
config.add_route("data_source", "/data_source")
config.add_route("browse", "/browse")
config.add_route("overview", "/overview")
config.add_route("login", "/login")
config.add_route("notebook", "/notebook")
config.add_route("status", "/status")
config.add_route("sys_usage", "/status/usage")
config.add_route("accept_login", "/accept_login")
config.add_route("api_home", "/api")
config.add_route("api_home.category", "/api/{category}")
config.add_route("api_home.category.id", "/api/{category}/{id}")
connect(settings['db_name'])
register_connection('raw-records', settings['raw_db_name'])
config.scan()
return config.make_wsgi_app()
def test_register_connection_defaults(self):
"""Ensure that defaults are used when the host and port are None.
"""
register_connection('testdb', 'mongoenginetest', host=None, port=None)
conn = get_connection('testdb')
self.assertTrue(isinstance(conn, pymongo.mongo_client.MongoClient))
def init_mongoengine():
hosts = []
for item in MONGO_DATABASES:
hosts.append('%s:%s'% (item['host'], item['port']))
print '-' * 40,'init mongoengine connect!'
mongoengine.register_connection('default', DBNAME, host=','.join(hosts))
def setUp(self):
super(CreateMongoTxsAndBlocksMixin, self).setUp()
self.api_key = ApiKeyFactory()
blocks = []
mongoengine.register_connection(config.MONGO_TEST_DATABASE_NAME,
name=config.MONGO_TEST_DATABASE_NAME)
with switch_db(EthTransactions,
config.MONGO_TEST_DATABASE_NAME) as TestEthTransactions:
with switch_db(EthBlocks,
config.MONGO_TEST_DATABASE_NAME) as TestEthBlocks:
client = MongoClient(connect=False)
for test_block_data in test_block_data_list:
block = TestEthBlocks(**test_block_data)
block.save()
blocks.append(block)
for index, test_tx_data in enumerate(test_tx_data_list):
data_to_create = copy.deepcopy(test_tx_data)
data_to_create['fromAddress'] = test_tx_data.get('from')
data_to_create['toAddress'] = test_tx_data.get('to')
tx = TestEthTransactions(**data_to_create)
if index + 1 <= len(test_block_data_list):
tx.block = blocks[index]
tx.timestamp = blocks[index]['timestamp']
else:
tx.block = blocks[-1]
tx.timestamp = blocks[-1]['timestamp']
tx.save()
def init_connection(config=MONGODB_CONFIG[DEFAULT_CONNECTION_NAME]):
u'holy shit!'.encode('idna')
connect(**config)
register_connection('statistics',
host=config['host'],
port=config['port'],
name='metrics')
def mongo_connect():
register_connection('default',
app.config.get('MONGODB_DATABASE', 'grano'),
host=app.config.get('MONGODB_HOST', 'localhost'),
port=app.config.get('MONGODB_PORT', 27017),
username=app.config.get('MONGODB_USERNAME'),
password=app.config.get('MONGODB_PASSWORD'))
def global_init():
mongoengine.register_connection(alias='core',
name=db,
host=mongodb_ip,
port=27017,
username='******',
db=db)
def tenant_handler():
host = request.headers.get('Host')
try:
tenant = etcd_client.read('/atrium/hosts/' + host + '/tenant_id').value
except etcd.EtcdKeyNotFound:
return abort(404)
tenant_keys = etcd_client.read('/atrium/tenants/' + tenant)
tenant_config = {}
for result in tenant_keys.children:
tenant_config[result.key.split('/atrium/tenants/' + tenant + '/')[1]] = result.value
g.tenant = tenant
openid_items = [
('authorization_endpoint', 'openid_authorization'),
('token_endpoint', 'openid_token'),
('userinfo_endpoint', 'openid_userinfo'),
('client_id', 'openid_client'),
('client_secret', 'openid_secret'),
('redirect_uri', 'openid_redirect'),
('issuer_key', 'openid_issuer_key'),
('issuer_claim', 'openid_issuer_claim')
]
g.openid = {}
for item in openid_items:
g.openid[item[0]] = str(tenant_config[item[1]])
mongoengine.register_connection(tenant, name=tenant, host=app.config['MONGODB_HOST'], port=app.config['MONGODB_PORT'], connect=False, tz_aware=True)
for schema in schemas:
setattr(g, schema.__name__, schema)
getattr(g, schema.__name__)._meta['db_alias'] = tenant
getattr(g, schema.__name__)._collection = None
def test_register_connection_defaults(self):
"""Ensure that defaults are used when the host and port are None.
"""
register_connection("testdb", "mongoenginetest", host=None, port=None)
conn = get_connection("testdb")
self.assertIsInstance(conn, pymongo.mongo_client.MongoClient)
def test_sync_blocks_from_position_to_end(self, mock_get_block,
mock_get_transaction):
mongoengine.register_connection(config.MONGO_TEST_DATABASE_NAME,
name=config.MONGO_TEST_DATABASE_NAME)
with switch_db(EthTransactions,
config.MONGO_TEST_DATABASE_NAME) as TestEthTransactions:
with switch_db(EthBlocks,
config.MONGO_TEST_DATABASE_NAME) as TestEthBlocks:
real_block_position = config.SYNC_BLOCKS_POSITION
config.SYNC_BLOCKS_POSITION = 1
mock_get_block.return_value = copy.deepcopy(
self.test_block_data)
mock_get_transaction.return_value = copy.deepcopy(
self.test_tx_data)
with mock.patch.object(Eth,
'blockNumber') as mock_block_number:
mock_block_number.__get__ = mock.Mock(return_value=2)
sync_blocks_from_position_to_end()
client = MongoClient(connect=False)
db = client[config.MONGO_TEST_DATABASE_NAME]
self.assertEqual(db.eth_blocks.count(), 1)
self.assertEqual(db.eth_transactions.count(), 1)
config.SYNC_BLOCKS_POSITION = real_block_position
client[config.MONGO_TEST_DATABASE_NAME].eth_blocks.remove()
client[config.MONGO_TEST_DATABASE_NAME].eth_transactions.remove()
def init_db_connection():
register_connection(alias="default",
name=config["DB_NAME"],
username=config["DB_USER"],
password=config["DB_PASSWORD"],
host=config["DB_URL"],
port=config["DB_PORT"])
def create_app(config_class):
app = Flask(__name__)
app.config.from_object(config_class)
# Initialize MongoEngine
try:
register_connection(alias="default",
name=app.config["DB_NAME"],
username=app.config["DB_USER"],
password=app.config["DB_PASSWORD"],
host=app.config["DB_URL"],
port=app.config["DB_PORT"])
except:
print("Database could not be configured.")
return None
nav.init_app(app)
login_manager.init_app(app)
bootstrap.init_app(app)
app.redis = Redis.from_url(app.config['REDIS_URL'])
app.task_queue = rq.Queue(app.config['REDIS_QUEUE_NAME'],
connection=app.redis)
from app.jobs import bp as jobs_bp
app.register_blueprint(jobs_bp, url_prefix='')
from app.web import bp as web_bp
app.register_blueprint(web_bp, url_prefix='')
from app.user import bp as user_bp
app.register_blueprint(user_bp, url_prefix='')
return app
def global_init():
mongoengine.register_connection(alias='businessapp', name='com_chay')
mongoengine.connect(
'Cluster0',
host=
'mongodb+srv://Kpm:[email protected]/test?retryWrites=true&w=majority'
)
def global_init():
try:
mongoengine.register_connection(alias="default", name="Johnson", host=host)
svc.Logging.log(__name__, "Connected!")
except pymongo.errors.ConfigurationError or pymongo.errors.ConnectionFailure:
svc.Logging.warning("global_init", "Could not connect; Retrying...")
global_init()
def test_register_connection_defaults(self):
"""Ensure that defaults are used when the host and port are None.
"""
register_connection('testdb', 'mongoenginetest', host=None, port=None)
conn = get_connection('testdb')
self.assertTrue(isinstance(conn, pymongo.mongo_client.MongoClient))
def init():
db = 'dealership'
# Other connection options here (server, port, username, etc.)
mongoengine.register_connection(alias=alias_core,
name=db,
host="hermione",
port=27017)
def setUp( self ):
mongoengine.register_connection( mongoengine.DEFAULT_CONNECTION_NAME, 'mongoengine_relational_test' )
c = mongoengine.connection.get_connection()
c.drop_database( 'mongoengine_relational_test' )
# Setup application/request config
self.request = Request.blank( '/api/v1/' )
# Instantiate a DocumentCache; it will attach itself to `request.cache`.
DocumentCache( self.request )
self.config = testing.setUp( request=self.request )
# Setup data
d = self.data = Struct()
d.blijdorp = Zoo( id=ObjectId(), name='Blijdorp' )
d.office = Office( tenant=d.blijdorp )
d.bear = Animal( name='Baloo', species='bear', zoo=d.blijdorp )
d.mammoth = Animal( id=ObjectId(), name='Manny', species='mammoth' )
d.artis = Zoo( id=ObjectId(), name='Artis', animals=[ d.mammoth ] )
d.tiger = Animal( id=ObjectId(), name='Shere Khan', species='tiger', zoo=d.artis )
d.node = Node()
def global_init():
mongoengine.register_connection(
alias='core',
name='rss',
host=
'mongodb+srv://felipe:[email protected]/news?retryWrites=true'
)
def register_connection():
mongoengine.register_connection(
alias="core",
name="emerg_shutdown",
username=config.mongo_db_user,
password=config.mongo_db_pass,
authentication_source="admin",
)
def reconnect(cls):
for entry in cls.get_entries():
# there is bug in the current implementation that prevents
# reconnection from work so workaround this
# get_connection(entry.alias, reconnect=True)
disconnect(entry.alias)
register_connection(**entry.to_struct())
get_connection(entry.alias)
def setUp(self):
"""
Connects to the test database.
"""
# register_connection(TEST_DB_ALIAS, name=TEST_DB_NAME, host="10.1.8.102")
register_connection(TEST_DB_ALIAS, name=TEST_DB_NAME, host='mongodb://mongo')
self.client = Client()
self.maxDiff = None
def global_init(auth_dict):
mongoengine.register_connection(alias=auth_dict['alias'],
name=auth_dict['name'],
host=auth_dict['ip'],
port=auth_dict['port'])
# TODO - Delete below if not needed.
"""
def register_mongo_connection(db):
register_connection(
"default",
name=db["MONGO_NAME"],
host=db["MONGO_HOST"],
port=db["MONGO_PORT"],
**{"maxPoolSize": db["MONGO_MAX_POOL_SIZE"]}
)
def __init__(self, dbname, port=27017, host='127.0.0.1', basefs='/data/'):
self.db = pymongo.MongoClient(host=host, port=port)[dbname]
self.fs_resolver = FilesystemResolver(basefs)
connect(dbname, host=host, port=port)
register_connection("pdb", name="pdb", host=host, port=port)
self.paths = {"gene": []}
def setup_db( drop=True ):
mongoengine.register_connection( mongoengine.DEFAULT_CONNECTION_NAME, 'tastymongo_test' )
c = mongoengine.connection.get_connection()
if drop:
c.drop_database( 'tastymongo_test' )
return c
def main():
app.secret_key = 'as90kSDKO#1@|4245losadim'
build_app(app)
register_connection('irianas_web', 'irianas_web')
register_connection('irianas_server', 'irianas_server')
app.run(debug=debug, ssl_context=context, host='0.0.0.0', port=9001,
threaded=True)
def global_init():
subprocess.run(["sudo", "service", "mongod", "start"])
p = subprocess.Popen(["systemctl", "is-active", "mongod"], stdout=subprocess.PIPE)
(output, err) = p.communicate()
output = output.decode('utf-8').rstrip("\n")
if output == "active":
mongoengine.register_connection(alias='core', db='mindflayer', replicaset='rs', read_preference=ReadPreference.PRIMARY)
print("[!]Started MongoDB")
def setUpClass(cls):
mongoengine.register_connection('SteamDB', 'SteamDB')
my_user = userdb.SteamUser\
.objects(name = 'CHADVS MAXIMVS').first()
steam_user = requests.get(
'http://api.steampowered.com/IPlayerService/GetOwnedGames/v0001/?key=24E7A4CB6C2041D4C08EC325A5F4FFC3&steamid='
+ my_user.steamid + '&include_played_free_games=true&format=json')
cls.user = my_user
cls.request = steam_user.json()
def mongo_connect():
register_connection(
"default",
app.config.get("MONGODB_DATABASE", "grano"),
host=app.config.get("MONGODB_HOST", "localhost"),
port=app.config.get("MONGODB_PORT", 27017),
username=app.config.get("MONGODB_USERNAME"),
password=app.config.get("MONGODB_PASSWORD"),
)
def connect_db():
from mongoengine import register_connection
from django.conf import settings
if hasattr(settings, "MONGODB_DATABASES"):
databases = settings.MONGODB_DATABASES
else:
databases = {}
db_name = getattr(settings, "MONGODB_DATABASE", "supportcenter")
host = getattr(settings, "MONGODB_HOST", None)
port = getattr(settings, "MONGODB_PORT", None)
username = getattr(settings, "MONGODB_USERNAME", None)
password = getattr(settings, "MONGODB_PASSWORD", None)
d = {'NAME': db_name}
if host:
d['HOST'] = host
if port:
d['PORT'] = port
if username:
d['USERNAME'] = username
if password:
d['PASSWORD'] = password
databases['default'] = d
for alias in ('fs', 'tmp', 'log'):
d = {'NAME': db_name + alias}
if host:
d['HOST'] = host
if port:
d['PORT'] = port
if username:
d['USERNAME'] = username
if password:
d['PASSWORD'] = password
databases[alias] = d
for alias, db in databases.iteritems():
kwargs = {}
kwargs['name'] = db.get('NAME')
if db.get('HOST'):
kwargs['host'] = db['HOST']
if db.get('PORT'):
kwargs['port'] = db['PORT']
if db.get('USERNAME'):
kwargs['username'] = db['USERNAME']
if db.get('PASSWORD'):
kwargs['password'] = db['PASSWORD']
register_connection(alias, **kwargs)
preload_models()
def global_init():
mongoengine.register_connection(alias="core", name="uniques")
username = urllib.parse.quote_plus(os.getenv('MONGODB_USERNAME'))
password = urllib.parse.quote_plus(os.getenv('MONGODB_PASSWORD'))
mongoengine.connect(
host=
"mongodb+srv://%s:%[email protected]/xander?retryWrites=true&w=majority"
% (username, password))
print("MongoDB connection successful")
def connect_db():
from mongoengine import register_connection
from django.conf import settings
if hasattr(settings, "MONGODB_DATABASES"):
databases = settings.MONGODB_DATABASES
else:
databases = {}
db_name = getattr(settings, "MONGODB_DATABASE", "supportcenter")
host = getattr(settings, "MONGODB_HOST", None)
port = getattr(settings, "MONGODB_PORT", None)
username = getattr(settings, "MONGODB_USERNAME", None)
password = getattr(settings, "MONGODB_PASSWORD", None)
d = {'NAME': db_name}
if host:
d['HOST'] = host
if port:
d['PORT'] = port
if username:
d['USERNAME'] = username
if password:
d['PASSWORD'] = password
databases['default'] = d
for alias in ('fs', 'tmp', 'log'):
d = {'NAME': db_name+alias}
if host:
d['HOST'] = host
if port:
d['PORT'] = port
if username:
d['USERNAME'] = username
if password:
d['PASSWORD'] = password
databases[alias] = d
for alias, db in databases.iteritems():
kwargs = {}
kwargs['name'] = db.get('NAME')
if db.get('HOST'):
kwargs['host'] = db['HOST']
if db.get('PORT'):
kwargs['port'] = db['PORT']
if db.get('USERNAME'):
kwargs['username'] = db['USERNAME']
if db.get('PASSWORD'):
kwargs['password'] = db['PASSWORD']
register_connection(alias, **kwargs)
preload_models()
def test_changes_in dbs(NUM_SERVERS=NUM_SERVERS):
dbNames = ['RAFT_DB_' + str(i) for i in range(0,NUM_SERVERS)]
for dbName in dbNames:
mongoengine.register_connection(alias='core', name=dbName)
users = databaseRead.readUsers(dbName=dbName)
print("Database Name : ", dbName)
print("Num of Users : ", len(users))
print('----------Users------------')
databaseRead.print_users(users)
mongoengine.disconnect(alias='core')
def wrapper(*args, **kwargs):
if (DEBUG==True) and 'dbName' in kwargs:
dbName = kwargs.get('dbName')
if dbName is not None:
mongoengine.register_connection(alias='middle', name=dbName)
results = orig_func(*args, **kwargs)
mongoengine.disconnect(alias='middle')
return results
return orig_func(*args, **kwargs)
def register_mongo_connection(db):
register_connection(
'default',
name=db['MONGO_NAME'],
host=db['MONGO_HOST'],
port=db['MONGO_PORT'],
**{
'maxPoolSize': db['MONGO_MAX_POOL_SIZE'],
}
)
def setUp(self):
"""
Connects to the test database.
"""
# register_connection(TEST_DB_ALIAS, name=TEST_DB_NAME, host="10.1.8.102")
register_connection(TEST_DB_ALIAS,
name=TEST_DB_NAME,
host='mongodb://mongo')
self.client = Client()
self.maxDiff = None
def register() -> None:
register_connection(
"IGeniusTest",
"IGeniusTest",
username="",
password="",
host=os.environ["DB_NAME"],
port=27017,
authentication_source="admin",
)
def global_init():
mongoengine.register_connection(
alias="core",
name=MEALIE_DB_NAME,
host=DB_HOST,
port=int(DB_PORT),
username=DB_USERNAME,
password=DB_PASSWORD,
authentication_source="admin",
)
def test_register_connection(self):
"""Ensure that connections with different aliases may be registered.
"""
register_connection('testdb', 'mongoenginetest2')
self.assertRaises(ConnectionError, get_connection)
conn = get_connection('testdb')
self.assertTrue(isinstance(conn, pymongo.mongo_client.MongoClient))
db = get_db('testdb')
self.assertTrue(isinstance(db, pymongo.database.Database))
self.assertEqual(db.name, 'mongoenginetest2')
def setup(self):
mongoengine.register_connection(mongoengine.DEFAULT_CONNECTION_NAME, TEST_DB_NAME, DB_HOST, DB_PORT)
# TODO
# this code is using pymongo
# maybe mongoengine have the same code
db = _get_db()
for name in db.collection_names(False):
db[name].remove()
self.cover = tempfile.TemporaryFile()
self.cover.write('cover')
self.cover.seek(0)
self.audio = tempfile.TemporaryFile()
self.audio.write('audio')
self.audio.seek(0)
def config():
import mongoengine
print '[config] start', datetime.datetime.now()
try:
mongoengine.connect('miao_fm', host=MONGODB_URL, port=MONGODB_PORT)
mongoengine.register_connection('miao_fm_cdn', 'miao_fm_cdn', host=MONGODB_URL, port=MONGODB_PORT)
except mongoengine.connection.ConnectionError:
print '[Error] Can\'t connect to MongoDB!'
os._exit(-1)
print '[user] add admin...'
add_user_admin()
print '[music] add demo music...'
add_demo_music()
print '[status] gen init status...'
update_init_status()
print '[config] finish', datetime.datetime.now()
def init_app(self, app):
conn_settings = {
'db': app.config.get('MONGODB_DB', None),
'username': app.config.get('MONGODB_USERNAME', None),
'password': app.config.get('MONGODB_PASSWORD', None),
'host': app.config.get('MONGODB_HOST', None),
'port': int(app.config.get('MONGODB_PORT') or 0) or None
}
conn_settings = dict([(k, v) for k, v in conn_settings.items() if v])
# lazy connection
mongoengine.register_connection(
mongoengine.DEFAULT_CONNECTION_NAME,
conn_settings.pop('db', None),
**conn_settings
)
def global_init(user=None, password=None, port=27017, server='localhost', use_ssl=True):
if user or password:
data = dict(
username=user,
password=password,
host=server,
port=port,
authentication_source='admin',
authentication_mechanism='SCRAM-SHA-1',
ssl=use_ssl,
ssl_cert_reqs=ssl.CERT_NONE)
mongoengine.register_connection(alias='core', name='demo_dealership', **data)
data['password'] = '******'
print(" --> Registering prod connection: {}".format(data))
else:
print(" --> Registering dev connection")
mongoengine.register_connection(alias='core', name='demo_dealership')
def main():
register_connection(alias="default", name=TARGET_DB_NAME)
with switch_db(Shred, "default") as TargetShred, \
switch_db(Cluster, "default") as TargetCluster:
client = pymongo.MongoClient()
source_db = client[SOURCE_DB_NAME]
for src_shred in source_db.shreds.find({}):
new_shred, cluster = transform_shred(src_shred)
shred_obj = TargetShred.objects.create(**new_shred)
cluster_member = ClusterMember(shred=shred_obj, position=[0, 0],
angle=0)
cluster['members'] = [cluster_member]
cluster_obj = TargetCluster.objects.create(**cluster)
def __init__(self, app, user_repr=None):
app.before_request(self.track_before)
app.after_request(self.track_after)
app.wsgi_app = WSGICopyBody(app.wsgi_app)
register_connection('tracking_db',name=app.config.get('TRACKING_DB_NAME','tracking_db'),host=app.config.get('DB_URL','localhost'),port=app.config.get('DB_PORT',27017))
self.hostname = socket.gethostname()
self.max_body_length = app.config.get('TRACKING_MAX_BODY_LENGTH', 64*1024)
self.exclude_paths = app.config.get('TRACKING_EXCLUDE', [])
self.exclude_body_paths = app.config.get('TRACKING_EXCLUDE_BODY', [])
self.exclude_status_codes = app.config.get('TRACKING_EXCLUDE_STATUS_CODES', [])
self.min_execution_time = app.config.get('TRACKING_MIN_EXECUTION_TIME', 0)
self.table_size = app.config.get('TRACKING_TABLE_SIZE', 100*1024*1024)
documents.Tracking._meta['max_size'] = self.table_size
if user_repr:
documents.Tracking.user_repr = user_repr
def register_database(alias, host, dbname, replica_set=None,
username=None, password=None, auth_db=None,
logger=None):
if alias in _connection_settings:
# already registered
return
kwargs = dict()
if replica_set:
# the very presence of a replicaSet argument to 'register_connection' triggers the behavior
kwargs['replicaSet'] = replica_set
# MongoEngine converts 'host' to 'hosts_or_uri' for replica sets
kwargs['host'] = host
else:
hostname, _, port = host.partition(':')
kwargs['host'] = hostname
if port:
kwargs['port'] = int(port)
register_connection(
alias=alias,
name=dbname,
username=username,
password=password,
**kwargs)
# TODO: handle auth_db
# 'get_connection' has the side effect of creating and caching the
# connection on the first time that it's called
connection = get_connection(alias)
# automatically retry any queries that get disconnected. using exponential
# backoff for up to 60 seconds
if logger is None:
logger = current_app.logger
connection = MongoProxy(connection, logger=logger, wait_time=60)
# subsequent calls to 'get_connection' will return from the cache, so update
# the cached connection
_connections[alias] = connection
def setup_database(self):
# Disconnect from the default mongo db, and use a test db instead.
self.disconnect_dbs()
connection.connect(rhic_serve,
alias='default', tz_aware=True)
register_connection(rhic_serve, rhic_serve)
register_connection(checkin_service, checkin_service)
register_connection(report, report)
register_connection('default', rhic_serve)
for collection in ['rhic', 'account', 'user', 'fs.chunks']:
#print 'importing %s collection' % collection
call(['mongoimport', '--db', rhic_serve,
'-c', collection, '--file',
'%s.json' % os.path.join(settings.DUMP_DIR, collection)],
stdout=PIPE, stderr=PIPE)
for collection in ['splice_server', 'product']:
#print 'importing %s collection' % collection
call(['mongoimport', '--db', checkin_service,
'-c', collection, '--file',
'%s.json' % os.path.join(settings.DUMP_DIR, collection)],
stdout=PIPE, stderr=PIPE)
def register_connections():
# OPAC PROC
logger.debug(u'Registrando conexão - {db}: mongo://{host}:{port}/{db}'.format(
**config.MONGODB_SETTINGS))
opac_proc_db_credentials = get_connection_credentials(config.MONGODB_SETTINGS)
opac_proc_db_name = get_opac_proc_db_name()
register_connection(opac_proc_db_name, **opac_proc_db_credentials)
# OPAC WEBAPP
logger.debug(u'Registrando conexão - {db}: mongo://{host}:{port}/{db}'.format(
**config.OPAC_MONGODB_SETTINGS))
opac_webapp_db_credentials = get_connection_credentials(config.OPAC_MONGODB_SETTINGS)
opac_webapp_db_name = get_opac_webapp_db_name()
register_connection(opac_webapp_db_name, **opac_webapp_db_credentials)
# OPAC PROC LOGS
logger.debug(u'Registrando conexão - {db}: mongo://{host}:{port}/{db}'.format(
**config.OPAC_PROC_LOG_MONGODB_SETTINGS))
opac_logs_db_credentials = get_connection_credentials(config.OPAC_PROC_LOG_MONGODB_SETTINGS)
opac_logs_db_name = get_opac_logs_db_name()
register_connection(opac_logs_db_name, **opac_logs_db_credentials)
'''
Created on 2015年8月13日
@author: likaiguo
不用list存储倒排索引,用单一键
'''
import datetime
import json
from mongoengine import Document, IntField
from mongoengine import StringField, ListField, DateTimeField
from mongoengine import register_connection
from config import simhash_mongodb_config
register_connection(**simhash_mongodb_config)
print 'simhash db\n', json.dumps(simhash_mongodb_config, indent=4)
class SimHashCache(Document):
obj_id = StringField()
hash_value = StringField() # OverflowError: MongoDB can only handle up to 8-byte ints
text = StringField()
hash_type = StringField() # 对应不同类型的hash来源,如:简历,职位描述
dup_obj_ids = ListField(StringField())
dup_count = IntField(default=0)
add_time = DateTimeField(default=datetime.datetime.now())
update_time = DateTimeField(default=datetime.datetime.now())
meta = {
MONGODB_HOST = os.environ.get('MONGODB_HOST')
MONGODB_PORT = int(os.environ.get('MONGODB_PORT', 27017))
MONGODB_NAME_ARCHIVE = os.environ.get('MONGODB_NAME_ARCHIVE')
MONGODB_HOST_ARCHIVE = os.environ.get('MONGODB_HOST_ARCHIVE')
MONGODB_PORT_ARCHIVE = int(os.environ.get('MONGODB_PORT_ARCHIVE', 27017))
mongoengine.connect(MONGODB_NAME,
host=MONGODB_HOST,
port=MONGODB_PORT,
tz_aware=USE_TZ)
# http://mongoengine-odm.readthedocs.org/en/latest/apireference.html#connecting
mongoengine.register_connection('archive', # alias
MONGODB_NAME_ARCHIVE, # name
host=MONGODB_HOST_ARCHIVE,
port=MONGODB_PORT_ARCHIVE,
tz_aware=USE_TZ)
DBCACHE_SERVER = os.environ.get('DBCACHE_SERVER', None)
if DBCACHE_SERVER is None:
raise RuntimeError('DBCACHE_SERVER setting must be defined '
'before common.py inclusion!')
REDIS_HOST = os.environ.get('REDIS_HOST', DBCACHE_SERVER)
REDIS_PORT = int(os.environ.get('REDIS_PORT', 6379))
REDIS_DB = int(os.environ.get('REDIS_DB'))
REDIS_TEST_HOST = os.environ.get('REDIS_TEST_HOST', DBCACHE_SERVER)
def init_mongodb():
mongoengine.register_connection(alias='settings', name='apps_settings', host='userver', port=27017)
mongoengine.register_connection(alias='NewsDb', name='ticker3_db', host='userver', port=27017)
mongoengine.register_connection(alias='ticker', name='ticker3_db', host='userver', port=27017)
# coding:utf-8
'''
Created on 2015年11月5日
@author: likaiguo
'''
from mongoengine import (Document,
StringField,
)
from mongoengine import register_connection
from config import mongodb_config
register_connection(**mongodb_config)
class Feed(Document):
"""
@summary: 职位描述
"""
keywords = StringField(default='')
job_type = StringField(default='') # 工作类型
talent_level = StringField(default='') # 人才级别
expect_area = StringField(default='') # 期望工作地
job_desc = StringField(default='') # 职位描述
job_url = StringField(default='')
if __name__ == '__main__':
feed = Feed.objects.first()
print(feed)
from mongoengine import connect
from mongoengine import register_connection
from settings import *
try:
from settings_dev import *
except:
pass
host = OTHER_DATABASE.get('mongo').get('host')
username = OTHER_DATABASE.get('mongo').get('user')
password = OTHER_DATABASE.get('mongo').get('password')
port = OTHER_DATABASE.get('mongo').get('port')
replicaset = OTHER_DATABASE.get('mongo').get('replicaset')
tag_sets = OTHER_DATABASE.get('mongo').get('tag_sets')
if replicaset:
from pymongo import ReadPreference
mongo_conn = connect(
'recruiting',
host=host,
replicaSet=replicaset,
read_preference=ReadPreference.SECONDARY_PREFERRED,
connectTimeoutMS=30000,
tag_sets=tag_sets,
)
else:
connect('recruiting', host=host, username=username, password=password, port=int(port))
register_connection(**SPIDERS_MONGO)
#'djcelery',
#'rhic_serve.rhic_rcs',
#'report_server.report_import',
# End of SAM integration change.
)
##
## Adding mongoengine specifics ##
##
import mongoengine
MONGO_DATABASE_NAME = config.CONFIG.get('server', 'db_name')
MONGO_DATABASE_HOST = config.CONFIG.get('server', 'db_host')
MONGO_DATABASE_PORT = config.CONFIG.get('server', 'db_port')
mongoengine.connect(MONGO_DATABASE_NAME, host="%s:%s" % (MONGO_DATABASE_HOST, MONGO_DATABASE_PORT), tz_aware=True)
mongoengine.register_connection("rhic_serve", MONGO_DATABASE_NAME)
AUTHENTICATION_BACKENDS = (
'mongoengine.django.auth.MongoEngineBackend',
)
SESSION_ENGINE = 'mongoengine.django.sessions'
##
## End mongoengine specifics
##
#############################
# Celery Configuration
#
# Only configured if celery is enabled in the config file.
CELERY_ENABLED = config.CONFIG.getboolean('celery', 'enabled')
"""All mongoengine models are should be described here."""
from datetime import datetime
from makechat import config as settings
from mongoengine import Document, StringField, ReferenceField, \
BooleanField, EmailField, DateTimeField, ListField, CASCADE, PULL, \
register_connection
register_connection(
alias='makechat', host=settings.get('DEFAULT', 'mongo_uri'),
connect=False)
register_connection(
alias='makechat_test', host=settings.get('DEFAULT', 'test_mongo_uri'),
connect=False)
TEST_MODE = settings.getboolean('DEFAULT', 'test_mode')
SESSION_TTL = settings.getint('DEFAULT', 'session_ttl')
USER_ROLES = (
('admin', 'Superuser'), # can create chat rooms and manage chat members
('owner', 'Chat owner'), # can read/write and manage chat members
('member', 'Chat member'), # can read/write, but can't manage chat members
('guest', 'Chat guest'), # can read, but can't write
)
class MetaData(Document):
"""Base class for all models.
The goal is to have metadata for all objects and do not repeat itself.
"""
def init_application(self):
self.debug = self.settings.getboolean('server', 'debug')
# Work around a flask issue: https://github.com/pallets/flask/issues/1907
if self.debug:
self.jinja_env.auto_reload = True
self.logger_formatter = logging.Formatter(
'%(asctime)s [%(process)5d] [%(name)20.20s] %(levelname)8s - %(message)s '
'[in %(pathname)s:%(lineno)d]'
)
self.logger.setLevel(logging.DEBUG)
log_handler = logging.StreamHandler()
log_handler.setLevel(logging.DEBUG)
if sys.stdout.isatty():
from termcolor import colored
class ColoredFormatter(logging.Formatter):
def format(self, record):
levelname = record.levelname
formatted = logging.Formatter.format(self, record)
if levelname == 'INFO':
return colored(formatted, 'green')
if levelname in ['WARNING', 'ERROR', 'CRITICAL']:
return colored(formatted, 'red')
return formatted
log_handler.setFormatter(ColoredFormatter(
'%(asctime)s [%(process)5d] [%(name)20.20s] %(levelname)8s - %(message)s '
'[in %(pathname)s:%(lineno)d]'
))
else:
log_handler.setFormatter(self.logger_formatter)
# Set the root handler, and clear the app-specific one
logging.root.handlers = [] # When init_application is called more than once, we don't want to acrue handlers
logging.root.addHandler(log_handler)
logging.root.setLevel(logging.DEBUG)
self.logger.handlers = []
werkzeug_logger = logging.getLogger('werkzeug')
werkzeug_logger.setLevel(logging.INFO)
werkzeug_logger.propagate = False
werkzeug_logger.addHandler(logging.StreamHandler())
logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.WARNING)
# Identity does this, we don't want whatever flask might be doing
self.config['WTF_CSRF_ENABLED'] = False
server_url = furl(self.settings.get('server', 'url'))
self.config['SERVER_NAME'] = server_url.netloc
self.config['SESSION_COOKIE_DOMAIN'] = server_url.host
if server_url.scheme == 'https':
self.config['SESSION_COOKIE_SECURE'] = True
self.config['PREFERRED_URL_SCHEME'] = 'https'
def add_security_headers(response):
# See https://en.wikipedia.org/wiki/Strict_Transport_Security
if 'Strict-Transport-Security' not in response.headers:
response.headers.set('Strict-Transport-Security', 'max-age=31536000')
if 'X-Frame-Options' not in response.headers:
if self.settings.get('security','frames') == 'deny':
response.headers.set('X-Frame-Options','DENY')
elif self.settings.get('security', 'frames') == 'sameorigin':
response.headers.set('X-Frame-Options','SAMEORIGIN')
return response
self.after_request(add_security_headers)
mongo_url = furl(self.settings.get('mongo', 'url'))
mongo_db = mongo_url.path.segments[0]
if 'username' in self.settings.options('mongo') and 'password' in self.settings.options('mongo'):
mongo_url.username = self.settings.get('mongo', 'username')
mongo_url.password = self.settings.get('mongo', 'password')
self.db = pymongo.MongoClient(str(mongo_url))[mongo_db]
mongoengine.register_connection('default', mongo_db, host=str(mongo_url))
mongoengine.connection.get_db(reconnect=True)
register_app(self)
'username': '',
'password': '',
'tz_aware': True,
# 'test': {
# 'name': 'test_mysite'
# }
}
}
SESSION_ENGINE = 'django_mongoengine.sessions'
SESSION_SERIALIZER = 'django_mongoengine.sessions.BSONSerializer'
TEST_FILE_ROOT = os.path.join(STATIC_ROOT, "test")
AUTH_USER_MODEL = 'mongo_auth.MongoUser'
MONGOENGINE_USER_DOCUMENT = 'mysite.myapp.models.MyUser'
AUTHENTICATION_BACKENDS = (
'django_mongoengine.mongo_auth.backends.MongoEngineBackend',
)
from mongoengine import register_connection
register_connection('default', 'mysite', host = 'localhost', port = 27017)
register_connection('test', 'test_mysite', host = 'localhost', port = 27017)
TEST_MONGO_DATABASE = {
'db': 'test_mysite',
'host': ['localhost'],
'port': 27017,
}
