def __init__(self):
""" 初始化中国城市统计数据库接口
"""
mongo = MongoDB(
conn_str='mongodb://*****:*****@123.207.185.126:27017/')
self.conn = MonCollection(mongo,
database='region',
collection_name='citystatistics').collection
def __init__(self, mongo=None, redis=None):
if mongo is None:
self._mongo = MonCollection(mongodb=MongoDB(),
database='region',
collection_name='CEIC')
else:
self._mongo = mongo
if redis is None:
self._redis = Redis()
else:
self._redis = redis
def __init__(self, mongo=None, redis=None):
if mongo is None:
self._mongo = MonCollection(mongodb=MongoDB(),
database='scraperdata',
collection_name='airqualityfromMin')
else:
self._mongo = mongo
if redis is None:
self._redis = Redis()
else:
self._redis = redis
# coding = UTF-8
from sheldon.database.class_mongodb import MongoDB, MonDatabase, MonCollection
# 1. 连接数据库
mongo = MongoDB(conn_str='mongodb://*****:*****@123.207.185.126:27017/')
mdb = MonDatabase(mongodb=mongo, database_name='papers')
con = MonCollection(mongo, mdb, collection_name='cnki').collection
journals = con.find().distinct('journal')
i = 1
for journal in journals:
period = con.find({'journal': journal}).distinct('year')
if len(period) < 8:
print(journal)
#raise Exception
print(journal, '->', sorted(period))
#print(i)
i += 1
def __init__(self):
# 连接admindivision集合
mongo = MongoDB()
mdb = MonDatabase(mongodb=mongo, database_name='region')
self.collection = MonCollection(database=mdb,
collection_name='admindivision')
# coding = UTF-8
from sheldon.database.class_mongodb import MongoDB, MonDatabase, MonCollection
LOCAL_TO_REMOTE = False
REMOTE_TO_LOCAL = True
backup_databases = ['region','microdata','papers','proxy','scraperdata','statsgov']
remote_mongo = MongoDB(conn_str='mongodb://*****:*****@123.207.185.126:27017/')
local_mongo = MongoDB(conn_str=None)
def backup(source,target):
for database in backup_databases:
if database not in source.database_names:
print('No Database named {} remotely!'.format(database))
raise Exception
if database not in target.database_names:
print('No Database name {} locally!'.format(database))
raise Exception
source_database = MonDatabase(source, database_name=database)
target_database = MonDatabase(target, database_name=database)
for collection in source_database.collection_names:
if collection not in target_database.collection_names:
target_database.create_collection(collection)
source_collection = MonCollection(source,source_database,collection)
@author: glen
@date: 2017.12.23
@tag: mongodb backup
"""
from sheldon.database.class_mongodb import MongoDB, MonDatabase, MonCollection
LOCAL_TO_REMOTE = True
REMOTE_TO_LOCAL = False
backup_databases = [
'region', 'microdata', 'papers', 'proxy', 'scraperdata', 'statsgov',
'webdata'
]
local_mongo = MongoDB(
conn_str='mongodb://*****:*****@123.207.185.126:27017/')
remote_mongo = MongoDB(
conn_str=
'mongodb://*****:*****@dds-bp162bb74b8184e41658-pub.mongodb.rds.aliyuncs.com:3717,dds-bp162bb74b8184e42438-pub.mongodb.rds.aliyuncs.com:3717/admin?replicaSet=mgset-4970435'
)
def backup(source, target):
for database in backup_databases:
if database not in source.database_names:
print('No Database named {} remotely!'.format(database))
raise Exception
if database not in target.database_names:
print(source.database_names)
print('No Database name {} locally!'.format(database))
# coding = UTF-8
from sheldon.database.class_mongodb import MongoDB, MonDatabase, MonCollection
mongodb = MongoDB()
database_name = 'papers'
collection_name = 'cnki'
conn = MonCollection(mongodb=mongodb,
database=database_name,
collection_name=collection_name).collection
# 作者信息：{姓名：{年份：address}}
unmatched_records = []
matched_authors = dict()
for record in conn.find():
if (record.get('author') is not None) and (record.get('address')
is not None):
if len(record.get('author')) == len(record.get('address')):
for author in record.get('author'):
if author in matched_authors.keys():
pass
else:
matched_authors.update({
author:
set(record.get('year'), record.get('address'))
})
else:
pass
'region': True
})['region']
for acode in ceic_acode
]
#update
self._redis.set('ceic_acode', ceic_acode)
self._redis.set('ceic_period', ceic_period)
self._redis.set('ceic_variable', ceic_variable)
self._redis.set('ceic_region', ceic_region)
class MongoDBInfoUpdater:
updates = {'ceic': CEICInfoUpdater, 'airquality': AirQualityUpdater}
def __init__(self, to_be_update=['airquality']):
self._to_be_update = list(to_be_update)
def __call__(self, *args, **kwargs):
for to_be_update in self._to_be_update:
print('update: ', to_be_update)
self.updates[to_be_update]()()
if __name__ == '__main__':
mcollection = MonCollection(mongodb=MongoDB(),
database='region',
collection_name='CEIC')
updater = MongoDBInfoUpdater()
updater()