class CEICInfoUpdater:
def __init__(self, mongo=None, redis=None):
if mongo is None:
self._mongo = MonCollection(mongodb=MongoDB(),
database='region',
collection_name='CEIC')
else:
self._mongo = mongo
if redis is None:
self._redis = Redis()
else:
self._redis = redis
def __call__(self):
ceic_period = sorted(self._mongo.distinct('year'))
ceic_variable = sorted(self._mongo.distinct('variable'))
ceic_acode = sorted(self._mongo.distinct('acode'))
ceic_region = [
self._mongo.collection.find_one({'acode': acode},
projection={
'_id': False,
'region': True
})['region']
for acode in ceic_acode
]
#update
self._redis.set('ceic_acode', ceic_acode)
self._redis.set('ceic_period', ceic_period)
self._redis.set('ceic_variable', ceic_variable)
self._redis.set('ceic_region', ceic_region)
class AirQualityUpdater:
def __init__(self, mongo=None, redis=None):
if mongo is None:
self._mongo = MonCollection(mongodb=MongoDB(),
database='scraperdata',
collection_name='airqualityfromMin')
else:
self._mongo = mongo
if redis is None:
self._redis = Redis()
else:
self._redis = redis
def __call__(self):
airquality_citycode = sorted(self._mongo.distinct('CITYCODE'))
airquality_city = [
self._mongo.collection.find_one({'CITYCODE': ccode},
projection={
'_id': False,
'CITY': True
})['CITY']
for ccode in airquality_citycode
]
airquality_date = ([
date.strftime("%y-%m-%d")
for date in sorted(self._mongo.distinct('OPER_DATE'))
])
# update
self._redis.set('airquality_city', airquality_city)
self._redis.set('airquality_citycode', airquality_citycode)
self._redis.set('airquality_date', airquality_date)
def backup(source, target):
for database in backup_databases:
if database not in source.database_names:
print('No Database named {} remotely!'.format(database))
raise Exception
if database not in target.database_names:
print(source.database_names)
print('No Database name {} locally!'.format(database))
raise Exception
source_database = MonDatabase(source, database_name=database)
target_database = MonDatabase(target, database_name=database)
for collection in source_database.collection_names:
if collection not in target_database.collection_names:
target_database.create_collection(collection)
else:
source_collection = MonCollection(source, source_database,
collection)
target_collection = MonCollection(target, target_database,
collection)
if source_collection.collection.count(
) <= target_collection.collection.count():
print('No need to update the collection: {}'.format(
collection))
continue
else:
print('updating...{}'.format(collection))
target_collection.collection.insert_many(
source_collection.collection.find())
def __init__(self,
mongodb='mongodb://*****:*****@123.207.185.126:27017/',
database='proxy',
collection_name='proxys'):
# 设置数据库
self._conn = MonCollection(mongodb=mongodb,
database=database,
collection_name=collection_name)
def __init__(self):
""" 初始化中国城市统计数据库接口
"""
mongo = MongoDB(
conn_str='mongodb://*****:*****@123.207.185.126:27017/')
self.conn = MonCollection(mongo,
database='region',
collection_name='citystatistics').collection
def __init__(self, mongo=None, redis=None):
if mongo is None:
self._mongo = MonCollection(mongodb=MongoDB(),
database='scraperdata',
collection_name='airqualityfromMin')
else:
self._mongo = mongo
if redis is None:
self._redis = Redis()
else:
self._redis = redis
def __init__(self, mongo=None, redis=None):
if mongo is None:
self._mongo = MonCollection(mongodb=MongoDB(),
database='region',
collection_name='CEIC')
else:
self._mongo = mongo
if redis is None:
self._redis = Redis()
else:
self._redis = redis
def backup(source,target):
for database in backup_databases:
if database not in source.database_names:
print('No Database named {} remotely!'.format(database))
raise Exception
if database not in target.database_names:
print('No Database name {} locally!'.format(database))
raise Exception
source_database = MonDatabase(source, database_name=database)
target_database = MonDatabase(target, database_name=database)
for collection in source_database.collection_names:
if collection not in target_database.collection_names:
target_database.create_collection(collection)
source_collection = MonCollection(source,source_database,collection)
target_collection = MonCollection(target,target_database,collection)
for record in source_collection.find():
target_collection.collection.insert_one(record)
class CityStatisticsDatabase:
def __init__(self):
""" 初始化中国城市统计数据库接口
"""
mongo = MongoDB(
conn_str='mongodb://*****:*****@123.207.185.126:27017/')
self.conn = MonCollection(mongo,
database='region',
collection_name='citystatistics').collection
def find(self, *args, **kwargs):
""" 调用查询接口
:param args:
:param kwargs:
:return:
"""
found = list(self.conn.find(*args, **kwargs))
if len(found) > 0:
found_data = pd.DataFrame(found)
found_data['var'] = found_data['variable'] + found_data[
'unit'].apply(lambda x: ''.join(['(', x, ')']))
pdata = pd.pivot_table(found_data,
values='value',
index=['year', 'acode', 'region'],
columns=['var'])
pdata = pdata.swaplevel(0, 1, axis=0)
return pdata
@property
def variables(self):
found = self.conn.find().distinct('variable')
return pd.DataFrame(sorted(found))
@property
def regions(self):
return None
class AdminDivisionDatabase():
""" 类AdminDivisionDatabase连接admindivision集合
"""
def __init__(self):
# 连接admindivision集合
mongo = MongoDB()
mdb = MonDatabase(mongodb=mongo, database_name='region')
self.collection = MonCollection(database=mdb,
collection_name='admindivision')
# 查询
def find(self, **conds):
# 设置projection
projection = conds.get('projection')
if projection is None:
projection = {
'region': 1,
'year': 1,
'adminlevel': 1,
'acode': 1,
'_id': 1,
'parent': 1,
'uid': 1
}
else:
conds.pop('projection')
# 设置sorts
sorts = conds.get('sorts')
if sorts is None:
sorts = [('year', ASCENDING), ('acode', ASCENDING)]
else:
conds.pop('sorts')
# 设置查询条件
condition = dict()
for key in conds:
if isinstance(conds[key], list):
condition[key] = {'$in': conds[key]}
else:
condition[key] = conds[key]
# 返回查询结果
return self.collection.find(condition, projection).sort(sorts)
# 年份
@property
def period(self):
return sorted(self.find().distinct('year'))
import pandas as pd
from sheldon.database.class_mongodb import MongoDB, MonDatabase, MonCollection
# 0. Initialization
OUTPUT_PATH = r'D:\data\output\mongodb_info.xlsx'
# 1. 连接数据库
mongo = MongoDB(conn_str='mongodb://*****:*****@123.207.185.126:27017/')
# 2. 数据库信息列表
database_info = []
for database_name in sorted(mongo.database_names):
database_con = MonDatabase(mongo,database_name=database_name)
if len(database_con.collection_names) > 0:
for collection_name in sorted(database_con.collection_names):
conn = MonCollection(mongo,database=database_name,collection_name=collection_name)
database_info.append([database_name, collection_name, conn.collection.count()])
else:
database_info.append([database_name,None,None])
database_info_pdrame = pd.DataFrame(database_info,columns=['database','collection','count'])
#database_info_pdrame.to_excel(OUTPUT_PATH)
def __init__(self):
# 连接admindivision集合
mongo = MongoDB()
mdb = MonDatabase(mongodb=mongo, database_name='region')
self.collection = MonCollection(database=mdb,
collection_name='admindivision')
# coding = UTF-8
import os
import numpy as np
import pandas as pd
import re
from sheldon.database.class_mongodb import MongoDB, MonDatabase, MonCollection
# 0. Initialize
PATH = r'D:\data\cnki\forimport'
# 1. connect to db
mongo = MongoDB(conn_str='mongodb://*****:*****@123.207.185.126:27017/')
mdb = MonDatabase(mongodb=mongo, database_name='papers')
con = MonCollection(mongo,mdb,collection_name='cnki').collection
for file in os.listdir(PATH):
file_name = os.path.join(PATH,file)
mdata = pd.read_excel(file_name)
records = mdata.to_dict(orient='index')
for num in records:
db_record = dict()
# 标题
db_record['title'] = records[num]['Title']
# 作者
if isinstance(records[num]['Author'],str):
db_record['author'] = [author for author in re.split(';',records[num]['Author']) if re.match('^\s*$',author) is None]
else:
continue
# 地址
if isinstance(records[num]['Author Address'],str):
import aiohttp
import asyncio
import async_timeout
from bs4 import BeautifulSoup
import json
from sheldon.webscraper.class_proxymanager import ProxyManager
from sheldon.database.class_mongodb import MongoDB, MonDatabase, MonCollection
conn = MonCollection(
mongodb='mongodb://*****:*****@123.207.185.126:27017/',
database='scraperdata',
collection_name='airqualityfromMin').collection
async def get_web(session, url):
with async_timeout.timeout(10):
async with session.get(url) as response:
return await response.text()
async def post_web(session, url, pagenum):
try:
pdata = {
'page.pageNo': pagenum,
'xmlname': '1462259560614',
'V_DATE': '2014-01-01',
'E_DATE': '2017-08-20'
}
proxy = ProxyManager().random_proxy
print('Start Proxy... ', proxy)
async with session.post(url,
# coding = UTF-8
import pandas as pd
from sheldon.database.class_mongodb import MongoDB, MonDatabase, MonCollection
# 1. 连接数据库
mongodb = 'mongodb://*****:*****@123.207.185.126:27017/'
database = 'papers'
collection_name = 'csscijournals'
conn = MonCollection(mongodb=mongodb,
database=database,
collection_name=collection_name)
# 2. 导入杂志文件
journals = pd.read_excel('d:/data/cnki/economic_journals.xlsx')
for item in journals.to_dict(orient='record'):
print(item)
conn.collection.insert_one(item)
# coding = UTF-8
from sheldon.database.class_mongodb import MongoDB, MonDatabase, MonCollection
from pymongo import MongoClient, DESCENDING, ASCENDING
import pandas as pd
from datetime import datetime, date
mongodb = MongoDB()
database_name = 'scraperdata'
collection_name = 'airqualityfromMin'
moncollection = MonCollection(mongodb=mongodb,
database=database_name,
collection_name=collection_name).collection
def query(start_date, end_date):
print(type(start_date), end_date)
found = list(
moncollection.find(
{
'$and': [{
'OPER_DATE': {
'$gte': start_date
}
}, {
'OPER_DATE': {
'$lte': end_date
}
}]
},
projection={
class ProxyManager:
""" ProxyManager类用来管理、检验和更新代理服务器列表
:param str proxy_web: proxy的地址，默认为http://www.youdaili.net/Daili/guonei/
:return: 无返回值
"""
def __init__(self,
mongodb='mongodb://*****:*****@123.207.185.126:27017/',
database='proxy',
collection_name='proxys'):
# 设置数据库
self._conn = MonCollection(mongodb=mongodb,
database=database,
collection_name=collection_name)
def find(self, type=0, limit=None):
if type == 0:
found = self._conn.find(filter={'protocol': {
'$in': [0, 2]
}},
projection={
'_id': False,
'ip': True,
'port': True,
'type': True
},
sort=[('score', DESCENDING),
('speed', ASCENDING)],
limit=limit)
else:
found = self._conn.find(filter={'protocol': {
'$in': [1, 2]
}},
projection={
'_id': False,
'ip': True,
'port': True,
'type': True
},
sort=[('score', DESCENDING),
('speed', ASCENDING)],
limit=limit)
return {
Proxy(ip=item['ip'], port=item['port'], type=type).address
for item in found
}
@property
def random_proxy(self):
""" 随机返回一个代理服务器，选择的权重是它的count
:return: 随机返回一个代理服务器
"""
return random.choice(list(self.find(limit=40)))
@property
def top_50_proxies(self):
""" 随机返回一个代理服务器，选择的权重是它的count
:return: 随机返回一个代理服务器
"""
return list(self.find(limit=50))
'region': True
})['region']
for acode in ceic_acode
]
#update
self._redis.set('ceic_acode', ceic_acode)
self._redis.set('ceic_period', ceic_period)
self._redis.set('ceic_variable', ceic_variable)
self._redis.set('ceic_region', ceic_region)
class MongoDBInfoUpdater:
updates = {'ceic': CEICInfoUpdater, 'airquality': AirQualityUpdater}
def __init__(self, to_be_update=['airquality']):
self._to_be_update = list(to_be_update)
def __call__(self, *args, **kwargs):
for to_be_update in self._to_be_update:
print('update: ', to_be_update)
self.updates[to_be_update]()()
if __name__ == '__main__':
mcollection = MonCollection(mongodb=MongoDB(),
database='region',
collection_name='CEIC')
updater = MongoDBInfoUpdater()
updater()
# coding = UTF-8
from sheldon.database.class_mongodb import MongoDB, MonDatabase, MonCollection
# 1. 连接数据库
mongo = MongoDB(conn_str='mongodb://*****:*****@123.207.185.126:27017/')
mdb = MonDatabase(mongodb=mongo, database_name='papers')
con = MonCollection(mongo, mdb, collection_name='cnki').collection
journals = con.find().distinct('journal')
i = 1
for journal in journals:
period = con.find({'journal': journal}).distinct('year')
if len(period) < 8:
print(journal)
#raise Exception
print(journal, '->', sorted(period))
#print(i)
i += 1
from bokeh.plotting import figure, output_file, show
from bokeh.models import ColumnDataSource
from bokeh.transform import factor_cmap
from bokeh.palettes import Spectral6
# output to static HTML file
output_file("chart.html")
# 0. Setup
BASE_PATH = r'D:\data\cnki\pkl'
XLS_PATH = r'D:\data\cnki\xls'
# 1. 数据库
mongo = MongoDB(conn_str='mongodb://*****:*****@123.207.185.126:27017/')
mdb = MonDatabase(mongodb=mongo, database_name='papers')
con = MonCollection(mongo, mdb, collection_name='cnki').collection
journals = sorted(con.find().distinct('journal'))
period = [str(year) for year in range(2010, 2018)]
ORIGINAL_AUTHOR_OUTPUT = False
BUILD_AUTHOR_DICT = False
ORGANIZE_AUTHOR = False
RECORD_DATAFRAME = False
CREATE_WEIGHT = False
CREATE_WEIGHT2 = False
DEA = False
DEA2 = True
# 是否同一个地址
# coding = UTF-8
from sheldon.database.class_mongodb import MongoDB, MonDatabase, MonCollection
mongodb = MongoDB()
database_name = 'papers'
collection_name = 'cnki'
conn = MonCollection(mongodb=mongodb,
database=database_name,
collection_name=collection_name).collection
# 作者信息：{姓名：{年份：address}}
unmatched_records = []
matched_authors = dict()
for record in conn.find():
if (record.get('author') is not None) and (record.get('address')
is not None):
if len(record.get('author')) == len(record.get('address')):
for author in record.get('author'):
if author in matched_authors.keys():
pass
else:
matched_authors.update({
author:
set(record.get('year'), record.get('address'))
})
else:
pass
# coding = UTF-8
from sheldon.database.class_mongodb import MongoDB, MonDatabase, MonCollection
from pymongo import MongoClient, DESCENDING, ASCENDING
import re
from datetime import datetime, date
mongodb = MongoDB()
database_name = 'scraperdata'
collection_name = 'airqualityfromMin'
moncollection = MonCollection(mongodb=mongodb,
database=database_name,
collection_name=collection_name).collection
found_all = moncollection.find()
for item in found_all:
if isinstance(item['OPER_DATE'], str):
id = item['_id']
print(item['OPER_DATE'])
#oper_date = datetime(*[int(i) for i in re.split('-',item['OPER_DATE'])])
#moncollection.update_one({'_id':id},{'$set':{'OPER_DATE':oper_date}})