def __init__(self, account_file):
self.check_paths(['log', 'pic', 'cookies', 'driver', 'account'])
self.username, self.password = self.get_account(account_file)
self.login_api = 'https://passport.weibo.cn/sso/login'
self.user_info_api = 'https://m.weibo.cn/home/me?format=cards'
self.user_weibo_api = "https://m.weibo.cn/api/container/getIndex?uid={}&luicode=20000174&type=uid&value={}&containerid=107603{}"
self.headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:56.0) Gecko/20100101 Firefox/56.0',
}
self.logger = MyLogger('weibo').get_logger()
self.cookie_path = 'cookies'
self.conn = redis.Redis(connection_pool=redis.ConnectionPool(host='redis', port=6379, decode_responses=True))
self.cookies = self.get_cookies()
def __init__(self, host, user, password, db):
self.logger = MyLogger(name='api_db', prefix='api_db')
self.host = host
self.user = user
self.passwd = password
self.db = db
try:
self.connect = mysql.connect(host,
user,
password,
db,
charset='utf8')
except Exception as e:
self.logger.error(e)
raise e
def setUp(self):
unittest.TestCase.setUp(self)
agentFactory = PoleAgentFactory()
environmentFactory = PoleEnvironmentFactory()
trainerFactory = PoleTrainerFactory()
buildParameterFactory = PoleBuildParameterFactory()
store = Store(self.dbPath)
logger = MyLogger(console_print=True)
self.builder = Builder(trainerFactory, agentFactory,
environmentFactory, store, logger)
self.buildParameters = []
for k1 in range(2):
nIntervalSave = 3
nEpoch = 5
self.buildParameters.append(
PoleBuildParameter(int(nIntervalSave),
int(nEpoch),
label="test" + str(k1)))
for agentClass in ("agent002", "agent003", "agent004"):
self.buildParameters.append(
PoleBuildParameter(int(nIntervalSave),
int(nEpoch),
agentClass=agentClass,
label="test " + agentClass))
self.loader = Loader(agentFactory, buildParameterFactory,
environmentFactory, store)
def setUp(self):
unittest.TestCase.setUp(self)
agentFactory = WaeAgentFactory()
environmentFactory = WaeEnvironmentFactory()
trainerFactory = WaeTrainerFactory()
buildParameterFactory = WaeBuildParameterFactory()
store = Store(self.dbPath, self.trainLogFolderPath)
logger = MyLogger(console_print=True)
self.builder = Builder(trainerFactory, agentFactory,
environmentFactory, store, logger)
self.buildParameters = []
for k1 in range(2):
nIntervalSave = 10
nEpoch = 20
nLayer = int(np.random.choice((1, 2)))
self.buildParameters.append(
WaeBuildParameter(int(nIntervalSave),
int(nEpoch),
label="test" + str(k1),
nLayer=nLayer,
eps_given_sinkhorn=0.1,
tol_sinkhorn=0.1))
self.loader = Loader(agentFactory, buildParameterFactory,
environmentFactory, store)
class BaseDBAdapter(object):
def __init__(self, host, user, password, db):
self.logger = MyLogger(name='api_db', prefix='api_db')
self.host = host
self.user = user
self.passwd = password
self.db = db
try:
self.connect = mysql.connect(host,
user,
password,
db,
charset='utf8')
except Exception as e:
self.logger.error(e)
raise e
def close_connect(self):
if self.connect:
self.connect.close()
def store_specified_isbn(target: Path, isbn: str, honto: HontoSearchCliant,
ehon: EhonSearchCliant, config: dict,
logger: MyLogger, db: DatabaseCliant) -> None:
"""isbnを元に元情報を上書きし、移動する
Args:
target (Path): 対象書籍のpath
isbn (str): isbn
honto (HontoSearchCliant): Hontoの検索クライアント
ehon (EhonSearchCliant): E-honの検索クライアント
config (dict): configのdict
logger (MyLogger): logger
db (DatabaseCliant): データベースのクライアント
Raises:
HontoDoesNotHaveDataError: Hontoがその書籍のページを持っていないときのエラー
"""
try:
info = fetch_book_info_from_isbn(isbn, honto, ehon)
except HontoDoesNotHaveDataError as e:
send_err_dir(target, Path(config["output_dir"]))
logger.write("ERROR", str(e))
raise HontoDoesNotHaveDataError(e)
else:
dst = construct_dst(config["output_dir"], info)
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.move(str(target), dst)
logger.write("SUCCESS", str(dst))
# データベースに情報を追加
fetch_result = {
"title": info["title"],
"isbn": info["isbn"],
"authors": "\t".join(info["authors"] or [""]),
"publishers": info["publisher"],
"categories": info["category"],
"destination": str(dst)
}
db.store(fetch_result)
def completion_no_isbn(source_csv_path: str) -> None:
"""手作業でisbnや移動先を書いたcsvを元に補完作業をする
Args:
source_csv_path (str): ソースとなるcsv
"""
logger = MyLogger()
honto = HontoSearchCliant()
ehon = EhonSearchCliant()
with open(Path(__file__).resolve().parents[1] / "config.yml") as f:
config = yaml.safe_load(f)
Path(config["output_dir"]).resolve().mkdir(exist_ok=True, parents=True)
db_cliant = DatabaseCliant(Path(config["database_path"]))
csv_path = Path(source_csv_path).resolve()
with open(csv_path) as f:
reader = csv.reader(f)
for row in reader:
book_path = Path(row[0]).resolve()
neemock = str(row[1])
if re.fullmatch(r"[0-9-]+", neemock):
isbn = neemock
try:
store_specified_isbn(book_path, isbn, honto, ehon, config,
logger, db_cliant)
except HontoDoesNotHaveDataError as e:
send_err_dir(book_path, Path(config["output_dir"]))
logger.write("ERROR", str(e))
else:
dst = Path(neemock)
db_cliant.update_dst(book_path, dst)
shutil.move(str(book_path), str(dst / book_path.name))
db_cliant.close()
def main():
logger = MyLogger()
profect_dir = Path(__file__).resolve().parents[1]
config_path = profect_dir / "config.yml"
if not config_path.exists():
generate_config_file(config_path)
with open(config_path, "r") as f:
config = yaml.safe_load(f)
Path(config["output_dir"]).mkdir(exist_ok=True, parents=True)
db_cliant = DatabaseCliant(Path(config["database_path"]))
honto_cliant = HontoSearchCliant()
ehon_cliant = EhonSearchCliant()
# input_dir内のPDFに対して処理をする
for pdf_file in Path(config["input_dir"]).glob("**/*.pdf"):
try:
book_info = fetch_book_info_from_pdf(pdf_file, honto_cliant, ehon_cliant)
except NotFoundIsbnError as e:
send_err_dir(pdf_file, Path(config["output_dir"]))
logger.write("ERROR", str(e))
continue
except HontoDoesNotHaveDataError as e:
send_err_dir(pdf_file, Path(config["output_dir"]))
logger.write("ERROR", str(e))
continue
dst = construct_dst(config["output_dir"], book_info)
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.move(str(pdf_file), dst)
logger.write("SUCCESS", dst)
# データベースに情報を追加
fetch_result = {
"title": book_info["title"],
"isbn": book_info["isbn"],
"authors": "\t".join(book_info["authors"] or [""]),
"publishers": book_info["publisher"],
"categories": book_info["category"],
"destination": str(dst)
}
db_cliant.store(fetch_result)
db_cliant.close()
def setUp(self):
unittest.TestCase.setUp(self)
agentFactory = ConcAgentFactory()
environmentFactory = ConcEnvironmentFactory()
trainerFactory = WaeTrainerFactory()
buildParameterFactory = ConcBuildParameterFactory()
store = Store(self.dbPath)
logger = MyLogger(console_print=True)
self.builder = Builder(trainerFactory, agentFactory, environmentFactory, store, logger)
nIntervalSave = 1
nEpoch = 2
self.buildParameters = []
for k1 in range(2):
self.buildParameters.append(ConcBuildParameter(int(nIntervalSave), int(nEpoch), label="test" + str(k1)))
for k1 in range(2):
self.buildParameters.append(ConcBuildParameter(int(nIntervalSave), int(nEpoch)
, label="test case study 02a " + str(k1)
, nXi = 3
, target_casestudy = "cs02a"
))
for k1 in range(2):
self.buildParameters.append(ConcBuildParameter(int(nIntervalSave), int(nEpoch)
, label="test case study 03a " + str(k1)
, nXi = 3
, target_casestudy = "cs03a"
))
for k1 in range(2):
self.buildParameters.append(ConcBuildParameter(int(nIntervalSave), int(nEpoch)
, label="test case study 03b " + str(k1)
, nXi = 3
, target_casestudy = "cs03b"
))
for k1 in range(2):
self.buildParameters.append(ConcBuildParameter(int(nIntervalSave), int(nEpoch)
, label="test case study 03c " + str(k1)
, nXi = 3
, target_casestudy = "cs03c"
))
self.loader = Loader(agentFactory, buildParameterFactory, environmentFactory, store)
def setUp(self):
unittest.TestCase.setUp(self)
agentFactory = AgentFactory()
environmentFactory = EnvironmentFactory()
trainerFactory = TrainerFactory()
buildParameterFactory = BuildParameterFactory()
store = Store(self.dbPath)
logger = MyLogger(console_print=False)
self.builder = Builder(trainerFactory, agentFactory,
environmentFactory, store, logger)
self.buildParameters = []
for k1 in range(3):
nIntervalSave = 10
nEpoch = 100
self.buildParameters.append(
BuildParameter(int(nIntervalSave),
int(nEpoch),
label="test" + str(k1)))
self.loader = Loader(agentFactory, buildParameterFactory,
environmentFactory, store)
import time
import datetime
import argparse
import torch
from torch.utils.data import DataLoader
from torchvision import datasets
from torchvision import transforms
from torch.autograd import Variable
import torch.optim as optim
import warnings
from tqdm import tqdm
from mylogger import MyLogger
local_logger = MyLogger(filename='./logs/train.log',logger_name='train')
warnings.filterwarnings('ignore')
def img_label_map(img_list):
root_path = str(os.getcwd())
local_path = 'data/labels'
labels_dir = os.path.join(root_path,local_path)
label_file_list = []
for img_path in img_list:
img_path = img_path.strip()
temp_path = img_path.split('/')[-1].replace('.jpg','.txt')
label_file_list.append(os.path.join(labels_dir,temp_path))
return label_file_list
# -*- coding:utf-8 -*-
import requests
import sys
import logging
from urllib import urlencode
import re
import json
import time
import hospitals
from mylogger import MyLogger
MyLogger(show_console=True)
class SuDaHospital(object):
'''苏州大学第一附属医院，十梓街院'''
def __init__(self, user=None, doctor=None):
self.user = user
self.doctor = doctor
self.session = requests.Session()
headers = {}
headers['Accept'] = '*/*'
headers['Accept-Encoding'] = 'gzip, deflate'
headers['Cache-Control'] = 'no-cache'
headers['DNT'] = '1'
headers['Host'] = 'new.sdfyy.cn'
headers['Origin'] = 'http://new.sdfyy.cn'
headers['Pragma'] = 'no-cache'
headers['Upgrade-Insecure-Requests'] = '1'
headers[
# My modules
from preproc import load_data, flatten
from mylogger import MyLogger
# Paths
prefix = 'ANN'
timestamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
results_path = Path('{}_results'.format(prefix))
log_path = results_path / '{}_out_{}.txt'.format(prefix, timestamp)
model_path = results_path / '{}_model_{}.ckpt'.format(prefix, timestamp)
img_path = results_path / '{}_cross_entropy_{}.png'.format(prefix, timestamp)
# Init logger
print(log_path, end='\r\n')
logger = MyLogger(log_path)
## Disable TF log
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
## Load data
logger.debug('Loading data... ')
t0 = datetime.now()
Xtrain, Ytrain, Xtest, Ytest = load_data()
Xtrain = flatten(Xtrain)
Xtest = flatten(Xtest)
Xtrain = (Xtrain - Xtrain.mean()) / Xtrain.std()
Xtest = (Xtest - Xtest.mean()) / Xtest.std()
dt = datetime.now() - t0
logger.debug('Done. [Elapsed {}]\r\n'.format(dt))
import sys
import os
from time import gmtime, strftime
from datetime import datetime
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from urllib.error import HTTPError
from urllib.request import urlopen
from bs4 import BeautifulSoup
logger = MyLogger()
#############
# Functions #
#############
def parseURL(URL):
logger.debug("parseURL() called for URL: " + URL)
try:
html = urlopen(URL)
except HTTPError as e:
logger.error("HTTPError - Opening URL failed. Error code:'" + repr(e.code) + "', URL:'" + URL + "'")
"""
Created on June 18, 2019
DataBase connection pool.
@author: Zewen Huang
"""
from sys import path
path.append('../config')
import logging
from mylogger import MyLogger
from mysql.connector import Error, pooling
from DBPoolConfigParser import DBPoolConfigParser
logger = MyLogger('dbpoolconfigparser', logging.INFO).get_logger()
class MySQLPool(object):
def __init__(self, pool_token):
self._connection_pool = self._init_pool(pool_token)
def _init_pool(self, pool_token):
try:
connection_pool = pooling.MySQLConnectionPool(**pool_token)
return connection_pool
except Exception as e:
logger.error("Fail to initialize MySQL database pool.")
exit(1)
class Weibo(object):
def __init__(self, account_file):
self.check_paths(['log', 'pic', 'cookies', 'driver', 'account'])
self.username, self.password = self.get_account(account_file)
self.login_api = 'https://passport.weibo.cn/sso/login'
self.user_info_api = 'https://m.weibo.cn/home/me?format=cards'
self.user_weibo_api = "https://m.weibo.cn/api/container/getIndex?uid={}&luicode=20000174&type=uid&value={}&containerid=107603{}"
self.headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:56.0) Gecko/20100101 Firefox/56.0',
}
self.logger = MyLogger('weibo').get_logger()
self.cookie_path = 'cookies'
self.conn = redis.Redis(connection_pool=redis.ConnectionPool(host='redis', port=6379, decode_responses=True))
self.cookies = self.get_cookies()
def get_account(self, filename):
with open(filename, 'r') as f:
d = json.load(f)
return d.get('username'), d.get('password')
def check_paths(self, paths):
for p in paths:
if not os.path.exists(p):
os.mkdir(p)
def make_headers(self, items=None):
headers = self.headers.copy()
if items:
headers.update(items)
return headers
def login(self):
login_data = {
'username': self.username,
'password': self.password,
'savestate': 1,
'r': 'http%3A%2F%2Fm.weibo.cn%2F',
'ec': 0,
'pagerefer': '',
'entry': 'mweibo',
'wentry': '',
'loginfrom': '',
'client_id': '',
'code': '',
'qq': '',
'mainpageflag': 1,
'hff': '',
'hfp': ''
}
items = {
'Referer': 'https://passport.weibo.cn/signin/login?entry='
'mweibo&res=wel&wm=3349&r=https%3A%2F%2Fm.weibo.cn%2F'
}
try:
r = requests.post(self.login_api, headers=self.make_headers(items), data=login_data)
if r.json()['retcode'] == 20000000:
self.logger.info('登录成功')
r.cookies.pop('login')
self.save_cookie(r.cookies)
return r.cookies
else:
self.logger.error('登录失败, 可能需要验证码 {}'.format(r.text))
self.logger.info('正在尝试用selenium登录')
cookies = selenium_login(self.username, self.password)
if not cookies:
self.logger.error('selenium登录失败')
exit(1)
self.logger.info('selenium登录成功')
self.save_cookie(cookies)
return cookies
except Exception as e:
self.logger.error('登录失败 {}'.format(e), exc_info=True)
def save_cookie(self, cookies):
file = os.path.join(self.cookie_path, self.username + '.pkl')
try:
with open(file, 'wb') as f:
self.logger.info('cookie保存成功')
pickle.dump(cookies, f)
except Exception as e:
self.logger.error('保存cookie失败 {}'.format(e), exc_info=True)
def get_cookies(self):
file = os.path.join(self.cookie_path, self.username + '.pkl')
try:
with open(file, 'rb') as f:
cookies = pickle.load(f)
if self.verify_cookie(cookies):
self.logger.info('cookie验证成功')
return cookies
else:
self.logger.warning('cookie过期')
self.logger.warning('正在重新登录')
cookies = self.login()
if not cookies:
self.logger.error('登陆失败', exc_info=True)
except Exception as e:
self.logger.error('cookie不存在，重新登录 {}'.format(e), exc_info=True)
cookies = self.login()
if not cookies:
exit(1)
return cookies
def verify_cookie(self, cookies):
try:
r = requests.get(self.user_info_api, headers=self.make_headers(), cookies=cookies)
data = r.json()
user_info = data[0]["card_group"][0]["user"]
if user_info:
return True
except Exception as e:
self.logger.error('验证cookie失败 {}'.format(e), exc_info=True)
return False
def get_user_basic_info(self):
try:
r = requests.get(self.user_info_api, headers=self.make_headers(), cookies=self.cookies)
data = r.json()
user_info = data[0]["card_group"][0]["user"]
user = dict()
user["user_id"] = user_info["id"]
user["user_name"] = user_info["name"]
user["weibo_count"] = user_info["mblogNum"]
user["follow_count"] = user_info["attNum"]
return user
except Exception as e:
self.logger.error('获取个人信息失败 {}'.format(e), exc_info=True)
return
def get_user_weibo(self, uid): # 获取前十条微博
self.logger.info('正在获取 {}的微博'.format(uid))
url = self.user_weibo_api.format(uid, uid, uid)
try:
r = requests.get(url, headers=self.make_headers(), cookies=self.cookies)
cards = r.json().get("data").get("cards")
for card in cards:
# weibo = {}
try:
# weibo["weibo_content_id"] = card.get("mblog").get("id")
# weibo["weibo_content"] = card.get("mblog").get("text")
# weibo["weibo_user_id"] = card.get("mblog").get("user").get("id")
# weibo["weibo_username"] = card.get("mblog").get("user").get("screen_name")
# weibo["mid"] = card.get("mblog").get("mid")
wid = card.get("mblog").get("id")
mid = card.get("mblog").get("mid")
self.logger.info('{} - {}'.format(wid, mid))
yield wid, mid
except AttributeError:
continue # cards列表里面不一定是微博，用try来过滤
except Exception as e:
self.logger.error('获取用户微博失败 {}'.format(e))
def get_st(self): # st是转发微博post必须的参数
url = "https://m.weibo.cn/api/config"
# self.s.get(url, headers=self.make_cookie_header(), cookies=self.cookies, proxies=self.proxies, verify=False)
try:
s = requests.session()
r = s.get(url, headers=self.make_headers(), cookies=self.cookies)
data = r.json()
st = data["data"]["st"]
return st, s
except Exception as e:
self.logger.error('http error {}'.format(e), exc_info=True)
def forward_weibo(self, weibo, content):
st, s = self.get_st()
url = "https://m.weibo.cn/api/statuses/repost"
data = {"id": weibo["wid"], "content": content, "mid": weibo["mid"], "st": st}
# 这里一定要加referer， 不加会变成不合法的请求
items = {
'Referer': 'https://m.weibo.cn/compose/repost?id={}'.format(weibo['wid'])
}
try:
r = s.post(url, data=data, headers=self.make_headers(items), cookies=self.cookies)
if r.json().get("ok") == 1:
self.logger.info('转发微博{}成功'.format(weibo['wid']))
return True
else:
self.logger.warning('转发微博{}失败 {}'.format(weibo['wid'], r.text))
return None
except Exception as e:
self.logger.error('http error {}'.format(e), exc_info=True)
return None
# 发送原创微博
def send_original_weibo(self, content, pic_path=None):
st, s = self.get_st()
data = {
# "luicode": "10000011",
# "lfid": "2304135827525376_ - _WEIBO_SECOND_PROFILE_MORE_WEIBO",
# "featurecode": "20000320",
"content": content,
"st": st
}
if pic_path:
pic_id = self.upload_pic(pic_path)
if pic_id:
data["picId"] = pic_id
url = "https://m.weibo.cn/api/statuses/update"
items = {
'Referer': 'https://m.weibo.cn/compose/'
}
try:
r = s.post(url, data=data, headers=self.make_headers(items), cookies=self.cookies)
if r.json()["ok"] == 1:
self.logger.info('原创微博发送成功')
else:
self.logger.warning('原创微博发送失败')
except Exception as e:
self.logger.error('http error {}'.format(e))
return None
def upload_pic(self, pic_path):
url = "https://m.weibo.cn/api/statuses/uploadPic"
st, s = self.get_st()
pic_name = os.path.split(pic_path)[-1]
# 这里如果pic_name 中有 '\' 会上传失败， 在windows里 \是路径，要去掉
try:
files = {"pic": (pic_name, open(pic_path, "rb").read(), "image/jpeg")}
except Exception as e:
self.logger.error('打开图片失败 {}'.format(e), exc_info=True)
return None
items = {
'Referer': 'https://m.weibo.cn/compose/'
}
data = {"type": "json", "st": st}
# print(r.json())
try:
r = s.post(url, data=data, files=files, headers=self.make_headers(items), cookies=self.cookies)
pic_id = r.json()["pic_id"]
self.logger.info('图片上传成功')
return pic_id
except Exception as e:
self.logger.error('图片上传失败 {}'.format(e), exc_info=True)
return None
import datetime
import codecs
import struct
import urllib
import urllib.request
import urllib.parse
import socketserver
import redis
sys.path.append("..")
from mylogger import MyLogger
from common.KtvInfo import KtvInfo
from jobmanager import *
logger = MyLogger('ServiceSocket')
"""
class o2oupload(Thread):
def __init__(self, name):
threading.Thread.__init__(self)
self.queque={}
self.stopped = False
def add(self, data):
try:
if isinstance(data,list):
for item in data:
self.addsingle(item)
else:
self.addsingle(data)
#!/usr/bin/env python # vim: set fileencoding=UTF8 from fan import Fan from temp import Temperature from mylogger import MyLogger from datetime import datetime import sys, time, mylogger LOG_FILENAME = "/var/log/fan-control" LOG_LEVEL = mylogger.LOG_LEVEL_DEBUG logger = MyLogger(LOG_FILENAME, LOG_LEVEL) fan = Fan() temp = Temperature() delay = 5.0 thresholdUp = 31 thresholdDown = 29 count = 0 maxCountUp = int(15 / delay) # 15s / 5 = 3 maxCountDown = int(30 / delay) # 30s / 5 = 6 logCount = 0 maxLogCount = int(30 / delay) t = temp.read() currentStatus = fan.read()
#!/usr/bin/env python # vim: set fileencoding=UTF8 from fan import Fan from temp import Temperature from mylogger import MyLogger from datetime import datetime import sys, time, mylogger LOG_FILENAME = "/var/log/fan-control" LOG_LEVEL = mylogger.LOG_LEVEL_DEBUG logger = MyLogger(LOG_FILENAME, LOG_LEVEL) fan = Fan() temp = Temperature() delay = 5.0 thresholdUp = 31 thresholdDown = 29 count = 0 maxCountUp = int(15 / delay) # 15s / 5 = 3 maxCountDown = int(30 / delay) # 30s / 5 = 6 logCount = 0 maxLogCount = int(30 / delay) t = temp.read() currentStatus = fan.read()
#!/usr/bin/env python # vim: set fileencoding=UTF8 from temp import Temperature from templog import TempLog from mylogger import MyLogger import mylogger LOG_FILENAME = '/var/log/temp-monitor' LOG_LEVEL = mylogger.LOG_LEVEL_DEBUG logger = MyLogger(LOG_FILENAME, LOG_LEVEL) db = TempLog() temp = Temperature() t = temp.read() db.save(t) print "SAVED: %2.3f °C" % (t)
# -*- coding: utf-8 -*-
import os, sys
import re
import urlparse
import requests
CWD = os.path.dirname(os.path.abspath(__file__))
sys.path.append('%s/util' % CWD)
from myrequests import MyRequests
from mylogger import MyLogger
from lxml import etree
import socket
log = MyLogger('logging')
TOP_DOMAINS_SET = set([
"cc", "com", "cn", "org", "net", "edu.cn", "gov.cn", "com.cn", "ac.cn",
"net.cn", "org.cn", "mil.cn"
])
PROV_DOMAINS_SET = set(["ac.cn","bj.cn","sh.cn","tj.cn",\
"cq.cn","he.cn","sx.cn","nm.cn",\
"ln.cn","jl.cn","hl.cn","js.cn",\
"zj.cn","ah.cn","fj.cn","jx.cn",\
"sd.cn","ha.cn","hb.cn","hn.cn",\
"gd.cn","gx.cn","sc.cn","gz.cn",\
"yn.cn","gs.cn","qh.cn","nx.cn",\
"xj.cn","tw.cn","hk.cn","mo.cn",\
"xz.cn","hi.cn","sn.cn"])
#eg. anyang.gov.cn -> www.anyang.gov.cn
def is_root_site(site):
