def __init__(self, account_file): self.check_paths(['log', 'pic', 'cookies', 'driver', 'account']) self.username, self.password = self.get_account(account_file) self.login_api = 'https://passport.weibo.cn/sso/login' self.user_info_api = 'https://m.weibo.cn/home/me?format=cards' self.user_weibo_api = "https://m.weibo.cn/api/container/getIndex?uid={}&luicode=20000174&type=uid&value={}&containerid=107603{}" self.headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:56.0) Gecko/20100101 Firefox/56.0', } self.logger = MyLogger('weibo').get_logger() self.cookie_path = 'cookies' self.conn = redis.Redis(connection_pool=redis.ConnectionPool(host='redis', port=6379, decode_responses=True)) self.cookies = self.get_cookies()
def __init__(self, host, user, password, db): self.logger = MyLogger(name='api_db', prefix='api_db') self.host = host self.user = user self.passwd = password self.db = db try: self.connect = mysql.connect(host, user, password, db, charset='utf8') except Exception as e: self.logger.error(e) raise e
def setUp(self): unittest.TestCase.setUp(self) agentFactory = PoleAgentFactory() environmentFactory = PoleEnvironmentFactory() trainerFactory = PoleTrainerFactory() buildParameterFactory = PoleBuildParameterFactory() store = Store(self.dbPath) logger = MyLogger(console_print=True) self.builder = Builder(trainerFactory, agentFactory, environmentFactory, store, logger) self.buildParameters = [] for k1 in range(2): nIntervalSave = 3 nEpoch = 5 self.buildParameters.append( PoleBuildParameter(int(nIntervalSave), int(nEpoch), label="test" + str(k1))) for agentClass in ("agent002", "agent003", "agent004"): self.buildParameters.append( PoleBuildParameter(int(nIntervalSave), int(nEpoch), agentClass=agentClass, label="test " + agentClass)) self.loader = Loader(agentFactory, buildParameterFactory, environmentFactory, store)
def setUp(self): unittest.TestCase.setUp(self) agentFactory = WaeAgentFactory() environmentFactory = WaeEnvironmentFactory() trainerFactory = WaeTrainerFactory() buildParameterFactory = WaeBuildParameterFactory() store = Store(self.dbPath, self.trainLogFolderPath) logger = MyLogger(console_print=True) self.builder = Builder(trainerFactory, agentFactory, environmentFactory, store, logger) self.buildParameters = [] for k1 in range(2): nIntervalSave = 10 nEpoch = 20 nLayer = int(np.random.choice((1, 2))) self.buildParameters.append( WaeBuildParameter(int(nIntervalSave), int(nEpoch), label="test" + str(k1), nLayer=nLayer, eps_given_sinkhorn=0.1, tol_sinkhorn=0.1)) self.loader = Loader(agentFactory, buildParameterFactory, environmentFactory, store)
class BaseDBAdapter(object): def __init__(self, host, user, password, db): self.logger = MyLogger(name='api_db', prefix='api_db') self.host = host self.user = user self.passwd = password self.db = db try: self.connect = mysql.connect(host, user, password, db, charset='utf8') except Exception as e: self.logger.error(e) raise e def close_connect(self): if self.connect: self.connect.close()
def store_specified_isbn(target: Path, isbn: str, honto: HontoSearchCliant, ehon: EhonSearchCliant, config: dict, logger: MyLogger, db: DatabaseCliant) -> None: """isbnを元に元情報を上書きし、移動する Args: target (Path): 対象書籍のpath isbn (str): isbn honto (HontoSearchCliant): Hontoの検索クライアント ehon (EhonSearchCliant): E-honの検索クライアント config (dict): configのdict logger (MyLogger): logger db (DatabaseCliant): データベースのクライアント Raises: HontoDoesNotHaveDataError: Hontoがその書籍のページを持っていないときのエラー """ try: info = fetch_book_info_from_isbn(isbn, honto, ehon) except HontoDoesNotHaveDataError as e: send_err_dir(target, Path(config["output_dir"])) logger.write("ERROR", str(e)) raise HontoDoesNotHaveDataError(e) else: dst = construct_dst(config["output_dir"], info) dst.parent.mkdir(parents=True, exist_ok=True) shutil.move(str(target), dst) logger.write("SUCCESS", str(dst)) # データベースに情報を追加 fetch_result = { "title": info["title"], "isbn": info["isbn"], "authors": "\t".join(info["authors"] or [""]), "publishers": info["publisher"], "categories": info["category"], "destination": str(dst) } db.store(fetch_result)
def completion_no_isbn(source_csv_path: str) -> None: """手作業でisbnや移動先を書いたcsvを元に補完作業をする Args: source_csv_path (str): ソースとなるcsv """ logger = MyLogger() honto = HontoSearchCliant() ehon = EhonSearchCliant() with open(Path(__file__).resolve().parents[1] / "config.yml") as f: config = yaml.safe_load(f) Path(config["output_dir"]).resolve().mkdir(exist_ok=True, parents=True) db_cliant = DatabaseCliant(Path(config["database_path"])) csv_path = Path(source_csv_path).resolve() with open(csv_path) as f: reader = csv.reader(f) for row in reader: book_path = Path(row[0]).resolve() neemock = str(row[1]) if re.fullmatch(r"[0-9-]+", neemock): isbn = neemock try: store_specified_isbn(book_path, isbn, honto, ehon, config, logger, db_cliant) except HontoDoesNotHaveDataError as e: send_err_dir(book_path, Path(config["output_dir"])) logger.write("ERROR", str(e)) else: dst = Path(neemock) db_cliant.update_dst(book_path, dst) shutil.move(str(book_path), str(dst / book_path.name)) db_cliant.close()
def main(): logger = MyLogger() profect_dir = Path(__file__).resolve().parents[1] config_path = profect_dir / "config.yml" if not config_path.exists(): generate_config_file(config_path) with open(config_path, "r") as f: config = yaml.safe_load(f) Path(config["output_dir"]).mkdir(exist_ok=True, parents=True) db_cliant = DatabaseCliant(Path(config["database_path"])) honto_cliant = HontoSearchCliant() ehon_cliant = EhonSearchCliant() # input_dir内のPDFに対して処理をする for pdf_file in Path(config["input_dir"]).glob("**/*.pdf"): try: book_info = fetch_book_info_from_pdf(pdf_file, honto_cliant, ehon_cliant) except NotFoundIsbnError as e: send_err_dir(pdf_file, Path(config["output_dir"])) logger.write("ERROR", str(e)) continue except HontoDoesNotHaveDataError as e: send_err_dir(pdf_file, Path(config["output_dir"])) logger.write("ERROR", str(e)) continue dst = construct_dst(config["output_dir"], book_info) dst.parent.mkdir(parents=True, exist_ok=True) shutil.move(str(pdf_file), dst) logger.write("SUCCESS", dst) # データベースに情報を追加 fetch_result = { "title": book_info["title"], "isbn": book_info["isbn"], "authors": "\t".join(book_info["authors"] or [""]), "publishers": book_info["publisher"], "categories": book_info["category"], "destination": str(dst) } db_cliant.store(fetch_result) db_cliant.close()
def setUp(self): unittest.TestCase.setUp(self) agentFactory = ConcAgentFactory() environmentFactory = ConcEnvironmentFactory() trainerFactory = WaeTrainerFactory() buildParameterFactory = ConcBuildParameterFactory() store = Store(self.dbPath) logger = MyLogger(console_print=True) self.builder = Builder(trainerFactory, agentFactory, environmentFactory, store, logger) nIntervalSave = 1 nEpoch = 2 self.buildParameters = [] for k1 in range(2): self.buildParameters.append(ConcBuildParameter(int(nIntervalSave), int(nEpoch), label="test" + str(k1))) for k1 in range(2): self.buildParameters.append(ConcBuildParameter(int(nIntervalSave), int(nEpoch) , label="test case study 02a " + str(k1) , nXi = 3 , target_casestudy = "cs02a" )) for k1 in range(2): self.buildParameters.append(ConcBuildParameter(int(nIntervalSave), int(nEpoch) , label="test case study 03a " + str(k1) , nXi = 3 , target_casestudy = "cs03a" )) for k1 in range(2): self.buildParameters.append(ConcBuildParameter(int(nIntervalSave), int(nEpoch) , label="test case study 03b " + str(k1) , nXi = 3 , target_casestudy = "cs03b" )) for k1 in range(2): self.buildParameters.append(ConcBuildParameter(int(nIntervalSave), int(nEpoch) , label="test case study 03c " + str(k1) , nXi = 3 , target_casestudy = "cs03c" )) self.loader = Loader(agentFactory, buildParameterFactory, environmentFactory, store)
def setUp(self): unittest.TestCase.setUp(self) agentFactory = AgentFactory() environmentFactory = EnvironmentFactory() trainerFactory = TrainerFactory() buildParameterFactory = BuildParameterFactory() store = Store(self.dbPath) logger = MyLogger(console_print=False) self.builder = Builder(trainerFactory, agentFactory, environmentFactory, store, logger) self.buildParameters = [] for k1 in range(3): nIntervalSave = 10 nEpoch = 100 self.buildParameters.append( BuildParameter(int(nIntervalSave), int(nEpoch), label="test" + str(k1))) self.loader = Loader(agentFactory, buildParameterFactory, environmentFactory, store)
import time import datetime import argparse import torch from torch.utils.data import DataLoader from torchvision import datasets from torchvision import transforms from torch.autograd import Variable import torch.optim as optim import warnings from tqdm import tqdm from mylogger import MyLogger local_logger = MyLogger(filename='./logs/train.log',logger_name='train') warnings.filterwarnings('ignore') def img_label_map(img_list): root_path = str(os.getcwd()) local_path = 'data/labels' labels_dir = os.path.join(root_path,local_path) label_file_list = [] for img_path in img_list: img_path = img_path.strip() temp_path = img_path.split('/')[-1].replace('.jpg','.txt') label_file_list.append(os.path.join(labels_dir,temp_path)) return label_file_list
# -*- coding:utf-8 -*- import requests import sys import logging from urllib import urlencode import re import json import time import hospitals from mylogger import MyLogger MyLogger(show_console=True) class SuDaHospital(object): '''苏州大学第一附属医院,十梓街院''' def __init__(self, user=None, doctor=None): self.user = user self.doctor = doctor self.session = requests.Session() headers = {} headers['Accept'] = '*/*' headers['Accept-Encoding'] = 'gzip, deflate' headers['Cache-Control'] = 'no-cache' headers['DNT'] = '1' headers['Host'] = 'new.sdfyy.cn' headers['Origin'] = 'http://new.sdfyy.cn' headers['Pragma'] = 'no-cache' headers['Upgrade-Insecure-Requests'] = '1' headers[
# My modules from preproc import load_data, flatten from mylogger import MyLogger # Paths prefix = 'ANN' timestamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S') results_path = Path('{}_results'.format(prefix)) log_path = results_path / '{}_out_{}.txt'.format(prefix, timestamp) model_path = results_path / '{}_model_{}.ckpt'.format(prefix, timestamp) img_path = results_path / '{}_cross_entropy_{}.png'.format(prefix, timestamp) # Init logger print(log_path, end='\r\n') logger = MyLogger(log_path) ## Disable TF log os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' ## Load data logger.debug('Loading data... ') t0 = datetime.now() Xtrain, Ytrain, Xtest, Ytest = load_data() Xtrain = flatten(Xtrain) Xtest = flatten(Xtest) Xtrain = (Xtrain - Xtrain.mean()) / Xtrain.std() Xtest = (Xtest - Xtest.mean()) / Xtest.std() dt = datetime.now() - t0 logger.debug('Done. [Elapsed {}]\r\n'.format(dt))
import sys import os from time import gmtime, strftime from datetime import datetime from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from urllib.error import HTTPError from urllib.request import urlopen from bs4 import BeautifulSoup logger = MyLogger() ############# # Functions # ############# def parseURL(URL): logger.debug("parseURL() called for URL: " + URL) try: html = urlopen(URL) except HTTPError as e: logger.error("HTTPError - Opening URL failed. Error code:'" + repr(e.code) + "', URL:'" + URL + "'")
""" Created on June 18, 2019 DataBase connection pool. @author: Zewen Huang """ from sys import path path.append('../config') import logging from mylogger import MyLogger from mysql.connector import Error, pooling from DBPoolConfigParser import DBPoolConfigParser logger = MyLogger('dbpoolconfigparser', logging.INFO).get_logger() class MySQLPool(object): def __init__(self, pool_token): self._connection_pool = self._init_pool(pool_token) def _init_pool(self, pool_token): try: connection_pool = pooling.MySQLConnectionPool(**pool_token) return connection_pool except Exception as e: logger.error("Fail to initialize MySQL database pool.") exit(1)
class Weibo(object): def __init__(self, account_file): self.check_paths(['log', 'pic', 'cookies', 'driver', 'account']) self.username, self.password = self.get_account(account_file) self.login_api = 'https://passport.weibo.cn/sso/login' self.user_info_api = 'https://m.weibo.cn/home/me?format=cards' self.user_weibo_api = "https://m.weibo.cn/api/container/getIndex?uid={}&luicode=20000174&type=uid&value={}&containerid=107603{}" self.headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:56.0) Gecko/20100101 Firefox/56.0', } self.logger = MyLogger('weibo').get_logger() self.cookie_path = 'cookies' self.conn = redis.Redis(connection_pool=redis.ConnectionPool(host='redis', port=6379, decode_responses=True)) self.cookies = self.get_cookies() def get_account(self, filename): with open(filename, 'r') as f: d = json.load(f) return d.get('username'), d.get('password') def check_paths(self, paths): for p in paths: if not os.path.exists(p): os.mkdir(p) def make_headers(self, items=None): headers = self.headers.copy() if items: headers.update(items) return headers def login(self): login_data = { 'username': self.username, 'password': self.password, 'savestate': 1, 'r': 'http%3A%2F%2Fm.weibo.cn%2F', 'ec': 0, 'pagerefer': '', 'entry': 'mweibo', 'wentry': '', 'loginfrom': '', 'client_id': '', 'code': '', 'qq': '', 'mainpageflag': 1, 'hff': '', 'hfp': '' } items = { 'Referer': 'https://passport.weibo.cn/signin/login?entry=' 'mweibo&res=wel&wm=3349&r=https%3A%2F%2Fm.weibo.cn%2F' } try: r = requests.post(self.login_api, headers=self.make_headers(items), data=login_data) if r.json()['retcode'] == 20000000: self.logger.info('登录成功') r.cookies.pop('login') self.save_cookie(r.cookies) return r.cookies else: self.logger.error('登录失败, 可能需要验证码 {}'.format(r.text)) self.logger.info('正在尝试用selenium登录') cookies = selenium_login(self.username, self.password) if not cookies: self.logger.error('selenium登录失败') exit(1) self.logger.info('selenium登录成功') self.save_cookie(cookies) return cookies except Exception as e: self.logger.error('登录失败 {}'.format(e), exc_info=True) def save_cookie(self, cookies): file = os.path.join(self.cookie_path, self.username + '.pkl') try: with open(file, 'wb') as f: self.logger.info('cookie保存成功') pickle.dump(cookies, f) except Exception as e: self.logger.error('保存cookie失败 {}'.format(e), exc_info=True) def get_cookies(self): file = os.path.join(self.cookie_path, self.username + '.pkl') try: with open(file, 'rb') as f: cookies = pickle.load(f) if self.verify_cookie(cookies): self.logger.info('cookie验证成功') return cookies else: self.logger.warning('cookie过期') self.logger.warning('正在重新登录') cookies = self.login() if not cookies: self.logger.error('登陆失败', exc_info=True) except Exception as e: self.logger.error('cookie不存在,重新登录 {}'.format(e), exc_info=True) cookies = self.login() if not cookies: exit(1) return cookies def verify_cookie(self, cookies): try: r = requests.get(self.user_info_api, headers=self.make_headers(), cookies=cookies) data = r.json() user_info = data[0]["card_group"][0]["user"] if user_info: return True except Exception as e: self.logger.error('验证cookie失败 {}'.format(e), exc_info=True) return False def get_user_basic_info(self): try: r = requests.get(self.user_info_api, headers=self.make_headers(), cookies=self.cookies) data = r.json() user_info = data[0]["card_group"][0]["user"] user = dict() user["user_id"] = user_info["id"] user["user_name"] = user_info["name"] user["weibo_count"] = user_info["mblogNum"] user["follow_count"] = user_info["attNum"] return user except Exception as e: self.logger.error('获取个人信息失败 {}'.format(e), exc_info=True) return def get_user_weibo(self, uid): # 获取前十条微博 self.logger.info('正在获取 {}的微博'.format(uid)) url = self.user_weibo_api.format(uid, uid, uid) try: r = requests.get(url, headers=self.make_headers(), cookies=self.cookies) cards = r.json().get("data").get("cards") for card in cards: # weibo = {} try: # weibo["weibo_content_id"] = card.get("mblog").get("id") # weibo["weibo_content"] = card.get("mblog").get("text") # weibo["weibo_user_id"] = card.get("mblog").get("user").get("id") # weibo["weibo_username"] = card.get("mblog").get("user").get("screen_name") # weibo["mid"] = card.get("mblog").get("mid") wid = card.get("mblog").get("id") mid = card.get("mblog").get("mid") self.logger.info('{} - {}'.format(wid, mid)) yield wid, mid except AttributeError: continue # cards列表里面不一定是微博,用try来过滤 except Exception as e: self.logger.error('获取用户微博失败 {}'.format(e)) def get_st(self): # st是转发微博post必须的参数 url = "https://m.weibo.cn/api/config" # self.s.get(url, headers=self.make_cookie_header(), cookies=self.cookies, proxies=self.proxies, verify=False) try: s = requests.session() r = s.get(url, headers=self.make_headers(), cookies=self.cookies) data = r.json() st = data["data"]["st"] return st, s except Exception as e: self.logger.error('http error {}'.format(e), exc_info=True) def forward_weibo(self, weibo, content): st, s = self.get_st() url = "https://m.weibo.cn/api/statuses/repost" data = {"id": weibo["wid"], "content": content, "mid": weibo["mid"], "st": st} # 这里一定要加referer, 不加会变成不合法的请求 items = { 'Referer': 'https://m.weibo.cn/compose/repost?id={}'.format(weibo['wid']) } try: r = s.post(url, data=data, headers=self.make_headers(items), cookies=self.cookies) if r.json().get("ok") == 1: self.logger.info('转发微博{}成功'.format(weibo['wid'])) return True else: self.logger.warning('转发微博{}失败 {}'.format(weibo['wid'], r.text)) return None except Exception as e: self.logger.error('http error {}'.format(e), exc_info=True) return None # 发送原创微博 def send_original_weibo(self, content, pic_path=None): st, s = self.get_st() data = { # "luicode": "10000011", # "lfid": "2304135827525376_ - _WEIBO_SECOND_PROFILE_MORE_WEIBO", # "featurecode": "20000320", "content": content, "st": st } if pic_path: pic_id = self.upload_pic(pic_path) if pic_id: data["picId"] = pic_id url = "https://m.weibo.cn/api/statuses/update" items = { 'Referer': 'https://m.weibo.cn/compose/' } try: r = s.post(url, data=data, headers=self.make_headers(items), cookies=self.cookies) if r.json()["ok"] == 1: self.logger.info('原创微博发送成功') else: self.logger.warning('原创微博发送失败') except Exception as e: self.logger.error('http error {}'.format(e)) return None def upload_pic(self, pic_path): url = "https://m.weibo.cn/api/statuses/uploadPic" st, s = self.get_st() pic_name = os.path.split(pic_path)[-1] # 这里如果pic_name 中有 '\' 会上传失败, 在windows里 \是路径,要去掉 try: files = {"pic": (pic_name, open(pic_path, "rb").read(), "image/jpeg")} except Exception as e: self.logger.error('打开图片失败 {}'.format(e), exc_info=True) return None items = { 'Referer': 'https://m.weibo.cn/compose/' } data = {"type": "json", "st": st} # print(r.json()) try: r = s.post(url, data=data, files=files, headers=self.make_headers(items), cookies=self.cookies) pic_id = r.json()["pic_id"] self.logger.info('图片上传成功') return pic_id except Exception as e: self.logger.error('图片上传失败 {}'.format(e), exc_info=True) return None
import datetime import codecs import struct import urllib import urllib.request import urllib.parse import socketserver import redis sys.path.append("..") from mylogger import MyLogger from common.KtvInfo import KtvInfo from jobmanager import * logger = MyLogger('ServiceSocket') """ class o2oupload(Thread): def __init__(self, name): threading.Thread.__init__(self) self.queque={} self.stopped = False def add(self, data): try: if isinstance(data,list): for item in data: self.addsingle(item) else: self.addsingle(data)
#!/usr/bin/env python # vim: set fileencoding=UTF8 from fan import Fan from temp import Temperature from mylogger import MyLogger from datetime import datetime import sys, time, mylogger LOG_FILENAME = "/var/log/fan-control" LOG_LEVEL = mylogger.LOG_LEVEL_DEBUG logger = MyLogger(LOG_FILENAME, LOG_LEVEL) fan = Fan() temp = Temperature() delay = 5.0 thresholdUp = 31 thresholdDown = 29 count = 0 maxCountUp = int(15 / delay) # 15s / 5 = 3 maxCountDown = int(30 / delay) # 30s / 5 = 6 logCount = 0 maxLogCount = int(30 / delay) t = temp.read() currentStatus = fan.read()
#!/usr/bin/env python # vim: set fileencoding=UTF8 from temp import Temperature from templog import TempLog from mylogger import MyLogger import mylogger LOG_FILENAME = '/var/log/temp-monitor' LOG_LEVEL = mylogger.LOG_LEVEL_DEBUG logger = MyLogger(LOG_FILENAME, LOG_LEVEL) db = TempLog() temp = Temperature() t = temp.read() db.save(t) print "SAVED: %2.3f °C" % (t)
# -*- coding: utf-8 -*- import os, sys import re import urlparse import requests CWD = os.path.dirname(os.path.abspath(__file__)) sys.path.append('%s/util' % CWD) from myrequests import MyRequests from mylogger import MyLogger from lxml import etree import socket log = MyLogger('logging') TOP_DOMAINS_SET = set([ "cc", "com", "cn", "org", "net", "edu.cn", "gov.cn", "com.cn", "ac.cn", "net.cn", "org.cn", "mil.cn" ]) PROV_DOMAINS_SET = set(["ac.cn","bj.cn","sh.cn","tj.cn",\ "cq.cn","he.cn","sx.cn","nm.cn",\ "ln.cn","jl.cn","hl.cn","js.cn",\ "zj.cn","ah.cn","fj.cn","jx.cn",\ "sd.cn","ha.cn","hb.cn","hn.cn",\ "gd.cn","gx.cn","sc.cn","gz.cn",\ "yn.cn","gs.cn","qh.cn","nx.cn",\ "xj.cn","tw.cn","hk.cn","mo.cn",\ "xz.cn","hi.cn","sn.cn"]) #eg. anyang.gov.cn -> www.anyang.gov.cn def is_root_site(site):