def _do_insert(self, conn, item, spider):
try:
conn.execute("select * from base_location where lac=%s and cid=%s",
(item['lac'], item['cid']))
ret = conn.fetchone()
if ret:
logging.info(LogUtil.get_time_now() +
"do db update, item ==>" + str(item))
conn.execute(
"update base_location set errcode=%s, lat=%s, lon=%s, address=%s, type=%s, update_time=now() where lac=%s and cid=%s",
(item['errcode'], item['lat'], item['lon'],
item['address'],
self._get_base_type(
item['lac'], item['cid']), item['lac'], item['cid']))
else:
logging.info(LogUtil.get_time_now() +
"do db insert, item ==>" + str(item))
conn.execute(
"insert into base_location (errcode, lac, cid, lat, lon, address, type, update_time) values (%s, %s, %s, %s, %s, %s, %s, now())",
(item['errcode'], item['lac'], item['cid'], item['lat'],
item['lon'], item['address'],
self._get_base_type(item['lac'], item['cid'])))
except:
logging.info(LogUtil.get_time_now() + traceback.format_exc())
def compsum(fname, hash, mode):
newhash = getsum(fname,mode)
if hash != newhash:
LogUtil.printlog("Hash mismatch: Expecting: %s got %s for %s" % (hash, newhash, fname))
Globals.failed += 1
else:
if Globals.verbose: LogUtil.printlog("Hash match: Expecting: %s got %s for %s" % (hash, newhash, fname))
Globals.passed += 1
def excute(self):
"excute Method"
service = RemoteConfigService.RemoteConfigService()
while True:
current_time = time.strftime('%Y-%m-%d %X', time.localtime())
LogUtil.print_i('Excute RemoteConfig update at ' + current_time)
service.update_remote_config()
time.sleep(60)
return
def checkfile(path, method):
checksum = getsum(path, method)
if checksum in basename(path) or checksum in basename(path).upper():
Globals.passed += 1
if Globals.verbose:
LogUtil.printlog("Found: %s in: %s" % (checksum, path))
else:
Globals.failed += 1
LogUtil.printlog("Not found: %s in: %s" % (checksum,path))
def update_config_file(self):
"update config file"
config_file_path = Constant.GIT_LOCAL_CONFIG_FILE_PATH
local_config_dic = JsonUtil.json_file_2_dic(config_file_path)
result_config_dic = self.get_result_dic()
local_config_json = JsonUtil.dic_2_json(local_config_dic)
result_config_json = JsonUtil.dic_2_json(result_config_dic)
if local_config_json == result_config_json:
LogUtil.print_e('nothing need to update')
return True
result_file = open(config_file_path, 'wb')
result_file.write(result_config_json)
return True
def disable_ip(self, ip, port):
conn = mysql.connector.connect(host=settings.MYSQL_HOST, user=settings.MYSQL_USER, password=settings.MYSQL_PASSWD, db=settings.MYSQL_PROXY_DBNAME)
cursor = conn.cursor()
sqlStr = "update ip_proxy_info i set i.isvalid=0 where i.ip='%s' and i.port='%s'" % (ip, port)
try:
cursor.execute(sqlStr)
logging.info(LogUtil.get_time_now() + 'Disableing ip proxy %s' % ip)
except:
print LogUtil.get_time_now(), traceback.print_exc()
conn.commit()
cursor.close()
conn.close()
def printsum(path,method,cont=False):
fname = None
if cont:
fname = _getnextfname(method)
if isdir(path):
#walk here
for root, dirs, files in walk(path):
for file in files:
if cont:
if fname == relpath(join(root,file)):
log.debug("FOUND: %s" % fname)
fname = _getnextfname(method)
continue
else:
log.warn("NOT FOUND: %s is not %s" % (fname,relpath(join(root,file))))
log.info("Continue from this point? y/(n):")
yesno = raw_input()
if yesno != "y":
"Bailing..."
exit()
cont=False
if method == "crc":
LogUtil.printlog("%s %s" % (relpath(join(root,file)), getsum(join(root,file), method)))
else:
LogUtil.printlog("%s %s" % (getsum(join(root,file), method), relpath(join(root,file))))
else:
if method == "crc":
LogUtil.printlog("%s %s" % (relpath(path), getsum(path,method)))
else:
LogUtil.printlog("%s %s" % (getsum(path,method), relpath(path)))
def _do_insert(self, conn, item, spider):
try:
conn.execute("select * from base_location where lac=%s and cid=%s", (item['lac'], item['cid']))
ret = conn.fetchone()
if ret:
logging.info(LogUtil.get_time_now() + "do db update, item ==>" + str(item))
conn.execute("update base_location set errcode=%s, lat=%s, lon=%s, address=%s, type=%s, update_time=now() where lac=%s and cid=%s",
(item['errcode'], item['lat'], item['lon'], item['address'], self._get_base_type(item['lac'], item['cid']), item['lac'], item['cid']))
else:
logging.info(LogUtil.get_time_now() + "do db insert, item ==>" + str(item))
conn.execute("insert into base_location (errcode, lac, cid, lat, lon, address, type, update_time) values (%s, %s, %s, %s, %s, %s, %s, now())",
(item['errcode'], item['lac'], item['cid'], item['lat'], item['lon'], item['address'], self._get_base_type(item['lac'], item['cid'])))
except:
logging.info(LogUtil.get_time_now() + traceback.format_exc())
def checkpath(path, method):
if not exists(path):
log.warn("Not found: %s" % path)
return
if isdir(path):
for fname in listdir(path):
if isdir(join(path,fname)):
LogUtil.printlog("--- DIR: %s ---" % fname)
checkpath(join(path,fname), method)
LogUtil.printlog("--- END ---")
else:
checkfile(join(path,fname), method)
else:
checkfile(path, method)
def getsum(path, method):
f = open(path,"rb")
progress = None
size = stat(path).st_size
if Globals.progress and size != 0:
widgets = ["%s" % Console.trimstring(basename(path).encode("utf-8")), Percentage(), ' ', Bar(left="[", right="]"), ' ', ETA()]
progress = ProgressBar(term_width=Globals.consolesize, widgets=widgets, maxval=size).start()
#lol arbitrary 2MB
try: h = Hash(f.read(2097152), method)
except Exception as e:
LogUtil.printlog("Error: %s in %s" % (e, path))
return
if Globals.progress and progress: progress.update(f.tell())
try: chunk = f.read(2097152)
except Exception as e:
LogUtil.printlog("Error: %s in %s" % (e, path))
return
if Globals.progress and progress: progress.update(f.tell())
while chunk:
h.update(chunk)
try: chunk = f.read(2097152)
except Exception as e:
LogUtil.printlog("Error: %s in %s" % (e, path))
return
if Globals.progress and progress: progress.update(f.tell())
f.close()
#if Globals.progress: progress.finish()
if Globals.progress: Console.clearline()
return h.gibehex()
def checksums(fname, method):
f = codopen(fname,"rb","utf-8")
mode = None
if splitext(path)[1] == ".crc" or splitext(path)[1] == ".sfv":
mode = "crc"
else: mode = "md5"
lineno = 0
for line in f:
if line.startswith("#") or line.startswith(";"):
lineno += 1
continue
line = line.strip()
hash = None
if mode == "crc":
try: fname, hash = line.rsplit(None, 1)
except:
log.warn("Invalid line (%s): %s" % (lineno, line))
Globals.invalid += 1
else:
try: hash, fname = line.split(None, 1)
except:
log.warn("Invalid line (%s): %s" % (lineno, line))
Globals.invalid += 1
lineno += 1
if isabs(fname):
if exists(fname):
compsum(fname,hash,mode)
else:
LogUtil.printlog("File not found: %s" % fname)
Globals.notfound += 1
else:
if Globals.options.aster:
if fname[0] == "*":
fname = fname[1:]
if exists(join(Globals.cwd, fname)):
compsum(join(Globals.cwd, fname),hash,mode)
else:
LogUtil.printlog("File not found: %s" % fname)
def process_request(self, request, spider):
print request.meta
if settings.PROXY:
if request.meta.has_key('tried'):
request.meta['tried'] = int(request.meta['tried']) + 1
else:
proxy = random.choice(settings.PROXY)
request.meta['proxy'] = "%s://%s:%s" % (proxy['protocol'], proxy['ip'], proxy['port'])
request.meta['ip'] = proxy['ip']
request.meta['port'] = proxy['port']
request.meta['tried'] = 1
logging.info(LogUtil.get_time_now() + "use ip: %s proxy: %s, try for %s times" % (request.url, request.meta['proxy'], request.meta['tried']))
else:
logging.info('NO PROXY')
def update_remote_repository(self):
"更新远程仓库"
os.chdir(Constant.GIT_LOCAL_REPOSITORY_PATH)
local_time = time.localtime(time.time())
local_time_str = time.strftime("%Y-%m-%d %H:%M:%S", local_time)
git_add_str = 'git add *'
git_commit_str = 'git commit -m "Auto Commit Time ' + local_time_str + '"'
git_push_str = 'git push --force'
# add 更新文件
(status, output) = commands.getstatusoutput(git_add_str)
# commit 文件提交本地
(status, output) = commands.getstatusoutput(git_commit_str)
# push 远程仓库
(status, output) = commands.getstatusoutput(git_push_str)
if status == 0:
LogUtil.print_i(output)
LogUtil.print_i('success')
else:
LogUtil.print_e(output)
LogUtil.print_e('error')
return
class DbUtil(object):
LOG = LogUtil.getLogger()
def __init__(self):
self.LOG.info("数据库操作工具类初始化完成")
def getConnect(self):
self.LOG.info("获取数据库连接")
try:
return pymysql.connect(host=Config.mysql_host, port=Config.mysql_port, user=Config.mysql_user,
passwd=Config.mysql_passwd, db=Config.mysql_db, charset=Config.mysql_charset)
except Exception as e:
error = '数据库连接异常! ERROR : %s' % (e)
self.LOG.error(error)
return None
def dbClose(self, db):
self.LOG.info("关闭数据库连接")
if db:
db.close()
def _query(self, sql=''):
self.LOG.info("执行查询语句，SQL:" + sql)
db = self.getConnect()
try:
cursor = db.cursor()
# 执行SQL语句
cursor.execute(sql)
# 获取所有记录列表
results = cursor.fetchall()
return results
except Exception as e:
error = '数据查询异常! ERROR : %s' % (e)
self.LOG.error(error)
finally:
self.dbClose(db)
def printtree(node, options, level):
#if not dirfirst, combine lists
#sort lists
if options.depth >= 0 and level > options.depth:
return
combined = None
if options.sortfirst:
node.dirs.sort()
node.files.sort()
if options.dirfirst: combined = node.dirs+node.files
else: combined = node.files+node.dirs
else:
if options.dirfirst: combined = node.dirs+node.files
else: combined = node.files+node.dirs
combined.sort()
prefix = options.indent*level
#{name} #{size} #{modtime} #{createtime}
for item in combined:
if options.minsize and (item.size < options.minsize): continue
if options.maxsize and (item.size > options.maxsize): continue
size = item.size if not options.human else size_to_human(item.size, round=options.rounding)
if isinstance(item, Dir):
#recurse
if not options.nodirs:
if options.dirnamebefore:
LogUtil.printlog(options.dir.format(name=item.name, path=item.path, size=size, modtime=item.modtime,
createtime=item.createtime, prefix=prefix))
printtree(item, options, level+1)
if not options.nodirs:
if not options.dirnamebefore:
LogUtil.printlog(options.dir.format(name=item.name, path=item.path, size=size, modtime=item.modtime,
createtime=item.createtime, prefix=prefix))
else:
if not options.nofiles:
#print stats
LogUtil.printlog(options.file.format(name=item.name, path=item.parent.path, size=size, modtime=item.modtime,
createtime=item.createtime, prefix=prefix))
def __init__(self, **args) -> None:
self._logger = LogUtil.create(args['options']['log_level'])
self.use_case = PluginUseCase(args['options'])
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException, TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
import logging
from selenium.webdriver.support.select import Select
from selenium.webdriver.support.wait import WebDriverWait
from util import LogUtil
BASE_URL: str = "https://tellburgerking.com.cn/"
# TODO 调查码应该在文件中读取或者以参数传递
BASE_CODE: str = "997723220095015AAA9"
log1 = LogUtil.LogHelper()
# 校验调查码的长度、形态是否合法，并且修正格式
if not BASE_CODE.isdigit():
logging.error("请输入正确的调查码")
sys.exit()
BASE_CODE = BASE_CODE.strip()
if len(BASE_CODE) != 16:
logging.error("请输入16位调查码")
sys.exit()
options = webdriver.ChromeOptions()
# 静默模式 无图模式
prefs = {'profile.managed_default_content_settings.images': 2}
options.add_experimental_option('prefs', prefs)
options.add_argument("--headless")
def __init__(self, options: Dict) -> None:
self._logger = LogUtil.create(options['log_level'])
self.plugins_package: str = options['directory']
self.plugin_util = PluginUtility(self._logger)
self.modules = list()
def _handle_error(self, failure, item, spider):
logging.info(LogUtil.get_time_now(), "Error ==>" + failure)
for path in paths:
drive, rest = splitdrive(path)
drive = drives[drive.upper()]
drive.stats = True
progress = None
if options.progress:
#allow progress
if not rest.lstrip("/\\"):
widgets = ["Reading %s" % drive.letter, Percentage(), ' ', Bar(left="[", right="]"), ' ', ETA()]
progress = ProgressWrapper(ProgressBar(term_width=consolesize, widgets=widgets, maxval=drive.totalsize-drive.free).start())
else:
widgets = ["Reading %s " % drive.letter, AnimatedMarker(), ' ', Timer()]
progress = ProgressWrapper(ProgressBar(term_width=consolesize, widgets=widgets, maxval=drive.totalsize-drive.free).start())
start(path, drive, progress)
if progress: progress.finish()
#do final printing stuff here I guess
for drive in sorted(drives.keys()):
if drives[drive].stats:
d = drives[drive]
#{name}{letter}{fs}{free}{totalsize}{serial}{type}{stats}{size}
size = d.size if not options.human else size_to_human(d.size)
if options.drivehead:
LogUtil.printlog(options.drivehead.format(name=d.name, letter=d.letter, fs=d.fs, free=d.free, totalsize=d.totalsize,
serial=d.serial, type=d.type, stats=d.stats, size=size, totaldirs=d.totaldirs, totalfiles=d.totalfiles))
printtree(d, options, 0)
if options.drivefoot:
LogUtil.printlog(options.drivefoot.format(name=d.name, letter=d.letter, fs=d.fs, free=d.free, totalsize=d.totalsize,
serial=d.serial, type=d.type, stats=d.stats, size=size, totaldirs=d.totaldirs, totalfiles=d.totalfiles))
#!/usr/bin/python
import datetime
import os
import sys
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(base_dir)
import util.DbUtil as DbUtil
import util.LogUtil as LogUtil
dbUtil = DbUtil.DbUtil()
LOG = LogUtil.getLogger()
# 广告点击记录每日汇总
def day():
LOG.info("广告点击记录每日汇总开始=============>")
db = dbUtil.getConnect()
try:
cursor = db.cursor()
sql = '''SELECT DATE_FORMAT(NOW(),'%Y-%m-%d %H')'''
cursor.execute(sql)
now_str = cursor.fetchone()[0]
now = datetime.datetime.strptime(now_str, '%Y-%m-%d %H')
count = 1
if now_str[11:13] == '00':
now = (now + datetime.timedelta(days=-2))
count = 2
else:
def _handle_error(self, failure, item, spider):
logging.info(LogUtil.get_time_now(), "Error ==>" + failure)
# -*- coding: utf-8 -*-
from logging import getLogger, config, StreamHandler, DEBUG
import settings
import json
from difflib import SequenceMatcher
import sys
sys.path.append('./')
from commentutil import OutputCommentKeyEnum, CommentTypeEnum
from util import LogUtil
from judgement import NotUseMPLGJudgement, UseMPLGJudegement
logger = getLogger(__name__)
log_conf = LogUtil.get_log_conf('./log_config.json')
config.dictConfig(log_conf)
handler = StreamHandler()
handler.setLevel(DEBUG)
logger.setLevel(DEBUG)
logger.addHandler(handler)
logger.propagate = False
# envのキーを読み込む。
VIDEO_ID = settings.ENV_DIC['video_id']
# 形態素解析を使用するかどうか。
USE_MPLG = bool(int(settings.ENV_DIC['mplg']))
# 類似度閾値
SIMILARITY_THRESHOLD = float(settings.ENV_DIC['similarity_threshold'])
OUTPUT_DIR_ALL = './output/all/'
def test_log_debug(self):
print(LogUtil.log_debug("debug"))
print(LogUtil.log_info("info"))
print(LogUtil.log_warn("warn"))
print(LogUtil.log_error("error"))