示例#1
0
    def _do_insert(self, conn, item, spider):
        try:
            conn.execute("select * from base_location where lac=%s and cid=%s",
                         (item['lac'], item['cid']))
            ret = conn.fetchone()

            if ret:
                logging.info(LogUtil.get_time_now() +
                             "do db update, item ==>" + str(item))
                conn.execute(
                    "update base_location set errcode=%s, lat=%s, lon=%s, address=%s, type=%s, update_time=now() where lac=%s and cid=%s",
                    (item['errcode'], item['lat'], item['lon'],
                     item['address'],
                     self._get_base_type(
                         item['lac'], item['cid']), item['lac'], item['cid']))
            else:
                logging.info(LogUtil.get_time_now() +
                             "do db insert, item ==>" + str(item))
                conn.execute(
                    "insert into base_location (errcode, lac, cid, lat, lon, address, type, update_time) values (%s, %s, %s, %s, %s, %s, %s, now())",
                    (item['errcode'], item['lac'], item['cid'], item['lat'],
                     item['lon'], item['address'],
                     self._get_base_type(item['lac'], item['cid'])))
        except:
            logging.info(LogUtil.get_time_now() + traceback.format_exc())
示例#2
0
def compsum(fname, hash, mode):
	newhash = getsum(fname,mode)
	if hash != newhash:
		LogUtil.printlog("Hash mismatch: Expecting: %s got %s for %s" % (hash, newhash, fname))
		Globals.failed += 1
	else:
		if Globals.verbose: LogUtil.printlog("Hash match: Expecting: %s got %s for %s" % (hash, newhash, fname))
		Globals.passed += 1
示例#3
0
 def excute(self):
     "excute Method"
     service = RemoteConfigService.RemoteConfigService()
     while True:
         current_time = time.strftime('%Y-%m-%d %X', time.localtime())
         LogUtil.print_i('Excute RemoteConfig update at ' + current_time)
         service.update_remote_config()
         time.sleep(60)
     return
示例#4
0
def checkfile(path, method):
	checksum = getsum(path, method)
	if checksum in basename(path) or checksum in basename(path).upper():
		Globals.passed += 1
		if Globals.verbose:
			LogUtil.printlog("Found: %s in: %s" % (checksum, path))
	else:
		Globals.failed += 1
		LogUtil.printlog("Not found: %s in: %s" % (checksum,path))
 def update_config_file(self):
     "update config file"
     config_file_path = Constant.GIT_LOCAL_CONFIG_FILE_PATH
     local_config_dic = JsonUtil.json_file_2_dic(config_file_path)
     result_config_dic = self.get_result_dic()
     local_config_json = JsonUtil.dic_2_json(local_config_dic)
     result_config_json = JsonUtil.dic_2_json(result_config_dic)
     if local_config_json == result_config_json:
         LogUtil.print_e('nothing need to update')
         return True
     result_file = open(config_file_path, 'wb')
     result_file.write(result_config_json)
     return True
示例#6
0
 def disable_ip(self, ip, port):
     conn = mysql.connector.connect(host=settings.MYSQL_HOST, user=settings.MYSQL_USER, password=settings.MYSQL_PASSWD, db=settings.MYSQL_PROXY_DBNAME)
     cursor = conn.cursor()
     sqlStr = "update ip_proxy_info i set i.isvalid=0 where i.ip='%s' and i.port='%s'" % (ip, port)
     
     try:
         cursor.execute(sqlStr)    
         logging.info(LogUtil.get_time_now() + 'Disableing ip proxy %s' % ip)
     except:
         print LogUtil.get_time_now(), traceback.print_exc()
         
     conn.commit()
     cursor.close()
     conn.close()
示例#7
0
def printsum(path,method,cont=False):
	fname = None
	if cont:
		fname = _getnextfname(method)
	if isdir(path):
		#walk here
		for root, dirs, files in walk(path):
			for file in files:
				if cont:
					if fname == relpath(join(root,file)):
						log.debug("FOUND: %s" % fname)
						fname = _getnextfname(method)
						continue
					else:
						log.warn("NOT FOUND: %s is not %s" % (fname,relpath(join(root,file))))
						log.info("Continue from this point? y/(n):")
						yesno = raw_input()
						if yesno != "y": 
							"Bailing..."
							exit()
						cont=False
				
				if method == "crc":
					LogUtil.printlog("%s %s" % (relpath(join(root,file)), getsum(join(root,file), method)))
				else:
					LogUtil.printlog("%s %s" % (getsum(join(root,file), method), relpath(join(root,file))))
	else:
		if method == "crc":
			LogUtil.printlog("%s %s" % (relpath(path), getsum(path,method)))
		else:
			LogUtil.printlog("%s %s" % (getsum(path,method), relpath(path)))
示例#8
0
 def _do_insert(self, conn, item, spider):
     try:
         conn.execute("select * from base_location where lac=%s and cid=%s", (item['lac'], item['cid']))
         ret = conn.fetchone()
         
         if ret:
             logging.info(LogUtil.get_time_now() + "do db update, item ==>" + str(item))
             conn.execute("update base_location set errcode=%s, lat=%s, lon=%s, address=%s, type=%s, update_time=now() where lac=%s and cid=%s", 
                     (item['errcode'], item['lat'], item['lon'], item['address'], self._get_base_type(item['lac'], item['cid']), item['lac'], item['cid']))
         else:
             logging.info(LogUtil.get_time_now() + "do db insert, item ==>" + str(item))
             conn.execute("insert into base_location (errcode, lac, cid, lat, lon, address, type, update_time) values (%s, %s, %s, %s, %s, %s, %s, now())",
                 (item['errcode'], item['lac'], item['cid'], item['lat'], item['lon'], item['address'], self._get_base_type(item['lac'], item['cid'])))
     except:
         logging.info(LogUtil.get_time_now() + traceback.format_exc())
示例#9
0
def checkpath(path, method):
	if not exists(path): 
		log.warn("Not found: %s" % path)
		return
		
	if isdir(path):
		for fname in listdir(path):
			if isdir(join(path,fname)):
				LogUtil.printlog("--- DIR: %s ---" % fname)
				checkpath(join(path,fname), method)
				LogUtil.printlog("--- END ---")
			else:
				checkfile(join(path,fname), method)
	else:
		checkfile(path, method)
示例#10
0
def getsum(path, method):
	f = open(path,"rb")
	progress = None
	size = stat(path).st_size
	if Globals.progress and size != 0:
		widgets = ["%s" % Console.trimstring(basename(path).encode("utf-8")), Percentage(), ' ', Bar(left="[", right="]"), ' ', ETA()]
		progress = ProgressBar(term_width=Globals.consolesize, widgets=widgets, maxval=size).start()
	#lol arbitrary 2MB
	try: h = Hash(f.read(2097152), method)
	except Exception as e: 
		LogUtil.printlog("Error: %s in %s" % (e, path))
		return
	if Globals.progress and progress: progress.update(f.tell())
	try: chunk = f.read(2097152)
	except Exception as e: 
		LogUtil.printlog("Error: %s in %s" % (e, path))
		return
	if Globals.progress and progress: progress.update(f.tell())
	while chunk:
		h.update(chunk)
		try: chunk = f.read(2097152)
		except Exception as e: 
			LogUtil.printlog("Error: %s in %s" % (e, path))
			return
		if Globals.progress and progress: progress.update(f.tell())
	f.close()
	#if Globals.progress: progress.finish()
	if Globals.progress: Console.clearline()
	return h.gibehex()
示例#11
0
def checksums(fname, method):
	f = codopen(fname,"rb","utf-8")
	mode = None
	if splitext(path)[1] == ".crc" or splitext(path)[1] == ".sfv":
		mode = "crc"
	else: mode = "md5"
	
	lineno = 0
	for line in f:
		if line.startswith("#") or line.startswith(";"):
			lineno += 1
			continue
		line = line.strip()
		hash = None
		if mode == "crc":
			try: fname, hash = line.rsplit(None, 1)
			except:	
				log.warn("Invalid line (%s): %s" % (lineno, line))
				Globals.invalid += 1
		else:
			try: hash, fname = line.split(None, 1)
			except:	
				log.warn("Invalid line (%s): %s" % (lineno, line))
				Globals.invalid += 1
		lineno += 1
		
		if isabs(fname):
			if exists(fname):
				compsum(fname,hash,mode)
			else:
				LogUtil.printlog("File not found: %s" % fname)
				Globals.notfound += 1
		else:
			if Globals.options.aster:
				if fname[0] == "*":
					fname = fname[1:]
			if exists(join(Globals.cwd, fname)):
				compsum(join(Globals.cwd, fname),hash,mode)
			else:
				LogUtil.printlog("File not found: %s" % fname)
示例#12
0
    def process_request(self, request, spider):
        print request.meta
        if settings.PROXY:
            if request.meta.has_key('tried'):
                request.meta['tried'] = int(request.meta['tried']) + 1
            else:
                proxy = random.choice(settings.PROXY)
                request.meta['proxy'] = "%s://%s:%s" % (proxy['protocol'], proxy['ip'], proxy['port'])
                request.meta['ip'] = proxy['ip']
                request.meta['port'] = proxy['port']
                request.meta['tried'] = 1

            logging.info(LogUtil.get_time_now() + "use ip: %s proxy: %s, try for %s times" % (request.url, request.meta['proxy'], request.meta['tried']))
        else:
            logging.info('NO PROXY')
 def update_remote_repository(self):
     "更新远程仓库"
     os.chdir(Constant.GIT_LOCAL_REPOSITORY_PATH)
     local_time = time.localtime(time.time())
     local_time_str = time.strftime("%Y-%m-%d %H:%M:%S", local_time)
     git_add_str = 'git add *'
     git_commit_str = 'git commit -m "Auto Commit Time ' + local_time_str + '"'
     git_push_str = 'git push --force'
     # add 更新文件
     (status, output) = commands.getstatusoutput(git_add_str)
     # commit 文件提交本地
     (status, output) = commands.getstatusoutput(git_commit_str)
     # push 远程仓库
     (status, output) = commands.getstatusoutput(git_push_str)
     if status == 0:
         LogUtil.print_i(output)
         LogUtil.print_i('success')
     else:
         LogUtil.print_e(output)
         LogUtil.print_e('error')
     return
示例#14
0
class DbUtil(object):
    LOG = LogUtil.getLogger()

    def __init__(self):
        self.LOG.info("数据库操作工具类初始化完成")

    def getConnect(self):
        self.LOG.info("获取数据库连接")
        try:
            return pymysql.connect(host=Config.mysql_host, port=Config.mysql_port, user=Config.mysql_user,
                                   passwd=Config.mysql_passwd, db=Config.mysql_db, charset=Config.mysql_charset)
        except Exception as e:
            error = '数据库连接异常! ERROR : %s' % (e)
            self.LOG.error(error)
            return None

    def dbClose(self, db):
        self.LOG.info("关闭数据库连接")
        if db:
            db.close()

    def _query(self, sql=''):
        self.LOG.info("执行查询语句,SQL:" + sql)
        db = self.getConnect()
        try:
            cursor = db.cursor()
            # 执行SQL语句
            cursor.execute(sql)
            # 获取所有记录列表
            results = cursor.fetchall()
            return results
        except Exception as e:
            error = '数据查询异常! ERROR : %s' % (e)
            self.LOG.error(error)
        finally:
            self.dbClose(db)
示例#15
0
def printtree(node, options, level):
	#if not dirfirst, combine lists
	#sort lists
	if options.depth >= 0 and level > options.depth:
		return
	combined = None
	if options.sortfirst:
		node.dirs.sort()
		node.files.sort()
		if options.dirfirst: combined = node.dirs+node.files
		else: combined = node.files+node.dirs
	else:
		if options.dirfirst: combined = node.dirs+node.files
		else: combined = node.files+node.dirs
		combined.sort()
	prefix = options.indent*level
	
	#{name} #{size} #{modtime} #{createtime}
	for item in combined:
		if options.minsize and (item.size < options.minsize): continue
		if options.maxsize and (item.size > options.maxsize): continue
		size = item.size if not options.human else size_to_human(item.size, round=options.rounding)
		if isinstance(item, Dir):
			#recurse
			if not options.nodirs:
				if options.dirnamebefore:
					LogUtil.printlog(options.dir.format(name=item.name, path=item.path, size=size, modtime=item.modtime,
						createtime=item.createtime, prefix=prefix))
			printtree(item, options, level+1)
			if not options.nodirs:
				if not options.dirnamebefore:
					LogUtil.printlog(options.dir.format(name=item.name, path=item.path, size=size, modtime=item.modtime,
						createtime=item.createtime, prefix=prefix))
		else:
			if not options.nofiles:
				#print stats
				LogUtil.printlog(options.file.format(name=item.name, path=item.parent.path, size=size, modtime=item.modtime,
					createtime=item.createtime, prefix=prefix))
示例#16
0
 def __init__(self, **args) -> None:
     self._logger = LogUtil.create(args['options']['log_level'])
     self.use_case = PluginUseCase(args['options'])
示例#17
0
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException, TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
import logging

from selenium.webdriver.support.select import Select
from selenium.webdriver.support.wait import WebDriverWait

from util import LogUtil

BASE_URL: str = "https://tellburgerking.com.cn/"
# TODO 调查码应该在文件中读取或者以参数传递
BASE_CODE: str = "997723220095015AAA9"
log1 = LogUtil.LogHelper()

# 校验调查码的长度、形态是否合法,并且修正格式
if not BASE_CODE.isdigit():
    logging.error("请输入正确的调查码")
    sys.exit()
BASE_CODE = BASE_CODE.strip()
if len(BASE_CODE) != 16:
    logging.error("请输入16位调查码")
    sys.exit()

options = webdriver.ChromeOptions()
# 静默模式 无图模式
prefs = {'profile.managed_default_content_settings.images': 2}
options.add_experimental_option('prefs', prefs)
options.add_argument("--headless")
示例#18
0
 def __init__(self, options: Dict) -> None:
     self._logger = LogUtil.create(options['log_level'])
     self.plugins_package: str = options['directory']
     self.plugin_util = PluginUtility(self._logger)
     self.modules = list()
示例#19
0
 def _handle_error(self, failure, item, spider):
     logging.info(LogUtil.get_time_now(), "Error ==>" + failure)
示例#20
0
for path in paths:
	drive, rest = splitdrive(path)
	drive = drives[drive.upper()]
	drive.stats = True
	progress = None
	if options.progress:
		#allow progress
		if not rest.lstrip("/\\"):
			widgets = ["Reading %s" % drive.letter, Percentage(), ' ', Bar(left="[", right="]"), ' ', ETA()]
			progress = ProgressWrapper(ProgressBar(term_width=consolesize, widgets=widgets, maxval=drive.totalsize-drive.free).start())
		else:
			widgets = ["Reading %s " % drive.letter, AnimatedMarker(), ' ', Timer()]
			progress = ProgressWrapper(ProgressBar(term_width=consolesize, widgets=widgets, maxval=drive.totalsize-drive.free).start())
	start(path, drive, progress)
	if progress: progress.finish()
	
#do final printing stuff here I guess
for drive in sorted(drives.keys()):
	if drives[drive].stats:
		d = drives[drive]
		#{name}{letter}{fs}{free}{totalsize}{serial}{type}{stats}{size}
		size = d.size if not options.human else size_to_human(d.size)
		if options.drivehead:
			LogUtil.printlog(options.drivehead.format(name=d.name, letter=d.letter, fs=d.fs, free=d.free, totalsize=d.totalsize,
				serial=d.serial, type=d.type, stats=d.stats, size=size, totaldirs=d.totaldirs, totalfiles=d.totalfiles))
		printtree(d, options, 0)
		if options.drivefoot:
			LogUtil.printlog(options.drivefoot.format(name=d.name, letter=d.letter, fs=d.fs, free=d.free, totalsize=d.totalsize,
				serial=d.serial, type=d.type, stats=d.stats, size=size, totaldirs=d.totaldirs, totalfiles=d.totalfiles))
示例#21
0
#!/usr/bin/python

import datetime
import os
import sys

base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(base_dir)

import util.DbUtil as DbUtil
import util.LogUtil as LogUtil

dbUtil = DbUtil.DbUtil()
LOG = LogUtil.getLogger()


# 广告点击记录每日汇总
def day():
    LOG.info("广告点击记录每日汇总开始=============>")
    db = dbUtil.getConnect()
    try:
        cursor = db.cursor()
        sql = '''SELECT DATE_FORMAT(NOW(),'%Y-%m-%d %H')'''
        cursor.execute(sql)
        now_str = cursor.fetchone()[0]
        now = datetime.datetime.strptime(now_str, '%Y-%m-%d %H')
        count = 1
        if now_str[11:13] == '00':
            now = (now + datetime.timedelta(days=-2))
            count = 2
        else:
示例#22
0
 def _handle_error(self, failure, item, spider):
     logging.info(LogUtil.get_time_now(), "Error ==>" + failure)
示例#23
0
# -*- coding: utf-8 -*-
from logging import getLogger, config, StreamHandler, DEBUG
import settings
import json
from difflib import SequenceMatcher

import sys
sys.path.append('./')
from commentutil import OutputCommentKeyEnum, CommentTypeEnum
from util import LogUtil
from judgement import NotUseMPLGJudgement, UseMPLGJudegement

logger = getLogger(__name__)
log_conf = LogUtil.get_log_conf('./log_config.json')
config.dictConfig(log_conf)
handler = StreamHandler()
handler.setLevel(DEBUG)
logger.setLevel(DEBUG)
logger.addHandler(handler)
logger.propagate = False

# envのキーを読み込む。
VIDEO_ID = settings.ENV_DIC['video_id']

# 形態素解析を使用するかどうか。
USE_MPLG = bool(int(settings.ENV_DIC['mplg']))

# 類似度閾値
SIMILARITY_THRESHOLD = float(settings.ENV_DIC['similarity_threshold'])

OUTPUT_DIR_ALL = './output/all/'
示例#24
0
 def test_log_debug(self):
     print(LogUtil.log_debug("debug"))
     print(LogUtil.log_info("info"))
     print(LogUtil.log_warn("warn"))
     print(LogUtil.log_error("error"))