def isValidHist(rootHist, histname, filename):
if not rootHist:
Log.logErrorMessage(
'Histogram \n"%s" \ncould not be found in rootFile:\n%s' %
(histname, filename))
return False
return True
def truncate(self, path, length, fh=None):
Log('truncate', path, length, level=1)
full_path = self._full_path(path)
with open(
full_path, 'r+'
) as f: # [r+] = [read & write] , [w+] = [read & write] , if file not exist,r+ calls error while w+ not
f.truncate(length)
def readlink(self, path):
Log('readlink', path, level=1)
pathname = os.readlink(self._full_path(path))
if pathname.startswith("/"):
# Path name is absolute, sanitize it.
return os.path.relpath(pathname, self.root)
else:
return pathname
def __init__(self):
# option from command line
self.verbose = False
self.log = Log()
# relevant directories
self.home = os.path.expanduser('~')
self.working = os.getcwd()
self.main = None
try:
exeFile = os.path.abspath(sys.modules['__main__'].__file__)
srcFile = os.path.realpath(exeFile) # resolve symlinks
self.main = os.path.dirname(srcFile)
except:
self.log.warn('no main file path, interactive interpreter')
self.find = _Find(self)
# operating system type
self.os = _which_os()
def statfs(self, path):
Log('statfs', path, level=1)
full_path = self._full_path(path)
stv = os.statvfs(full_path)
return dict((key, getattr(stv, key))
for key in ('f_bavail', 'f_bfree', 'f_blocks', 'f_bsize',
'f_favail', 'f_ffree', 'f_files', 'f_flag',
'f_frsize', 'f_namemax'))
def getattr(self, path, fh=None):
Log('getattr', path, fh, level=1)
full_path = self._full_path(path)
st = os.lstat(
full_path) # os.stat follow the symbol link while os.lstat not
ans = dict((key, getattr(st, key))
for key in ('st_atime', 'st_ctime', 'st_gid', 'st_mode',
'st_mtime', 'st_nlink', 'st_size', 'st_uid'))
return ans
def readdir(self, path, fh):
Log('readdir', path, level=1)
full_path = self._full_path(path)
dirents = ['.', '..']
if os.path.isdir(full_path):
dirents.extend(os.listdir(full_path))
for r in dirents:
yield r
def unlink(self, path):
Log('unlink', path, level=1)
return os.unlink(self._full_path(path))
def create(self, path, mode, fi=None):
Log('create', path, mode, level=1)
full_path = self._full_path(path)
return os.open(full_path, os.O_WRONLY | os.O_CREAT, mode)
def flush(self, path, fh):
Log('flush', path, level=1)
return os.fsync(fh)
class Shell():
def __init__(self):
# option from command line
self.verbose = False
self.log = Log()
# relevant directories
self.home = os.path.expanduser('~')
self.working = os.getcwd()
self.main = None
try:
exeFile = os.path.abspath(sys.modules['__main__'].__file__)
srcFile = os.path.realpath(exeFile) # resolve symlinks
self.main = os.path.dirname(srcFile)
except:
self.log.warn('no main file path, interactive interpreter')
self.find = _Find(self)
# operating system type
self.os = _which_os()
def respond(self, cmd_str_or_lst, shell=False, strip=False):
cmd = ' '.join(cmd_str_or_lst) if (type(cmd_str_or_lst) == type(
[])) else cmd_str_or_lst
if self.verbose: print(cmd)
byte_output = check_output(cmd, shell=shell)
string_output = byte_output.decode('utf-8')
if strip: return string_output.strip()
return string_output
def basename(self, path):
basename = os.path.basename(path)
if self.verbose: print('Basename of ', path, ' is ', basename)
return basename
def cd(self, path):
if self.verbose: print('Changing directory to: ', path)
os.chdir(path)
def chrome(self, url):
if self.os == 'linux':
self.command([
'google-chrome-stable', '--disable-features=NetworkService',
url, '&>/dev/null'
])
elif self.os == 'mac':
self.command(['open -a', r'Google\ Chrome', url])
def command(self, cmd_list):
if isinstance(cmd_list, str): cmd = cmd_list
else: cmd = ' '.join(cmd_list)
if self.verbose: print(cmd)
status = os.system(cmd)
return status
def cp(self, from_file, to_file):
if self.verbose: print('Copying from ', from_file, ' to ', to_file)
if self.exists(to_file): self.rm(to_file)
if os.path.isdir(from_file): shutil.copytree(from_file, to_file)
else: shutil.copyfile(from_file, to_file)
def rm(self, path):
if self.verbose: print('Recursively removing: ', path)
if os.path.isdir(path): shutil.rmtree(path)
else: os.remove(path)
def dirname(self, path):
dirname = os.path.dirname(path)
if self.verbose: print('Dirname of ', path, ' is ', dirname)
return dirname
def exists(self, path): # check for path of directory
if self.verbose: print('Checking the existance of path: ', path)
return os.path.exists(path.strip())
def is_dir(self, path): # check for path of directory
if self.verbose: print('Checking if is dir, path: ', path)
return os.path.isdir(path.strip())
def kill(self, command_name):
if self.os == 'mac':
pid = self.pid(command_name)
else:
self.log.error(
'Shell.kill is not yet implemented for this operating system')
status = self.command(['kill', '-9', pid])
if status == 0:
self.log.validate('killed ' + command_name + ' running with id ' +
pid)
def link(self, src, dest):
os.symlink(src, dest)
def readlink(self, link_path):
source = os.readlink(link_path)
if self.exists(source):
if self.verbose:
self.log.validate('the symlink at, ' + link_path +
', resolved')
return source
if self.verbose:
print('link, ' + link_path + ', exists, but did not resolve')
return False
def ls(self, path):
if self.verbose: print('Listing files in directory: ', path)
return os.listdir(path)
def make_executable(self, file):
self.command(['chmod +x', file])
def mkdir(self, path):
if self.is_dir(path):
self.log.validate('directory, ' + path + ', already exists')
return
os.mkdir(path)
if self.verbose: print('made directory: ', path)
def mv(self, from_path, to_path):
try:
os.rename(from_path, to_path)
if self.verbose:
print('renaming directory: ', from_path, ' to ', to_path)
except Exception as e:
raise e
def pid(self, command_name):
try:
if self.os == 'mac':
pid = self.respond(['pgrep', command_name],
shell=True,
strip=True)
else:
self.log.error(
'Shell.pid is not yet implemented for this operating system'
)
except:
self.log.error(
'failed to get a process id for the command name: ' +
command_name)
return pid
def vim(self, name, SystemEditor=False):
if SystemEditor: vim = os.environ.get('EDITOR', 'vim') # create editor
else: vim = 'vim'
if type(name) == type([]):
if len(name) > 1: raise TypeError('too many input arguments')
if len(name) == 0:
self.command([vim])
return 0
else:
name = name[0]
if self.is_dir(name):
self.command([vim, name])
return 0
try:
open_file_with_vim(vim, name, 'r+') # open file to read
except:
open_file_with_vim(vim, name, 'w+') # open file to write
return 0
import dns.resolver
from dns.resolver import NoAnswer
import logging
import socket
import smtplib
from smtplib import SMTPServerDisconnected, SMTPSenderRefused
from tools import email_verificator, domain_extract, MxRecord, Log
log = Log()
emailToVerify = "*****@*****.**"
server_port = None
server_name = None
server_encryption = 'smtp'
user_name = None
password = None
debug_level = 0
email_verify = email_verificator(emailToVerify)
if email_verify is False:
log.error("Email invalid" + emailToVerify)
raise ValueError("Email invalid")
domain = domain_extract(emailToVerify) if server_name is None else server_name
try:
records = dns.resolver.query(domain, 'MX')
except NoAnswer as e:
log.error(message=e)
if __name__ == "__main__":
# init
os.chdir(os.path.join(os.path.dirname(__file__), '..'))
port = 8888
includes = None
opts, argvs = getopt.getopt(sys.argv[1:], "c:p:h")
for op, value in opts:
if op == '-c':
includes = value
path._ETC_PATH = os.path.dirname(os.path.abspath(value))
elif op == '-p':
port = int(value)
elif op == '-h':
print u'''使用参数启动:
usage: [-p|-c]
-p [prot] ******启动端口,默认端口:%d
-c <file> ******加载配置文件
''' % port
sys.exit(0)
if not includes:
includes = os.path.join(path._ETC_PATH, 'includes_dev.json')
print "no configuration found!,will use [%s] instead" % includes
# main
init_application(includes)
application = get_application()
application.listen(port)
logger = Log().getLog()
logger.info("starting..., listen [%d], configurated by (%s)", port,
includes)
tornado.ioloop.IOLoop.instance().start()
def chown(self, path, uid, gid):
Log('chown', path, level=1)
full_path = self._full_path(path)
return os.chown(full_path, uid, gid)
def chmod(self, path, mode):
Log('chmod', path, level=1)
full_path = self._full_path(path)
return os.chmod(full_path, mode)
def access(self, path, mode):
Log('access', path, mode, level=1)
full_path = self._full_path(path)
if not os.access(full_path, mode):
raise FuseOSError(errno.EACCES)
def fsync(self, path, fdatasync, fh):
Log('fsync', path, level=1)
return self.flush(path, fh)
def release(self, path, fh):
Log('release', path, level=1)
return os.close(fh)
def testIfFalidFile(rootFile, filename):
if not rootFile:
Log.logErrorMessage('Could not find rootFile: ' + filename)
def read(self, path, length, offset, fh):
Log('read', path, length, offset, level=1)
os.lseek(fh, offset, os.SEEK_SET)
return os.read(fh, length)
print("Arguments ERROR.")
print("opt:\t" + str(opt))
print("val:\t" + str(val))
sys.exit(1)
src_path = work_dir + "/src"
tools_path = src_path + "/tools"
sys.path.append(src_path)
sys.path.append(tools_path)
#Cumtomize Moduls
import settings
from tools.Platform import *
from tools.MD5 import *
from tools.Log import *
from tools.SeleniumSpider import *
from tools.BS4Spider import *
from tools.RequestsSpider import *
from tools.Proxy import *
from tools.ListTool import *
from tools.MultiProcess import *
from tools.StringTool import *
from tools.Network import *
script_path = os.path.split(os.path.split(os.path.realpath(__file__))[0])[0]
logger = Log("./", "./log.log")
logger.init()
logger.info("info")
logger.error("error")
logger.warning("war")
max_epoch = configs['op']['max_epoch']
learning_rate = configs['op']['learning_rate']
decay_rate = configs['op']['decay_rate']
epoch_steps = configs['op']['epoch_steps']
snapshot = configs['op']['snapshot']
batch_size = configs['db']['batch_size']
loader_threads = configs['db']['loader_threads']
save_dir = configs['system']['save_dir']
# init Timer
if not os.path.exists(save_dir):
os.mkdir(save_dir)
_t = Timer()
# create log file
log = Log(args.log_dir, args.cfg)
log.wr_cfg(configs)
# load data set
training_set = load_data_set_from_factory(configs, 'train')
print('Data set: {} has been loaded'.format(configs['db']['name']))
# load model
trainer = load_training_model_from_factory(configs, ngpu=args.ngpu)
if configs['system']['resume']:
trainer.load_params(configs['system']['resume_path'])
print('Model: {} has been loaded'.format(configs['model']['name']))
# start training
epoch_size = len(training_set) // batch_size # init learning rate & iters
start_iter = start_epoch * epoch_size
def isValidHist(rootHist, histname, filename):
if not rootHist:
Log.logErrorMessage('Histogram \n"%s" \ncould not be found in rootFile:\n%s' % (histname, filename))
return False
return True
#coding:utf-8
import traceback
from bson import *
from pymongo import *
from pymongo.bulk import BulkOperationBuilder
import db_pool
from tools import Log
logger = Log().getLog()
class MongoDataModel(object):
"""mongodb dao
"""
def __init__(self, dbname):
client = db_pool.get_mongo(dbname)
self.db = client.get_default_database()
def find(self,
table,
SON=None,
pn=0,
rn=0,
sort=None,
count=False,
projection=None):
try:
data = []
col = collection.Collection(self.db, table)
cur = col.find(filter=SON,
projection=projection,
skip=pn * rn,
# init
os.chdir(os.path.join(os.path.dirname(__file__), '..'))
port = 8888
includes = None
opts, argvs = getopt.getopt(sys.argv[1:], "c:p:h")
for op, value in opts:
if op == '-c':
includes = value
path._ETC_PATH = os.path.dirname(os.path.abspath(value))
elif op == '-p':
port = int(value)
elif op == '-h':
print u'''使用参数启动:
usage: [-p|-c]
-p [prot] ******启动端口,默认端口:%d
-c <file> ******加载配置文件
''' % port
sys.exit(0)
if not includes:
includes = os.path.join(path._ETC_PATH, 'includes_dev.json')
print "no configuration found!,will use [%s] instead" % includes
# main
init_application(includes)
application = get_application()
application.listen(port)
logger = Log().getLog()
logger.info("starting..., listen [%d], configurated by (%s)", port, includes)
tornado.ioloop.IOLoop.instance().start()
def mknod(self, path, mode, dev):
Log('mknod', path, level=1)
return os.mknod(self._full_path(path), mode, dev)
def write(self, path, buf, offset, fh):
Log('write', path, buf, offset, level=1)
os.lseek(fh, offset, os.SEEK_SET)
return os.write(fh, buf)
def testIfFalidFile(rootFile, filename):
if not rootFile:
Log.logErrorMessage('Could not find rootFile: ' + filename)
def rmdir(self, path):
Log('rmdir', path)
full_path = self._full_path(path)
return os.rmdir(full_path)
#script_path = os.path.split(os.path.split(os.path.realpath(__file__))[0])[0]
#work_dir = script_path
src_path = _WORK_DIR_ + "/src"
tools_path = src_path + "/tools"
sys.path.append(src_path)
sys.path.append(tools_path)
#Cumtomize Moduls
import service_pb2
import settings
import masterSpider
from tools.Log import *
#----------------> log settings
log_dir = _WORK_DIR_ + "/logs"
log_file = log_dir + "/" + _FILE_NAME_[:_FILE_NAME_.index(_FILE_NAME_.split(".")[-1]) - 1] + ".log"
logger = Log(log_dir , log_file)
#----------------> multiprocessing.Manager() settings
_MGR_ = multiprocessing.Manager()
_SLAVES_ = _MGR_.dict() #key: slave ip val: last connection time
_TASKS_ = _MGR_.dict() #key: slave ip val: json of task
#----------------> masterSpider object
_MSP_ = masterSpider.MasterSpider(_WORK_DIR_)
#----------------> other settings
_ONE_DAY_IN_SECONDS_ = 60 * 60 * 24
#_LOCAL_IP_ = socket.gethostbyname(socket.gethostname())
_MASTER_IP_ = settings.master_ip
def mkdir(self, path, mode):
Log('mkdir', path, level=1)
return os.mkdir(self._full_path(path), mode)
from tools.MD5 import *
from tools.Log import *
from tools.SeleniumSpider import *
from tools.BS4Spider import *
from tools.RequestsSpider import *
from tools.Proxy import *
from tools.ListTool import *
from tools.MultiProcess import *
from tools.StringTool import *
from tools.Network import *
script_path = os.path.split(os.path.split(os.path.realpath(__file__))[0])[0]
#work_dir = "./"
log_dir = work_dir + settings.folder_path["log"]
log_file = log_dir + "/analyzeHTML.log"
logger = Log(log_dir , log_file)
logger.init()
#----------------------------------------------------------------------------------
#-----------------------------------Ready------------------------------------------
#----------------------------------------------------------------------------------
class AnalyzeHTML() :
def __init__(self , conf_path="./spider.conf" , in_configs=None) :
self.configs = in_configs
self.redis_pool = None
self.redisdb = None
def readConfig(self) :
self.config = {}
try :
def open(self, path, flags):
Log('open', path, flags, level=1)
full_path = self._full_path(path)
return os.open(full_path, flags)
