def isValidHist(rootHist, histname, filename): if not rootHist: Log.logErrorMessage( 'Histogram \n"%s" \ncould not be found in rootFile:\n%s' % (histname, filename)) return False return True
def truncate(self, path, length, fh=None): Log('truncate', path, length, level=1) full_path = self._full_path(path) with open( full_path, 'r+' ) as f: # [r+] = [read & write] , [w+] = [read & write] , if file not exist,r+ calls error while w+ not f.truncate(length)
def readlink(self, path): Log('readlink', path, level=1) pathname = os.readlink(self._full_path(path)) if pathname.startswith("/"): # Path name is absolute, sanitize it. return os.path.relpath(pathname, self.root) else: return pathname
def __init__(self): # option from command line self.verbose = False self.log = Log() # relevant directories self.home = os.path.expanduser('~') self.working = os.getcwd() self.main = None try: exeFile = os.path.abspath(sys.modules['__main__'].__file__) srcFile = os.path.realpath(exeFile) # resolve symlinks self.main = os.path.dirname(srcFile) except: self.log.warn('no main file path, interactive interpreter') self.find = _Find(self) # operating system type self.os = _which_os()
def statfs(self, path): Log('statfs', path, level=1) full_path = self._full_path(path) stv = os.statvfs(full_path) return dict((key, getattr(stv, key)) for key in ('f_bavail', 'f_bfree', 'f_blocks', 'f_bsize', 'f_favail', 'f_ffree', 'f_files', 'f_flag', 'f_frsize', 'f_namemax'))
def getattr(self, path, fh=None): Log('getattr', path, fh, level=1) full_path = self._full_path(path) st = os.lstat( full_path) # os.stat follow the symbol link while os.lstat not ans = dict((key, getattr(st, key)) for key in ('st_atime', 'st_ctime', 'st_gid', 'st_mode', 'st_mtime', 'st_nlink', 'st_size', 'st_uid')) return ans
def readdir(self, path, fh): Log('readdir', path, level=1) full_path = self._full_path(path) dirents = ['.', '..'] if os.path.isdir(full_path): dirents.extend(os.listdir(full_path)) for r in dirents: yield r
def unlink(self, path): Log('unlink', path, level=1) return os.unlink(self._full_path(path))
def create(self, path, mode, fi=None): Log('create', path, mode, level=1) full_path = self._full_path(path) return os.open(full_path, os.O_WRONLY | os.O_CREAT, mode)
def flush(self, path, fh): Log('flush', path, level=1) return os.fsync(fh)
class Shell(): def __init__(self): # option from command line self.verbose = False self.log = Log() # relevant directories self.home = os.path.expanduser('~') self.working = os.getcwd() self.main = None try: exeFile = os.path.abspath(sys.modules['__main__'].__file__) srcFile = os.path.realpath(exeFile) # resolve symlinks self.main = os.path.dirname(srcFile) except: self.log.warn('no main file path, interactive interpreter') self.find = _Find(self) # operating system type self.os = _which_os() def respond(self, cmd_str_or_lst, shell=False, strip=False): cmd = ' '.join(cmd_str_or_lst) if (type(cmd_str_or_lst) == type( [])) else cmd_str_or_lst if self.verbose: print(cmd) byte_output = check_output(cmd, shell=shell) string_output = byte_output.decode('utf-8') if strip: return string_output.strip() return string_output def basename(self, path): basename = os.path.basename(path) if self.verbose: print('Basename of ', path, ' is ', basename) return basename def cd(self, path): if self.verbose: print('Changing directory to: ', path) os.chdir(path) def chrome(self, url): if self.os == 'linux': self.command([ 'google-chrome-stable', '--disable-features=NetworkService', url, '&>/dev/null' ]) elif self.os == 'mac': self.command(['open -a', r'Google\ Chrome', url]) def command(self, cmd_list): if isinstance(cmd_list, str): cmd = cmd_list else: cmd = ' '.join(cmd_list) if self.verbose: print(cmd) status = os.system(cmd) return status def cp(self, from_file, to_file): if self.verbose: print('Copying from ', from_file, ' to ', to_file) if self.exists(to_file): self.rm(to_file) if os.path.isdir(from_file): shutil.copytree(from_file, to_file) else: shutil.copyfile(from_file, to_file) def rm(self, path): if self.verbose: print('Recursively removing: ', path) if os.path.isdir(path): shutil.rmtree(path) else: os.remove(path) def dirname(self, path): dirname = os.path.dirname(path) if self.verbose: print('Dirname of ', path, ' is ', dirname) return dirname def exists(self, path): # check for path of directory if self.verbose: print('Checking the existance of path: ', path) return os.path.exists(path.strip()) def is_dir(self, path): # check for path of directory if self.verbose: print('Checking if is dir, path: ', path) return os.path.isdir(path.strip()) def kill(self, command_name): if self.os == 'mac': pid = self.pid(command_name) else: self.log.error( 'Shell.kill is not yet implemented for this operating system') status = self.command(['kill', '-9', pid]) if status == 0: self.log.validate('killed ' + command_name + ' running with id ' + pid) def link(self, src, dest): os.symlink(src, dest) def readlink(self, link_path): source = os.readlink(link_path) if self.exists(source): if self.verbose: self.log.validate('the symlink at, ' + link_path + ', resolved') return source if self.verbose: print('link, ' + link_path + ', exists, but did not resolve') return False def ls(self, path): if self.verbose: print('Listing files in directory: ', path) return os.listdir(path) def make_executable(self, file): self.command(['chmod +x', file]) def mkdir(self, path): if self.is_dir(path): self.log.validate('directory, ' + path + ', already exists') return os.mkdir(path) if self.verbose: print('made directory: ', path) def mv(self, from_path, to_path): try: os.rename(from_path, to_path) if self.verbose: print('renaming directory: ', from_path, ' to ', to_path) except Exception as e: raise e def pid(self, command_name): try: if self.os == 'mac': pid = self.respond(['pgrep', command_name], shell=True, strip=True) else: self.log.error( 'Shell.pid is not yet implemented for this operating system' ) except: self.log.error( 'failed to get a process id for the command name: ' + command_name) return pid def vim(self, name, SystemEditor=False): if SystemEditor: vim = os.environ.get('EDITOR', 'vim') # create editor else: vim = 'vim' if type(name) == type([]): if len(name) > 1: raise TypeError('too many input arguments') if len(name) == 0: self.command([vim]) return 0 else: name = name[0] if self.is_dir(name): self.command([vim, name]) return 0 try: open_file_with_vim(vim, name, 'r+') # open file to read except: open_file_with_vim(vim, name, 'w+') # open file to write return 0
import dns.resolver from dns.resolver import NoAnswer import logging import socket import smtplib from smtplib import SMTPServerDisconnected, SMTPSenderRefused from tools import email_verificator, domain_extract, MxRecord, Log log = Log() emailToVerify = "*****@*****.**" server_port = None server_name = None server_encryption = 'smtp' user_name = None password = None debug_level = 0 email_verify = email_verificator(emailToVerify) if email_verify is False: log.error("Email invalid" + emailToVerify) raise ValueError("Email invalid") domain = domain_extract(emailToVerify) if server_name is None else server_name try: records = dns.resolver.query(domain, 'MX') except NoAnswer as e: log.error(message=e)
if __name__ == "__main__": # init os.chdir(os.path.join(os.path.dirname(__file__), '..')) port = 8888 includes = None opts, argvs = getopt.getopt(sys.argv[1:], "c:p:h") for op, value in opts: if op == '-c': includes = value path._ETC_PATH = os.path.dirname(os.path.abspath(value)) elif op == '-p': port = int(value) elif op == '-h': print u'''使用参数启动: usage: [-p|-c] -p [prot] ******启动端口,默认端口:%d -c <file> ******加载配置文件 ''' % port sys.exit(0) if not includes: includes = os.path.join(path._ETC_PATH, 'includes_dev.json') print "no configuration found!,will use [%s] instead" % includes # main init_application(includes) application = get_application() application.listen(port) logger = Log().getLog() logger.info("starting..., listen [%d], configurated by (%s)", port, includes) tornado.ioloop.IOLoop.instance().start()
def chown(self, path, uid, gid): Log('chown', path, level=1) full_path = self._full_path(path) return os.chown(full_path, uid, gid)
def chmod(self, path, mode): Log('chmod', path, level=1) full_path = self._full_path(path) return os.chmod(full_path, mode)
def access(self, path, mode): Log('access', path, mode, level=1) full_path = self._full_path(path) if not os.access(full_path, mode): raise FuseOSError(errno.EACCES)
def fsync(self, path, fdatasync, fh): Log('fsync', path, level=1) return self.flush(path, fh)
def release(self, path, fh): Log('release', path, level=1) return os.close(fh)
def testIfFalidFile(rootFile, filename): if not rootFile: Log.logErrorMessage('Could not find rootFile: ' + filename)
def read(self, path, length, offset, fh): Log('read', path, length, offset, level=1) os.lseek(fh, offset, os.SEEK_SET) return os.read(fh, length)
print("Arguments ERROR.") print("opt:\t" + str(opt)) print("val:\t" + str(val)) sys.exit(1) src_path = work_dir + "/src" tools_path = src_path + "/tools" sys.path.append(src_path) sys.path.append(tools_path) #Cumtomize Moduls import settings from tools.Platform import * from tools.MD5 import * from tools.Log import * from tools.SeleniumSpider import * from tools.BS4Spider import * from tools.RequestsSpider import * from tools.Proxy import * from tools.ListTool import * from tools.MultiProcess import * from tools.StringTool import * from tools.Network import * script_path = os.path.split(os.path.split(os.path.realpath(__file__))[0])[0] logger = Log("./", "./log.log") logger.init() logger.info("info") logger.error("error") logger.warning("war")
max_epoch = configs['op']['max_epoch'] learning_rate = configs['op']['learning_rate'] decay_rate = configs['op']['decay_rate'] epoch_steps = configs['op']['epoch_steps'] snapshot = configs['op']['snapshot'] batch_size = configs['db']['batch_size'] loader_threads = configs['db']['loader_threads'] save_dir = configs['system']['save_dir'] # init Timer if not os.path.exists(save_dir): os.mkdir(save_dir) _t = Timer() # create log file log = Log(args.log_dir, args.cfg) log.wr_cfg(configs) # load data set training_set = load_data_set_from_factory(configs, 'train') print('Data set: {} has been loaded'.format(configs['db']['name'])) # load model trainer = load_training_model_from_factory(configs, ngpu=args.ngpu) if configs['system']['resume']: trainer.load_params(configs['system']['resume_path']) print('Model: {} has been loaded'.format(configs['model']['name'])) # start training epoch_size = len(training_set) // batch_size # init learning rate & iters start_iter = start_epoch * epoch_size
def isValidHist(rootHist, histname, filename): if not rootHist: Log.logErrorMessage('Histogram \n"%s" \ncould not be found in rootFile:\n%s' % (histname, filename)) return False return True
#coding:utf-8 import traceback from bson import * from pymongo import * from pymongo.bulk import BulkOperationBuilder import db_pool from tools import Log logger = Log().getLog() class MongoDataModel(object): """mongodb dao """ def __init__(self, dbname): client = db_pool.get_mongo(dbname) self.db = client.get_default_database() def find(self, table, SON=None, pn=0, rn=0, sort=None, count=False, projection=None): try: data = [] col = collection.Collection(self.db, table) cur = col.find(filter=SON, projection=projection, skip=pn * rn,
# init os.chdir(os.path.join(os.path.dirname(__file__), '..')) port = 8888 includes = None opts, argvs = getopt.getopt(sys.argv[1:], "c:p:h") for op, value in opts: if op == '-c': includes = value path._ETC_PATH = os.path.dirname(os.path.abspath(value)) elif op == '-p': port = int(value) elif op == '-h': print u'''使用参数启动: usage: [-p|-c] -p [prot] ******启动端口,默认端口:%d -c <file> ******加载配置文件 ''' % port sys.exit(0) if not includes: includes = os.path.join(path._ETC_PATH, 'includes_dev.json') print "no configuration found!,will use [%s] instead" % includes # main init_application(includes) application = get_application() application.listen(port) logger = Log().getLog() logger.info("starting..., listen [%d], configurated by (%s)", port, includes) tornado.ioloop.IOLoop.instance().start()
def mknod(self, path, mode, dev): Log('mknod', path, level=1) return os.mknod(self._full_path(path), mode, dev)
def write(self, path, buf, offset, fh): Log('write', path, buf, offset, level=1) os.lseek(fh, offset, os.SEEK_SET) return os.write(fh, buf)
def rmdir(self, path): Log('rmdir', path) full_path = self._full_path(path) return os.rmdir(full_path)
#script_path = os.path.split(os.path.split(os.path.realpath(__file__))[0])[0] #work_dir = script_path src_path = _WORK_DIR_ + "/src" tools_path = src_path + "/tools" sys.path.append(src_path) sys.path.append(tools_path) #Cumtomize Moduls import service_pb2 import settings import masterSpider from tools.Log import * #----------------> log settings log_dir = _WORK_DIR_ + "/logs" log_file = log_dir + "/" + _FILE_NAME_[:_FILE_NAME_.index(_FILE_NAME_.split(".")[-1]) - 1] + ".log" logger = Log(log_dir , log_file) #----------------> multiprocessing.Manager() settings _MGR_ = multiprocessing.Manager() _SLAVES_ = _MGR_.dict() #key: slave ip val: last connection time _TASKS_ = _MGR_.dict() #key: slave ip val: json of task #----------------> masterSpider object _MSP_ = masterSpider.MasterSpider(_WORK_DIR_) #----------------> other settings _ONE_DAY_IN_SECONDS_ = 60 * 60 * 24 #_LOCAL_IP_ = socket.gethostbyname(socket.gethostname()) _MASTER_IP_ = settings.master_ip
def mkdir(self, path, mode): Log('mkdir', path, level=1) return os.mkdir(self._full_path(path), mode)
from tools.MD5 import * from tools.Log import * from tools.SeleniumSpider import * from tools.BS4Spider import * from tools.RequestsSpider import * from tools.Proxy import * from tools.ListTool import * from tools.MultiProcess import * from tools.StringTool import * from tools.Network import * script_path = os.path.split(os.path.split(os.path.realpath(__file__))[0])[0] #work_dir = "./" log_dir = work_dir + settings.folder_path["log"] log_file = log_dir + "/analyzeHTML.log" logger = Log(log_dir , log_file) logger.init() #---------------------------------------------------------------------------------- #-----------------------------------Ready------------------------------------------ #---------------------------------------------------------------------------------- class AnalyzeHTML() : def __init__(self , conf_path="./spider.conf" , in_configs=None) : self.configs = in_configs self.redis_pool = None self.redisdb = None def readConfig(self) : self.config = {} try :
def open(self, path, flags): Log('open', path, flags, level=1) full_path = self._full_path(path) return os.open(full_path, flags)