def truncate(self, path, length, fh=None):
Log('truncate', path, length, level=1)
full_path = self._full_path(path)
with open(
full_path, 'r+'
) as f: # [r+] = [read & write] , [w+] = [read & write] , if file not exist,r+ calls error while w+ not
f.truncate(length)
def statfs(self, path):
Log('statfs', path, level=1)
full_path = self._full_path(path)
stv = os.statvfs(full_path)
return dict((key, getattr(stv, key))
for key in ('f_bavail', 'f_bfree', 'f_blocks', 'f_bsize',
'f_favail', 'f_ffree', 'f_files', 'f_flag',
'f_frsize', 'f_namemax'))
def readlink(self, path):
Log('readlink', path, level=1)
pathname = os.readlink(self._full_path(path))
if pathname.startswith("/"):
# Path name is absolute, sanitize it.
return os.path.relpath(pathname, self.root)
else:
return pathname
def readdir(self, path, fh):
Log('readdir', path, level=1)
full_path = self._full_path(path)
dirents = ['.', '..']
if os.path.isdir(full_path):
dirents.extend(os.listdir(full_path))
for r in dirents:
yield r
def getattr(self, path, fh=None):
Log('getattr', path, fh, level=1)
full_path = self._full_path(path)
st = os.lstat(
full_path) # os.stat follow the symbol link while os.lstat not
ans = dict((key, getattr(st, key))
for key in ('st_atime', 'st_ctime', 'st_gid', 'st_mode',
'st_mtime', 'st_nlink', 'st_size', 'st_uid'))
return ans
def __init__(self):
# option from command line
self.verbose = False
self.log = Log()
# relevant directories
self.home = os.path.expanduser('~')
self.working = os.getcwd()
self.main = None
try:
exeFile = os.path.abspath(sys.modules['__main__'].__file__)
srcFile = os.path.realpath(exeFile) # resolve symlinks
self.main = os.path.dirname(srcFile)
except:
self.log.warn('no main file path, interactive interpreter')
self.find = _Find(self)
# operating system type
self.os = _which_os()
def symlink(self, name, target):
Log('symlink', name, target, level=1)
return os.symlink(name, self._full_path(target))
#coding:utf-8
import traceback
from bson import *
from pymongo import *
from pymongo.bulk import BulkOperationBuilder
import db_pool
from tools import Log
logger = Log().getLog()
class MongoDataModel(object):
"""mongodb dao
"""
def __init__(self, dbname):
client = db_pool.get_mongo(dbname)
self.db = client.get_default_database()
def find(self,
table,
SON=None,
pn=0,
rn=0,
sort=None,
count=False,
projection=None):
try:
data = []
col = collection.Collection(self.db, table)
cur = col.find(filter=SON,
projection=projection,
skip=pn * rn,
print("Arguments ERROR.")
print("opt:\t" + str(opt))
print("val:\t" + str(val))
sys.exit(1)
src_path = work_dir + "/src"
tools_path = src_path + "/tools"
sys.path.append(src_path)
sys.path.append(tools_path)
#Cumtomize Moduls
import settings
from tools.Platform import *
from tools.MD5 import *
from tools.Log import *
from tools.SeleniumSpider import *
from tools.BS4Spider import *
from tools.RequestsSpider import *
from tools.Proxy import *
from tools.ListTool import *
from tools.MultiProcess import *
from tools.StringTool import *
from tools.Network import *
script_path = os.path.split(os.path.split(os.path.realpath(__file__))[0])[0]
logger = Log("./", "./log.log")
logger.init()
logger.info("info")
logger.error("error")
logger.warning("war")
def rename(self, old, new):
Log('rename', old, new, level=1)
return os.rename(self._full_path(old), self._full_path(new))
max_epoch = configs['op']['max_epoch']
learning_rate = configs['op']['learning_rate']
decay_rate = configs['op']['decay_rate']
epoch_steps = configs['op']['epoch_steps']
snapshot = configs['op']['snapshot']
batch_size = configs['db']['batch_size']
loader_threads = configs['db']['loader_threads']
save_dir = configs['system']['save_dir']
# init Timer
if not os.path.exists(save_dir):
os.mkdir(save_dir)
_t = Timer()
# create log file
log = Log(args.log_dir, args.cfg)
log.wr_cfg(configs)
# load data set
training_set = load_data_set_from_factory(configs, 'train')
print('Data set: {} has been loaded'.format(configs['db']['name']))
# load model
trainer = load_training_model_from_factory(configs, ngpu=args.ngpu)
if configs['system']['resume']:
trainer.load_params(configs['system']['resume_path'])
print('Model: {} has been loaded'.format(configs['model']['name']))
# start training
epoch_size = len(training_set) // batch_size # init learning rate & iters
start_iter = start_epoch * epoch_size
def link(self, target, name):
Log('link', target, name, level=1)
return os.link(self._full_path(target), self._full_path(name))
def rmdir(self, path):
Log('rmdir', path)
full_path = self._full_path(path)
return os.rmdir(full_path)
def write(self, path, buf, offset, fh):
Log('write', path, buf, offset, level=1)
os.lseek(fh, offset, os.SEEK_SET)
return os.write(fh, buf)
def read(self, path, length, offset, fh):
Log('read', path, length, offset, level=1)
os.lseek(fh, offset, os.SEEK_SET)
return os.read(fh, length)
def create(self, path, mode, fi=None):
Log('create', path, mode, level=1)
full_path = self._full_path(path)
return os.open(full_path, os.O_WRONLY | os.O_CREAT, mode)
def open(self, path, flags):
Log('open', path, flags, level=1)
full_path = self._full_path(path)
return os.open(full_path, flags)
def utimens(self, path, times=None):
Log('utimes', path, level=1)
return os.utime(self._full_path(path), times)
def mknod(self, path, mode, dev):
Log('mknod', path, level=1)
return os.mknod(self._full_path(path), mode, dev)
def flush(self, path, fh):
Log('flush', path, level=1)
return os.fsync(fh)
def mkdir(self, path, mode):
Log('mkdir', path, level=1)
return os.mkdir(self._full_path(path), mode)
def release(self, path, fh):
Log('release', path, level=1)
return os.close(fh)
def unlink(self, path):
Log('unlink', path, level=1)
return os.unlink(self._full_path(path))
def fsync(self, path, fdatasync, fh):
Log('fsync', path, level=1)
return self.flush(path, fh)
import dns.resolver
from dns.resolver import NoAnswer
import logging
import socket
import smtplib
from smtplib import SMTPServerDisconnected, SMTPSenderRefused
from tools import email_verificator, domain_extract, MxRecord, Log
log = Log()
emailToVerify = "*****@*****.**"
server_port = None
server_name = None
server_encryption = 'smtp'
user_name = None
password = None
debug_level = 0
email_verify = email_verificator(emailToVerify)
if email_verify is False:
log.error("Email invalid" + emailToVerify)
raise ValueError("Email invalid")
domain = domain_extract(emailToVerify) if server_name is None else server_name
try:
records = dns.resolver.query(domain, 'MX')
except NoAnswer as e:
log.error(message=e)
def access(self, path, mode):
Log('access', path, mode, level=1)
full_path = self._full_path(path)
if not os.access(full_path, mode):
raise FuseOSError(errno.EACCES)
def chmod(self, path, mode):
Log('chmod', path, level=1)
full_path = self._full_path(path)
return os.chmod(full_path, mode)
def chown(self, path, uid, gid):
Log('chown', path, level=1)
full_path = self._full_path(path)
return os.chown(full_path, uid, gid)
from tools.MD5 import *
from tools.Log import *
from tools.SeleniumSpider import *
from tools.BS4Spider import *
from tools.RequestsSpider import *
from tools.Proxy import *
from tools.ListTool import *
from tools.MultiProcess import *
from tools.StringTool import *
from tools.Network import *
script_path = os.path.split(os.path.split(os.path.realpath(__file__))[0])[0]
#work_dir = "./"
log_dir = work_dir + settings.folder_path["log"]
log_file = log_dir + "/analyzeHTML.log"
logger = Log(log_dir , log_file)
logger.init()
#----------------------------------------------------------------------------------
#-----------------------------------Ready------------------------------------------
#----------------------------------------------------------------------------------
class AnalyzeHTML() :
def __init__(self , conf_path="./spider.conf" , in_configs=None) :
self.configs = in_configs
self.redis_pool = None
self.redisdb = None
def readConfig(self) :
self.config = {}
try :
#!flask/bin/python
from flask import Flask, request, abort, jsonify
from tools import Log, log_entry
import httplib2
import random
app = Flask(__name__)
log = Log().get_log()
def send_get_request(url):
http = httplib2.Http()
response, content = http.request(url, 'GET')
return content
@app.route('/info_planet', methods=['GET'])
@log_entry()
def info_planet():
''' curl -i -X GET http://localhost:5000/info_planet '''
planet = random.randrange(10)
response = send_get_request('https://swapi.co/api/planets/%s/' % planet)
return response
if __name__ == '__main__':
app.run(debug=True)
