def config_common(app, root_path):
app.json_encoder = json_encoder.CustomJSONEncoder
app.secret_key = 'blabla'
app.config['SECRET_KEY'] = APP_SECRET_KEY
app.config['APP_AUTH_HEADER_PREFIX'] = APP_AUTH_HEADER_PREFIX
app.config['WTF_CSRF_ENABLED'] = False
app.config['GMISSION_IMAGE_UPLOAD_DIR'] = os.path.join(
root_path, 'static', 'image', 'original')
app.config['GMISSION_IMAGE_THUMB_DIR'] = os.path.join(
root_path, 'static', 'image', 'thumb')
app.config['GMISSION_VIDEO_UPLOAD_DIR'] = os.path.join(
root_path, 'static', 'video', 'original')
app.config['GMISSION_VIDEO_THUMB_DIR'] = os.path.join(
root_path, 'static', 'video', 'thumb')
app.config['GMISSION_AUDIO_UPLOAD_DIR'] = os.path.join(
root_path, 'static', 'audio', 'original')
app.config['GMISSION_LOGS_DIR'] = os.path.join(root_path, '..', '..',
'logs')
# app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql://*****:*****@127.0.0.1:3306/gmission_hkust'
app.config[
'SQLALCHEMY_DATABASE_URI'] = 'mysql://*****:*****@docker-mysql/gmission_hkust'
log.set_logger(app)
app.config.from_object('email')
def main():
version = '1.0'
log.set_logger(filename='logs/mini_spider.log', level='DEBUG:INFO')
parser = argparse.ArgumentParser(description='a mini spider')
parser.add_argument("-c", "--conf", help="config file", required=True)
parser.add_argument('-v',
'--version',
action='version',
version='%(prog)s ' + version)
args = parser.parse_args()
# an hash(called dict in python) in config file spider block
spider_config = config_load.loadconfig(args.conf, 'spider')
# Global Configs
g_url_list_file = spider_config.get('url_list_file')
g_output_directory = spider_config.get('output_directory')
g_max_depth = int(spider_config.get('max_depth'))
g_crawl_interval = int(spider_config.get('crawl_interval'))
g_crawl_timeout = int(spider_config.get('crawl_timeout'))
g_target_url = spider_config.get('target_url')
g_thread_count = int(spider_config.get('thread_count'))
log.info('init complete, start running the spider with configs: ' +
str(spider_config))
seedfile_load.loadSeedFile(g_url_list_file)
crawl_thread.crawl(g_thread_count, g_output_directory, g_max_depth,
g_target_url, g_crawl_interval, g_crawl_timeout)
def __init__(self): self.collect_cpu_info = 0 self.collect_process_info = 0 self.collect_mem_info = 0 self.collect_disk_info = 0 self.collect_network_info = 0 self.collect_loadavg = 1 self.collect_interval = 3 log.set_logger(limit = 1024) log.set_logger(level = 'DEBUG')
def process(dataFilePath,logfilename = 'sqliteDataProcess.log'):
import log
DBlog = log.set_logger(filename = logfilename, isOnlyFile = False)
DBlog.debug('sqlite start！')
global sqlite3Obj
sqlite3Obj = DBHelper.Sqlite3Helper(dataFilePath)
sqlite3Obj.open(check_same_thread=False)
while True:
role,index = yield
if isinstance(role, str):
return
try:
# 跳过账号登陆不成功的
if role.ERRORList[0] != 0:
# 首次录入
firstEntering(role, sqlite3Obj)
# 查询本学期成绩 并更新数据库
# currentEntering(role, sqlite3Obj)
# 更新学生个人信息
# updateStudentInfo(role, sqlite3Obj)
DBlog.info('%d : %s firstEntering Success!' % (index, role.userId))
except Exception as e:
DBlog.error('%d : %s firstEntering Exception!\n -- %s' % (index,role.userId, traceback.format_exc()))
else:
pass
finally:
pass
def main():
set_logger()
cfgs = dict()
entry_points = [
(e.name, e.load()) for e in pkg_resources.iter_entry_points(
'es_setup.cfg')
]
for (_, fn) in entry_points:
fn(cfgs)
cfgs = dict(sorted(cfgs.iteritems(), key=lambda cfgs: cfgs[0])) # sort
# OK, we enter our core logic
LOG.info('Stage: Initializing\n')
# first, ask user some question
rebuild = 'no'
if os.path.exists(user_conf_file):
txt = "You have built eayunstack, do you want to reuse the same " \
"configuration (yes, no) [no]: "
rebuild = utils.ask_user(txt, ('yes, no'), 'no')
if rebuild.lower() == 'yes':
with file(user_conf_file, 'r') as f:
s = f.read().strip('\n')
user_conf.update((eval(s)))
if rebuild.lower() == 'no':
for c in cfgs:
cfgs[c].ask_user(user_conf)
# save for next using
with file(user_conf_file, 'w') as f:
f.write(str(user_conf))
# then, we output the result user set just
LOG.info('Stage: Setup validation\n')
utils.fmt_print('--== CONFIGURATION PREVIEW ==--')
for c in cfgs:
cfgs[c].validation(user_conf)
txt = 'Please confirm installation settings (OK, Cancel) [OK]: '
confirm = utils.ask_user(txt, ('ok, cancel'), 'ok')
if confirm.lower() == 'cancel':
sys.exit()
# last, run every configuration module to setup
LOG.info('Stage: Transaction setup')
for c in cfgs:
cfgs[c].run(user_conf)
def SpiderStart(q,
logfilename='URPSpider.log',
dataSource=None,
sprstatus=None):
import log as Spiderlog
Spiderlog.set_logger(filename=logfilename)
# loop = uvloop.new_event_loop()
# asyncio.set_event_loop(loop)
loop = asyncio.get_event_loop()
urp = URPSpider(loop, queue=q, spiderlog=Spiderlog)
urp.run(dataSource=dataSource)
urp.close()
if sprstatus is not None:
sprstatus['Spider'] = sprstatus['Spider'] - 1
Spiderlog.debug('任务结束! -- 当前剩余任务进程数:%s' % sprstatus['Spider'])
if sprstatus['Spider'] == 0:
# 当爬虫运行数目为0的时候增加数据储存结束标示
q.put('end')
def main():
fname = config.log_path + 'article_parse.' + time.strftime("%Y%m%d")
log.set_logger(level='DEBUG', when="D", limit=1, filename=fname)
alist = Mongo().scan()
if not alist:
log.warn("no articles in mongodb")
return False
MyObj = Mysql()
mobj = Mongo()
for doc in alist:
if Parse(MyObj).do(doc):
mobj.update(doc.get("_id"), done=1)
log.info("insert mysql success, url:%s" % doc.get('url'))
else:
mobj.update(doc.get("_id"), done=-1)
log.warning("insert mysql failure, task_id:%s, url:%s" %
(doc.get('taskid'), doc.get('url')))
def config_common(app, root_path):
app.json_encoder = json_encoder.CustomJSONEncoder
app.secret_key = 'blabla'
app.config['SECRET_KEY'] = APP_SECRET_KEY
app.config['APP_AUTH_HEADER_PREFIX'] = APP_AUTH_HEADER_PREFIX
app.config['WTF_CSRF_ENABLED'] = False
app.config['SECURITY_TOKEN_AUTHENTICATION_KEY'] = 'UserAuthToken'
app.config[
'SECURITY_TOKEN_AUTHENTICATION_HEADER'] = 'X-Xuewen-User-Auth-Token'
app.config['GMISSION_IMAGE_UPLOAD_DIR'] = os.path.join(
root_path, 'static', 'image', 'original')
app.config['GMISSION_IMAGE_THUMB_DIR'] = os.path.join(
root_path, 'static', 'image', 'thumb')
app.config['GMISSION_VIDEO_UPLOAD_DIR'] = os.path.join(
root_path, 'static', 'video', 'original')
app.config['GMISSION_VIDEO_THUMB_DIR'] = os.path.join(
root_path, 'static', 'video', 'thumb')
app.config['GMISSION_AUDIO_UPLOAD_DIR'] = os.path.join(
root_path, 'static', 'audio', 'original')
app.config['GMISSION_LOGS_DIR'] = os.path.join(root_path, 'logs')
# app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql://*****:*****@127.0.0.1:3306/gmission_hkust'
app.config[
'SQLALCHEMY_DATABASE_URI'] = 'mysql://*****:*****@docker-mysql/gmission_hkust'
FP_PATH = os.path.join(root_path, 'static', 'fp_collection')
app.config['APK_PATH'] = os.path.join(FP_PATH, 'app-debug-unaligned.apk')
app.config['DATA_PATH'] = os.path.join(FP_PATH, 'wherami.zip')
app.config['WIFIPAD_PATH'] = os.path.join(FP_PATH, 'wififorpad.apk')
app.config['LOCALIZATION_PATH'] = os.path.join(FP_PATH,
'wifilocalization.apk')
log.set_logger(app)
app.config.from_object('email')
def main():
version = '1.0'
log.set_logger(filename='logs/mini_spider.log', level='DEBUG:INFO')
parser = argparse.ArgumentParser(description='a mini spider')
parser.add_argument("-c", "--conf", help="config file", required=True)
parser.add_argument('-v', '--version', action='version', version='%(prog)s ' + version)
args = parser.parse_args()
# an hash(called dict in python) in config file spider block
spider_config = config_load.loadconfig(args.conf, 'spider')
# Global Configs
g_url_list_file = spider_config.get('url_list_file')
g_output_directory = spider_config.get('output_directory')
g_max_depth = int(spider_config.get('max_depth'))
g_crawl_interval = int(spider_config.get('crawl_interval'))
g_crawl_timeout = int(spider_config.get('crawl_timeout'))
g_target_url = spider_config.get('target_url')
g_thread_count = int(spider_config.get('thread_count'))
log.info('init complete, start running the spider with configs: ' + str(spider_config))
seedfile_load.loadSeedFile(g_url_list_file)
crawl_thread.crawl(g_thread_count, g_output_directory, g_max_depth, g_target_url, g_crawl_interval, g_crawl_timeout)
def config_common(app, root_path):
app.json_encoder = json_encoder.CustomJSONEncoder
app.secret_key = 'blabla'
app.config['SECRET_KEY'] = APP_SECRET_KEY
app.config['APP_AUTH_HEADER_PREFIX'] = APP_AUTH_HEADER_PREFIX
app.config['WTF_CSRF_ENABLED'] = False
app.config['GMISSION_IMAGE_UPLOAD_DIR'] = os.path.join(root_path, 'static', 'image', 'original')
app.config['GMISSION_IMAGE_THUMB_DIR'] = os.path.join(root_path, 'static', 'image', 'thumb')
app.config['GMISSION_VIDEO_UPLOAD_DIR'] = os.path.join(root_path, 'static', 'video', 'original')
app.config['GMISSION_VIDEO_THUMB_DIR'] = os.path.join(root_path, 'static', 'video', 'thumb')
app.config['GMISSION_AUDIO_UPLOAD_DIR'] = os.path.join(root_path, 'static', 'audio', 'original')
app.config['GMISSION_LOGS_DIR'] = os.path.join(root_path, '..', '..', 'logs')
# app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql://*****:*****@127.0.0.1:3306/gmission_hkust'
app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql://*****:*****@docker-mysql/gmission_hkust'
log.set_logger(app)
app.config.from_object('email')
def __init__(self, loop, q, _log):
self.loop = loop
self.q = q
self.logPip = _log
# limit_per_host=30 limit = 100`
# conn = aiohttp.TCPConnector (verify_ssl = False,limit_per_host = 2,use_dns_cache = True,loop = self.loop) connector = conn
self.session = aiohttp.ClientSession(loop=self.loop)
self.NETlog = log.set_logger(filename="netinterface.log",
isOnlyFile=False)
# NET最大并行数
self.netLimit = 10
# NET队列状态
self.netStatus = 0
# NET正在处理队列状态
self.netProcessNum = 0
def main(q, pindex=1):
logPip = log.set_logger(filename="URPPipelines.log", isOnlyFile=False)
loop = asyncio.new_event_loop()
process = MyPipeline(loop, q, logPip)
cDB = process.process_localDB()
cDB.send(None)
t = Thread(target=start_loop, args=(loop, ))
t.setDaemon(True)
t.start()
index = 0
while True:
try:
role = q.get()
# 写sqlite
cDB.send((role, index))
# 写微信端
asyncio.run_coroutine_threadsafe(process.process_NET(role), loop)
except StopIteration:
cDB.close()
logPip.info('数据存储结束! -- %s' % process.netStatus)
break
except Exception as e:
logPip.error(traceback.format_exc())
else:
pass
finally:
index = index + 1
while True:
# NET 上传结束标志
if process.netStatus == 0:
process.session.close()
break
logPip.error('等待微信端上传！-- %s' % process.netStatus)
time.sleep(30)
loop.stop()
logPip.error('所有任务结束！')
process.session.close()
# Internationalization # https://docs.djangoproject.com/en/1.8/topics/i18n/ LANGUAGE_CODE = 'en-us' TIME_ZONE = 'Asia/Shanghai' USE_I18N = True USE_L10N = True USE_TZ = True # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.8/howto/static-files/ STATIC_URL = '/static/' MEDIA_ROOT = '/data1/www/sport/media' MEDIA_URL = '/media/' #initialize the log # Use time-rotated file handler, each day has a different log file, see # logging.handlers.TimedRotatingFileHandler for more help about 'when' log.set_logger(when='D', limit=10240) # File log level set to INFO, and stdout log level set to DEBUG log.set_logger(level='DEBUG:WARN') # Change default log formatter #log.set_logger(filename = '/data1/www/logs/sport.log', fmt = '%(asctime)s - %(name)s - %(funcName)s -%(thread)d - %(levelname)s - %(message)s')
cDB.send((role, index))
# 写微信端
asyncio.run_coroutine_threadsafe(process.process_NET(role), loop)
except StopIteration:
cDB.close()
logPip.info('数据存储结束! -- %s' % process.netStatus)
break
except Exception as e:
logPip.error(traceback.format_exc())
else:
pass
finally:
index = index + 1
while True:
# NET 上传结束标志
if process.netStatus == 0:
process.session.close()
break
logPip.error('等待微信端上传！-- %s' % process.netStatus)
time.sleep(30)
loop.stop()
logPip.error('所有任务结束！')
process.session.close()
if __name__ == '__main__':
pass
logPip = log.set_logger(filename="URPPipelines.log", isOnlyFile=False)
loop = asyncio.new_event_loop()
process = MyPipeline(loop, None, logPip)
#!/bin/python
import log
log.set_logger(filename = '/tmp/yyy.log', mode = 'w')
log.debug("Houston, we have a %s", "thorny problem", exc_info=1)
log.error('this is error log for otomat')
# -*- coding: utf-8 -*-
from multiprocessing.dummy import Pool as ThreadPool
from multiprocessing import Queue
import log
import time
log.set_logger(level = 'INFO')
class NoticeObj(object):
'''
通知抽象类
'''
def discorver_new_file(self, new_file):
pass
class UploadToRemote(NoticeObj):
def discorver_new_file(self,new_file):
'''
上传新文件到远端
'''
time.sleep(3)
log.info(new_file)
pass
class CopyToLocal(NoticeObj):
def discorver_new_file(self,new_file):
'''
本地持久化新文件
from PySide import QtCore, QtGui
#from PyQt4 import QtOpenGL
from model import Models
from model import NetworkModel as NWM
from starmap import MapGlyphs
from starmap import MapScene
from projects import ProjectManager
#from forms import WorldBrowser
from forms import AllegianceEditor
from forms import NetworkManager as nm
#from forms import Trade
#from forms import Trade
from resources import starmap_rc
import log
log.set_logger()
InsertTextButton = 10
class OccurrenceSpinBox(QtGui.QSpinBox):
def __init__(self):
QtGui.QSpinBox.__init__(self)
self.setMinimum(-2)
self.setMaximum(2)
self.setValue(0)
self.lineEdit().setReadOnly(True)
def textFromValue(self, value):
if value >= 0:
text = '+' + str(value)
import logging
import pexpect
import subprocess
from pexpect import pxssh
from log import logger, set_logger
from remote_commands import cmd_list
import arguments
args = arguments.args
set_logger(logfile=args.logfile, loglevel=int(args.loglevel))
# Setting command type to interactive if given command is found in cmd_list
interactive = False
icmd = None
for i in cmd_list:
if args.command in i[0]:
interactive = True
icmd = i
break
class RemoteCommand(object):
def __init__(self, command, host, username, password, timeout):
"""
Initialize argument values
"""
self.command = command
self.host = host
self.username = username
self.password = password
def main():
log.set_logger(level='DEBUG')
log.debug("hello world");
log.info("info world");
log.error("error world");
pass
#!/usr/bin/python
# -*- coding: utf-8 -*-
import time
import os
import sys
import net
import collect
import config
import log
log.set_logger(limit = 1024)
log.set_logger(level = 'DEBUG')
''' 数据采集任务守护进程 '''
class CollectTask:
def upload(self):
log.info("数据采集任务守护进程启动")
obj = config.Config()
obj.load_config()
log.info("加载配置完成")
collector = collect.Collect()
data_sender = net.DataSender()
log.info("开始连接")
data_sender.connect("tcp://127.0.0.1:8899")
help='Chose an adaptor: FLASK(Default) or DJANGO.',
type=str)
parser.add_argument(
'-db',
'--create-database',
help='Creates a sql file that can be used to create a database.',
action='store_true')
parser.add_argument('-dl',
'--draw-lattice',
nargs='+',
help='Draws a lattice.')
args = parser.parse_args()
if __name__ == '__main__':
log.set_logger(args.log_level, show_path=False)
path = os.path.normpath(args.filepath)
directory = None
if args.project_root:
directory = os.path.normpath(args.project_root)
else:
directory = os.path.dirname(path)
project_modules = get_python_modules(directory)
local_modules = get_directory_modules(directory)
tree = generate_ast(path)
cfg = build_cfg(tree, project_modules, local_modules, path)
cfg_list = [cfg]
import log
import optparse
from crawler.crawler import Crawler
if __name__ == '__main__':
logger = log.set_logger("main")
parser = optparse.OptionParser()
parser.add_option("-c",
"--city",
dest="city",
default="warszawa",
help="City to crawl [warsaw]")
parser.add_option(
"-t",
"--type",
dest="property_type",
default="apartment",
help="Type of property to crawl (house/apartment) [apartment]")
parser.add_option("-o",
"--offer-type",
dest="offer_type",
default="rent",
help="Type of offer type to search (rent/sell) [rent]")
parser.add_option("-s",
"--start-page",
dest="start_page",
type="int",
# -*- coding: utf-8 -*-
from multiprocessing.dummy import Pool as ThreadPool
from multiprocessing import Queue
import log
import time
log.set_logger(level='INFO')
class NoticeObj(object):
'''
通知抽象类
'''
def discorver_new_file(self, new_file):
pass
class UploadToRemote(NoticeObj):
def discorver_new_file(self, new_file):
'''
上传新文件到远端
'''
time.sleep(3)
log.info(new_file)
pass
class CopyToLocal(NoticeObj):
def discorver_new_file(self, new_file):
'''
# -*- coding: utf-8 -*-
import tornado.web
import log
import json
import os
SELF_TEMPLATES = 1
GLOBAL_TEMPLATES = 0
log.set_logger(level = 'DEBUG:INFO')
class RealBaseHandler(tornado.web.RequestHandler):
@property
def user_name(self):
if self.current_user:
return tornado.escape.xhtml_escape(self.current_user)
return ""
def get_current_user(self):
return self.get_secure_cookie("user")
@tornado.gen.coroutine
def get_api_result(self,uri, method="GET", args=None):
'''
异步请求后端接口
'''
if args:
req = json.dumps(args)
else:
req = {}
