def config_common(app, root_path): app.json_encoder = json_encoder.CustomJSONEncoder app.secret_key = 'blabla' app.config['SECRET_KEY'] = APP_SECRET_KEY app.config['APP_AUTH_HEADER_PREFIX'] = APP_AUTH_HEADER_PREFIX app.config['WTF_CSRF_ENABLED'] = False app.config['GMISSION_IMAGE_UPLOAD_DIR'] = os.path.join( root_path, 'static', 'image', 'original') app.config['GMISSION_IMAGE_THUMB_DIR'] = os.path.join( root_path, 'static', 'image', 'thumb') app.config['GMISSION_VIDEO_UPLOAD_DIR'] = os.path.join( root_path, 'static', 'video', 'original') app.config['GMISSION_VIDEO_THUMB_DIR'] = os.path.join( root_path, 'static', 'video', 'thumb') app.config['GMISSION_AUDIO_UPLOAD_DIR'] = os.path.join( root_path, 'static', 'audio', 'original') app.config['GMISSION_LOGS_DIR'] = os.path.join(root_path, '..', '..', 'logs') # app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql://*****:*****@127.0.0.1:3306/gmission_hkust' app.config[ 'SQLALCHEMY_DATABASE_URI'] = 'mysql://*****:*****@docker-mysql/gmission_hkust' log.set_logger(app) app.config.from_object('email')
def main(): version = '1.0' log.set_logger(filename='logs/mini_spider.log', level='DEBUG:INFO') parser = argparse.ArgumentParser(description='a mini spider') parser.add_argument("-c", "--conf", help="config file", required=True) parser.add_argument('-v', '--version', action='version', version='%(prog)s ' + version) args = parser.parse_args() # an hash(called dict in python) in config file spider block spider_config = config_load.loadconfig(args.conf, 'spider') # Global Configs g_url_list_file = spider_config.get('url_list_file') g_output_directory = spider_config.get('output_directory') g_max_depth = int(spider_config.get('max_depth')) g_crawl_interval = int(spider_config.get('crawl_interval')) g_crawl_timeout = int(spider_config.get('crawl_timeout')) g_target_url = spider_config.get('target_url') g_thread_count = int(spider_config.get('thread_count')) log.info('init complete, start running the spider with configs: ' + str(spider_config)) seedfile_load.loadSeedFile(g_url_list_file) crawl_thread.crawl(g_thread_count, g_output_directory, g_max_depth, g_target_url, g_crawl_interval, g_crawl_timeout)
def __init__(self): self.collect_cpu_info = 0 self.collect_process_info = 0 self.collect_mem_info = 0 self.collect_disk_info = 0 self.collect_network_info = 0 self.collect_loadavg = 1 self.collect_interval = 3 log.set_logger(limit = 1024) log.set_logger(level = 'DEBUG')
def process(dataFilePath,logfilename = 'sqliteDataProcess.log'): import log DBlog = log.set_logger(filename = logfilename, isOnlyFile = False) DBlog.debug('sqlite start!') global sqlite3Obj sqlite3Obj = DBHelper.Sqlite3Helper(dataFilePath) sqlite3Obj.open(check_same_thread=False) while True: role,index = yield if isinstance(role, str): return try: # 跳过账号登陆不成功的 if role.ERRORList[0] != 0: # 首次录入 firstEntering(role, sqlite3Obj) # 查询本学期成绩 并更新数据库 # currentEntering(role, sqlite3Obj) # 更新学生个人信息 # updateStudentInfo(role, sqlite3Obj) DBlog.info('%d : %s firstEntering Success!' % (index, role.userId)) except Exception as e: DBlog.error('%d : %s firstEntering Exception!\n -- %s' % (index,role.userId, traceback.format_exc())) else: pass finally: pass
def main(): set_logger() cfgs = dict() entry_points = [ (e.name, e.load()) for e in pkg_resources.iter_entry_points( 'es_setup.cfg') ] for (_, fn) in entry_points: fn(cfgs) cfgs = dict(sorted(cfgs.iteritems(), key=lambda cfgs: cfgs[0])) # sort # OK, we enter our core logic LOG.info('Stage: Initializing\n') # first, ask user some question rebuild = 'no' if os.path.exists(user_conf_file): txt = "You have built eayunstack, do you want to reuse the same " \ "configuration (yes, no) [no]: " rebuild = utils.ask_user(txt, ('yes, no'), 'no') if rebuild.lower() == 'yes': with file(user_conf_file, 'r') as f: s = f.read().strip('\n') user_conf.update((eval(s))) if rebuild.lower() == 'no': for c in cfgs: cfgs[c].ask_user(user_conf) # save for next using with file(user_conf_file, 'w') as f: f.write(str(user_conf)) # then, we output the result user set just LOG.info('Stage: Setup validation\n') utils.fmt_print('--== CONFIGURATION PREVIEW ==--') for c in cfgs: cfgs[c].validation(user_conf) txt = 'Please confirm installation settings (OK, Cancel) [OK]: ' confirm = utils.ask_user(txt, ('ok, cancel'), 'ok') if confirm.lower() == 'cancel': sys.exit() # last, run every configuration module to setup LOG.info('Stage: Transaction setup') for c in cfgs: cfgs[c].run(user_conf)
def SpiderStart(q, logfilename='URPSpider.log', dataSource=None, sprstatus=None): import log as Spiderlog Spiderlog.set_logger(filename=logfilename) # loop = uvloop.new_event_loop() # asyncio.set_event_loop(loop) loop = asyncio.get_event_loop() urp = URPSpider(loop, queue=q, spiderlog=Spiderlog) urp.run(dataSource=dataSource) urp.close() if sprstatus is not None: sprstatus['Spider'] = sprstatus['Spider'] - 1 Spiderlog.debug('任务结束! -- 当前剩余任务进程数:%s' % sprstatus['Spider']) if sprstatus['Spider'] == 0: # 当爬虫运行数目为0的时候增加数据储存结束标示 q.put('end')
def main(): fname = config.log_path + 'article_parse.' + time.strftime("%Y%m%d") log.set_logger(level='DEBUG', when="D", limit=1, filename=fname) alist = Mongo().scan() if not alist: log.warn("no articles in mongodb") return False MyObj = Mysql() mobj = Mongo() for doc in alist: if Parse(MyObj).do(doc): mobj.update(doc.get("_id"), done=1) log.info("insert mysql success, url:%s" % doc.get('url')) else: mobj.update(doc.get("_id"), done=-1) log.warning("insert mysql failure, task_id:%s, url:%s" % (doc.get('taskid'), doc.get('url')))
def config_common(app, root_path): app.json_encoder = json_encoder.CustomJSONEncoder app.secret_key = 'blabla' app.config['SECRET_KEY'] = APP_SECRET_KEY app.config['APP_AUTH_HEADER_PREFIX'] = APP_AUTH_HEADER_PREFIX app.config['WTF_CSRF_ENABLED'] = False app.config['SECURITY_TOKEN_AUTHENTICATION_KEY'] = 'UserAuthToken' app.config[ 'SECURITY_TOKEN_AUTHENTICATION_HEADER'] = 'X-Xuewen-User-Auth-Token' app.config['GMISSION_IMAGE_UPLOAD_DIR'] = os.path.join( root_path, 'static', 'image', 'original') app.config['GMISSION_IMAGE_THUMB_DIR'] = os.path.join( root_path, 'static', 'image', 'thumb') app.config['GMISSION_VIDEO_UPLOAD_DIR'] = os.path.join( root_path, 'static', 'video', 'original') app.config['GMISSION_VIDEO_THUMB_DIR'] = os.path.join( root_path, 'static', 'video', 'thumb') app.config['GMISSION_AUDIO_UPLOAD_DIR'] = os.path.join( root_path, 'static', 'audio', 'original') app.config['GMISSION_LOGS_DIR'] = os.path.join(root_path, 'logs') # app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql://*****:*****@127.0.0.1:3306/gmission_hkust' app.config[ 'SQLALCHEMY_DATABASE_URI'] = 'mysql://*****:*****@docker-mysql/gmission_hkust' FP_PATH = os.path.join(root_path, 'static', 'fp_collection') app.config['APK_PATH'] = os.path.join(FP_PATH, 'app-debug-unaligned.apk') app.config['DATA_PATH'] = os.path.join(FP_PATH, 'wherami.zip') app.config['WIFIPAD_PATH'] = os.path.join(FP_PATH, 'wififorpad.apk') app.config['LOCALIZATION_PATH'] = os.path.join(FP_PATH, 'wifilocalization.apk') log.set_logger(app) app.config.from_object('email')
def config_common(app, root_path): app.json_encoder = json_encoder.CustomJSONEncoder app.secret_key = 'blabla' app.config['SECRET_KEY'] = APP_SECRET_KEY app.config['APP_AUTH_HEADER_PREFIX'] = APP_AUTH_HEADER_PREFIX app.config['WTF_CSRF_ENABLED'] = False app.config['GMISSION_IMAGE_UPLOAD_DIR'] = os.path.join(root_path, 'static', 'image', 'original') app.config['GMISSION_IMAGE_THUMB_DIR'] = os.path.join(root_path, 'static', 'image', 'thumb') app.config['GMISSION_VIDEO_UPLOAD_DIR'] = os.path.join(root_path, 'static', 'video', 'original') app.config['GMISSION_VIDEO_THUMB_DIR'] = os.path.join(root_path, 'static', 'video', 'thumb') app.config['GMISSION_AUDIO_UPLOAD_DIR'] = os.path.join(root_path, 'static', 'audio', 'original') app.config['GMISSION_LOGS_DIR'] = os.path.join(root_path, '..', '..', 'logs') # app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql://*****:*****@127.0.0.1:3306/gmission_hkust' app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql://*****:*****@docker-mysql/gmission_hkust' log.set_logger(app) app.config.from_object('email')
def __init__(self, loop, q, _log): self.loop = loop self.q = q self.logPip = _log # limit_per_host=30 limit = 100` # conn = aiohttp.TCPConnector (verify_ssl = False,limit_per_host = 2,use_dns_cache = True,loop = self.loop) connector = conn self.session = aiohttp.ClientSession(loop=self.loop) self.NETlog = log.set_logger(filename="netinterface.log", isOnlyFile=False) # NET最大并行数 self.netLimit = 10 # NET队列状态 self.netStatus = 0 # NET正在处理队列状态 self.netProcessNum = 0
def main(q, pindex=1): logPip = log.set_logger(filename="URPPipelines.log", isOnlyFile=False) loop = asyncio.new_event_loop() process = MyPipeline(loop, q, logPip) cDB = process.process_localDB() cDB.send(None) t = Thread(target=start_loop, args=(loop, )) t.setDaemon(True) t.start() index = 0 while True: try: role = q.get() # 写sqlite cDB.send((role, index)) # 写微信端 asyncio.run_coroutine_threadsafe(process.process_NET(role), loop) except StopIteration: cDB.close() logPip.info('数据存储结束! -- %s' % process.netStatus) break except Exception as e: logPip.error(traceback.format_exc()) else: pass finally: index = index + 1 while True: # NET 上传结束标志 if process.netStatus == 0: process.session.close() break logPip.error('等待微信端上传!-- %s' % process.netStatus) time.sleep(30) loop.stop() logPip.error('所有任务结束!') process.session.close()
# Internationalization # https://docs.djangoproject.com/en/1.8/topics/i18n/ LANGUAGE_CODE = 'en-us' TIME_ZONE = 'Asia/Shanghai' USE_I18N = True USE_L10N = True USE_TZ = True # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.8/howto/static-files/ STATIC_URL = '/static/' MEDIA_ROOT = '/data1/www/sport/media' MEDIA_URL = '/media/' #initialize the log # Use time-rotated file handler, each day has a different log file, see # logging.handlers.TimedRotatingFileHandler for more help about 'when' log.set_logger(when='D', limit=10240) # File log level set to INFO, and stdout log level set to DEBUG log.set_logger(level='DEBUG:WARN') # Change default log formatter #log.set_logger(filename = '/data1/www/logs/sport.log', fmt = '%(asctime)s - %(name)s - %(funcName)s -%(thread)d - %(levelname)s - %(message)s')
cDB.send((role, index)) # 写微信端 asyncio.run_coroutine_threadsafe(process.process_NET(role), loop) except StopIteration: cDB.close() logPip.info('数据存储结束! -- %s' % process.netStatus) break except Exception as e: logPip.error(traceback.format_exc()) else: pass finally: index = index + 1 while True: # NET 上传结束标志 if process.netStatus == 0: process.session.close() break logPip.error('等待微信端上传!-- %s' % process.netStatus) time.sleep(30) loop.stop() logPip.error('所有任务结束!') process.session.close() if __name__ == '__main__': pass logPip = log.set_logger(filename="URPPipelines.log", isOnlyFile=False) loop = asyncio.new_event_loop() process = MyPipeline(loop, None, logPip)
#!/bin/python import log log.set_logger(filename = '/tmp/yyy.log', mode = 'w') log.debug("Houston, we have a %s", "thorny problem", exc_info=1) log.error('this is error log for otomat')
# -*- coding: utf-8 -*- from multiprocessing.dummy import Pool as ThreadPool from multiprocessing import Queue import log import time log.set_logger(level = 'INFO') class NoticeObj(object): ''' 通知抽象类 ''' def discorver_new_file(self, new_file): pass class UploadToRemote(NoticeObj): def discorver_new_file(self,new_file): ''' 上传新文件到远端 ''' time.sleep(3) log.info(new_file) pass class CopyToLocal(NoticeObj): def discorver_new_file(self,new_file): ''' 本地持久化新文件
from PySide import QtCore, QtGui #from PyQt4 import QtOpenGL from model import Models from model import NetworkModel as NWM from starmap import MapGlyphs from starmap import MapScene from projects import ProjectManager #from forms import WorldBrowser from forms import AllegianceEditor from forms import NetworkManager as nm #from forms import Trade #from forms import Trade from resources import starmap_rc import log log.set_logger() InsertTextButton = 10 class OccurrenceSpinBox(QtGui.QSpinBox): def __init__(self): QtGui.QSpinBox.__init__(self) self.setMinimum(-2) self.setMaximum(2) self.setValue(0) self.lineEdit().setReadOnly(True) def textFromValue(self, value): if value >= 0: text = '+' + str(value)
import logging import pexpect import subprocess from pexpect import pxssh from log import logger, set_logger from remote_commands import cmd_list import arguments args = arguments.args set_logger(logfile=args.logfile, loglevel=int(args.loglevel)) # Setting command type to interactive if given command is found in cmd_list interactive = False icmd = None for i in cmd_list: if args.command in i[0]: interactive = True icmd = i break class RemoteCommand(object): def __init__(self, command, host, username, password, timeout): """ Initialize argument values """ self.command = command self.host = host self.username = username self.password = password
def main(): log.set_logger(level='DEBUG') log.debug("hello world"); log.info("info world"); log.error("error world"); pass
#!/usr/bin/python # -*- coding: utf-8 -*- import time import os import sys import net import collect import config import log log.set_logger(limit = 1024) log.set_logger(level = 'DEBUG') ''' 数据采集任务守护进程 ''' class CollectTask: def upload(self): log.info("数据采集任务守护进程启动") obj = config.Config() obj.load_config() log.info("加载配置完成") collector = collect.Collect() data_sender = net.DataSender() log.info("开始连接") data_sender.connect("tcp://127.0.0.1:8899")
help='Chose an adaptor: FLASK(Default) or DJANGO.', type=str) parser.add_argument( '-db', '--create-database', help='Creates a sql file that can be used to create a database.', action='store_true') parser.add_argument('-dl', '--draw-lattice', nargs='+', help='Draws a lattice.') args = parser.parse_args() if __name__ == '__main__': log.set_logger(args.log_level, show_path=False) path = os.path.normpath(args.filepath) directory = None if args.project_root: directory = os.path.normpath(args.project_root) else: directory = os.path.dirname(path) project_modules = get_python_modules(directory) local_modules = get_directory_modules(directory) tree = generate_ast(path) cfg = build_cfg(tree, project_modules, local_modules, path) cfg_list = [cfg]
import log import optparse from crawler.crawler import Crawler if __name__ == '__main__': logger = log.set_logger("main") parser = optparse.OptionParser() parser.add_option("-c", "--city", dest="city", default="warszawa", help="City to crawl [warsaw]") parser.add_option( "-t", "--type", dest="property_type", default="apartment", help="Type of property to crawl (house/apartment) [apartment]") parser.add_option("-o", "--offer-type", dest="offer_type", default="rent", help="Type of offer type to search (rent/sell) [rent]") parser.add_option("-s", "--start-page", dest="start_page", type="int",
# -*- coding: utf-8 -*- from multiprocessing.dummy import Pool as ThreadPool from multiprocessing import Queue import log import time log.set_logger(level='INFO') class NoticeObj(object): ''' 通知抽象类 ''' def discorver_new_file(self, new_file): pass class UploadToRemote(NoticeObj): def discorver_new_file(self, new_file): ''' 上传新文件到远端 ''' time.sleep(3) log.info(new_file) pass class CopyToLocal(NoticeObj): def discorver_new_file(self, new_file): '''
# -*- coding: utf-8 -*- import tornado.web import log import json import os SELF_TEMPLATES = 1 GLOBAL_TEMPLATES = 0 log.set_logger(level = 'DEBUG:INFO') class RealBaseHandler(tornado.web.RequestHandler): @property def user_name(self): if self.current_user: return tornado.escape.xhtml_escape(self.current_user) return "" def get_current_user(self): return self.get_secure_cookie("user") @tornado.gen.coroutine def get_api_result(self,uri, method="GET", args=None): ''' 异步请求后端接口 ''' if args: req = json.dumps(args) else: req = {}