def startLogging(logfilename, sysLog, prefix, nodaemon): if logfilename == '-': if not nodaemon: print 'daemons cannot log to stdout' os._exit(1) logFile = sys.stdout elif sysLog: syslog.startLogging(prefix) elif nodaemon and not logfilename: logFile = sys.stdout else: logFile = app.getLogFile(logfilename or 'twistd.log') try: import signal except ImportError: pass else: def rotateLog(signal, frame): from twisted.internet import reactor reactor.callFromThread(logFile.rotate) signal.signal(signal.SIGUSR1, rotateLog) if not sysLog: log.startLogging(logFile) sys.stdout.flush()
def startLogging(logfilename, sysLog, prefix, nodaemon): if logfilename == "-": if not nodaemon: print "daemons cannot log to stdout" os._exit(1) logFile = sys.stdout elif sysLog: syslog.startLogging(prefix) elif nodaemon and not logfilename: logFile = sys.stdout else: logFile = app.getLogFile(logfilename or "twistd.log") try: import signal except ImportError: pass else: def rotateLog(signal, frame): from twisted.internet import reactor reactor.callFromThread(logFile.rotate) signal.signal(signal.SIGUSR1, rotateLog) if not sysLog: log.startLogging(logFile) sys.stdout.flush()
def setup_syslog(): """Setup how we want syslogging to work""" # https://stackoverflow.com/questions/13699283 frame = inspect.stack()[-1] module = inspect.getmodule(frame[0]) filename = os.path.basename(module.__file__) syslog.startLogging(prefix='pyWWA/%s' % (filename, ), facility=LOG_LOCAL2)
def setup_syslog(): """Setup how we want syslogging to work""" # https://stackoverflow.com/questions/13699283 frame = inspect.stack()[-1] module = inspect.getmodule(frame[0]) filename = os.path.basename(module.__file__) syslog.startLogging(prefix="pyWWA/%s" % (filename, ), facility=LOG_LOCAL2) # pyIEM does logging via python stdlib logging, so we need to patch those # messages into twisted's logger. LOG.addHandler(logging.StreamHandler(stream=log.logfile)) # Allow for more verbosity when we are running this manually. LOG.setLevel(logging.DEBUG if sys.stdout.isatty() else logging.INFO)
def __init__(self, configFile): global _coreInstance if _coreInstance: raise Exception("Instance of app already exists") _coreInstance = self self._deferredStopList = [] from rcore.config import config config.reload(configFile) try: logDest = config()['log']['destination'] if logDest == 'syslog': from twisted.python import syslog try: prefix = config()['log']['syslogprefix'] except: prefix = os.path.basename(sys.argv[0]) syslog.startLogging(prefix) elif logDest == 'stdout': log.startLogging(sys.stdout) else: dn = os.path.dirname(logDest) if not dn: dn = self.get_default_log_dir() if dn and not os.path.exists(dn): os.makedirs(dn, 0o755) log.startLogging(DailyLogFile(os.path.basename(logDest), dn)) except Exception as e: log.startLogging(sys.stdout) log.msg( "Setting log from config file is failed. continue with logging to stdout: " + str(e)) from rcore.alarm import Alarm self._rpcServices = {} self._users = {} self.mainContextId = makeContext(MainContext) setCurrentContext(self.mainContextId)
def run(base_directory, usesyslog=None): if usesyslog: from twisted.python import log,syslog syslog.startLogging(prefix=usesyslog) if len(sys.argv) <= 1 or not sys.argv[1] in ("start", "stop", "tail"): print "Huh?" print "(start|stop|tail)" return 1 if sys.argv[1] == "tail": try: return subprocess.call(["tail", "-f", os.path.join(base_directory, "twistd.log")]) except KeyboardInterrupt: return 0 if sys.argv[1] == "start": sys.argv[1:] = ["start", base_directory] elif sys.argv[1] == "stop": sys.argv[1:] = ["stop", base_directory] base_run()
def __init__(self, configFile): global _coreInstance if _coreInstance: raise Exception("Instance of app already exists") _coreInstance = self self._deferredStopList = [] from rcore.config import config config.reload(configFile) try: logDest = config()['log']['destination'] if logDest == 'syslog': from twisted.python import syslog try: prefix = config()['log']['syslogprefix'] except: prefix = os.path.basename(sys.argv[0]) syslog.startLogging(prefix) elif logDest == 'stdout': log.startLogging(sys.stdout) else: dn = os.path.dirname(logDest) if not dn: dn = self.get_default_log_dir() if dn and not os.path.exists(dn): os.makedirs(dn, 0755) log.startLogging(DailyLogFile(os.path.basename(logDest), dn)) except Exception as e: log.startLogging(sys.stdout) log.msg("Setting log from config file is failed. continue with logging to stdout: " + str(e)) from rcore.alarm import Alarm self._rpcServices = {} self._users = {} self.mainContextId = makeContext(MainContext) setCurrentContext(self.mainContextId)
# Give process it's own session under init os.setsid() # Final fork try: pid = os.fork() if pid > 0: pid_file = open(working_dir + '/mailrfd.pid', 'w') pid_file.write(str(pid)) pid_file.close() sys.exit() except OSError, e: print e sys.exit(1) if __debug__: log.startLogging(sys.stderr) else: syslog.startLogging(prefix='mailrfd', options=LOG_PID, facility=LOG_MAIL) if __debug__: endpoint = TCP4ServerEndpoint(reactor, 8027, interface='localhost') else: endpoint = UNIXServerEndpoint(reactor, working_dir + '/socket') endpoint.listen(MailRfFactory()) reactor.run()
import os import re import datetime from twisted.python import log, syslog from twisted.internet import stdio from twisted.internet import reactor from twisted.protocols import basic from twisted.enterprise import adbapi from applib.parser import parser WMO_RE = re.compile( (r"^([0-9A-Za-z]{4,6}) ([A-Z0-9]{4}) ([0-9]{6})( [A-Z]{3})?" r"( /p[A-Z0-9]{3,6})?")) syslog.startLogging(prefix="id3b_ingest", facility=LOG_LOCAL2) CFGFN = "%s/settings.json" % (os.path.join(os.path.dirname(__file__), "../config"), ) CONFIG = json.load(open(CFGFN)) DBOPTS = CONFIG["databaserw"] DBPOOL = adbapi.ConnectionPool( "psycopg2", database=DBOPTS["name"], cp_reconnect=True, cp_max=20, host=DBOPTS["host"], user=DBOPTS["user"], password=DBOPTS["password"], )
connector.host = twisted.internet.reactor.dns_cache.get( self.host, self.port) connector.connect() return self.request.fail("Gateway Error", str(reason)) class ProxyFactory(twisted.web.http.HTTPFactory): protocol = ConnectProxy noisy = False if __name__ == '__main__': import sys if config.LOG_TYPE is 'syslog': syslog.startLogging(prefix='tecproxy', facility=config.LOG_FACILITY) else: log.startLogging(sys.stderr) factory = ProxyFactory() twisted.internet.reactor.dns_cache = DnsCache(ttl=config.DNS_TTL) for ip in config.LISTEN_IP: twisted.internet.reactor.listenTCP( config.LISTEN_PORT, factory, interface=ip ) twisted.internet.reactor.run()
Another twistd server is running, PID %s\n This could either be a previously started instance of your application or a different application entirely. To start a new one, either run it in some other directory, or use my --pidfile and --logfile parameters to avoid clashes. """ % pid) if config['logfile'] == '-': if not config['nodaemon']: print 'daemons cannot log to stdout' os._exit(1) logFile = sys.stdout elif config['nodaemon'] and not config['logfile']: logFile = sys.stdout elif config['syslog']: from twisted.python import syslog syslog.startLogging(config['prefix']) else: logPath = os.path.abspath(config['logfile'] or 'twistd.log') logFile = logfile.LogFile(os.path.basename(logPath), os.path.dirname(logPath)) # rotate logs on SIGUSR1 if os.name == "posix": import signal def rotateLog(signal, frame, logFile=logFile): from twisted.internet import reactor reactor.callLater(0, logFile.rotate) signal.signal(signal.SIGUSR1, rotateLog) oldstdin = sys.stdin oldstdout = sys.stdout
Ua.cId = SipCallId() req = Ua.genRequest("REGISTER") Ua.changeState((UacStateTrying, )) global_config['_sip_tm'].regConsumer(Ua, str(Ua.cId)) Ua.tr = global_config['_sip_tm'].newTransaction(req, Ua.recvResponse) def recvRequest(req): if req.getMethod() in ('NOTIFY', 'INFO', 'PING'): return (req.genResponse(200, 'OK'), None, None) else: return (req.genResponse(501, 'Not Implemented'), None, None) if __name__ == '__main__': syslog.startLogging('RegAgent') # Get config file configuration = ConfigFile('/etc/registrar_agent/config.ini') try: opts, args = getopt.getopt(sys.argv[1:], 'c:') except getopt.GetoptError: print 'usage: registrar_agent.py [-c config]' sys.exit(1) for o, a in opts: if o == '-c': configuration = a continue
""" Aviation Product Parser! """ import common import os # Twisted Python imports from twisted.internet import reactor from syslog import LOG_LOCAL2 from twisted.python import syslog syslog.startLogging(prefix='pyWWA/aviation', facility=LOG_LOCAL2) from twisted.python import log # pyWWA stuff from pyldm import ldmbridge from pyiem.nws.products.sigmet import parser DBPOOL = common.get_database('postgis') # Load LOCS table LOCS = {} _MYDIR = os.path.dirname(os.path.abspath(__file__)) TABLE_PATH = os.path.normpath(os.path.join(_MYDIR, "..", "tables")) def load_database(txn): txn.execute(""" SELECT id, name, ST_x(geom) as lon, ST_y(geom) as lat from stations WHERE network ~* 'ASOS' or network ~* 'AWOS' """) for row in txn:
""" import json import os import datetime from syslog import LOG_LOCAL2 from twisted.internet import reactor from twisted.enterprise import adbapi from twisted.python import syslog from applib import ldmbridge # This is a hack that prevents a strange exception with datetime and threading datetime.datetime.strptime("2017", "%Y") # Start logging syslog.startLogging(prefix="rtstats/app", facility=LOG_LOCAL2, setStdout=True) def ready(_, dbpool): """run the server components""" protocol = ldmbridge.RTStatsIngestor() protocol.dbpool = dbpool ldmbridge.LDMProductFactory(protocol) def load_dbtables(cursor): pass if __name__ == "__main__": fn = "%s/settings.json" % (os.path.join(os.path.dirname(__file__),
ua.conn_cbs = None ua.disc_cbs = None ua.fail_cbs = None ua.dead_cbs = None ua.tr = None def recvEvent(self, event, ua): msg("recvEvent") # Don't think that they need to incerconnect at all pass def recvRequest(req): pass if __name__ == '__main__': syslog.startLogging('callback') try: opts, args = getopt.getopt(sys.argv[1:], 'fp:c:') except getopt.GetoptError: print 'usage: callback.py [-f] [-p pidfile] [-c config]' sys.exit(1) pidfile = None config = '/etc/sip-callback/config.ini' foreground = False json_port = 8000 for o, a in opts: if o == '-p': pidfile = a
def parse_feed(self, xmlfeed): return feedparser.parse(xmlfeed) class RssParserProtocol(Protocol): def __init__(self, finished): self.finished = finished self.buffer = StringIO() def dataReceived(self, data): self.buffer.write(data) def connectionLost(self, reason=connectionDone): self.finished.callback(self.buffer.getvalue()) def error(traceback): print traceback if __name__ == "__main__": startLogging(prefix='myrefs') rss_feeds = RssFeedsRepository() root = Resource() root.putChild('rssfeeds', RssFeedsResource(rss_feeds)) root.putChild('', File('static')) root.putChild('updatefeeds', CheckRssFeedsResource(rss_feeds)) app_reactor.listenTCP(8888, server.Site(root)) app_reactor.run()
print e sys.exit(1) # Give process it's own session under init os.setsid() # Final fork try: pid = os.fork() if pid > 0: pid_file = open(working_dir + '/mailrfd.pid', 'w') pid_file.write(str(pid)) pid_file.close() sys.exit() except OSError, e: print e sys.exit(1) if __debug__: log.startLogging(sys.stderr) else: syslog.startLogging(prefix='mailrfd', options=LOG_PID, facility=LOG_MAIL) if __debug__: endpoint = TCP4ServerEndpoint(reactor, 8027, interface='localhost') else: endpoint = UNIXServerEndpoint(reactor, working_dir + '/socket') endpoint.listen(MailRfFactory()) reactor.run()
""" HML parser! """ from syslog import LOG_LOCAL2 from twisted.python import syslog syslog.startLogging(prefix='pyWWA/hml_parser', facility=LOG_LOCAL2) # Twisted Python imports from twisted.internet import reactor from twisted.python import log # Standard Python modules import datetime import common import os from pyldm import ldmbridge from pyiem.nws.products.hml import parser as hmlparser DBPOOL = common.get_database("hads") # LDM Ingestor class MyProductIngestor(ldmbridge.LDMProductReceiver): """ I receive products from ldmbridge and process them 1 by 1 :) """ def connectionLost(self, reason): log.msg('connectionLost') log.err(reason) reactor.callLater(5, self.shutdown) def shutdown(self): reactor.callWhenRunning(reactor.stop)
""" Twisted Way to dump data to the database """ # Twisted Python imports from syslog import LOG_LOCAL2 from twisted.python import syslog syslog.startLogging(prefix='pyWWA/afos_dump', facility=LOG_LOCAL2) from twisted.python import log from twisted.internet import reactor import sys from pyldm import ldmbridge from pyiem.nws import product from pyiem.nws.ugc import UGCParseException import common import datetime import pytz DBPOOL = common.get_database('afos') def shutdown(): """ Down we go! """ log.msg("Stopping...") reactor.callWhenRunning(reactor.stop) # LDM Ingestor class MyProductIngestor(ldmbridge.LDMProductReceiver): """ I receive products from ldmbridge and process them 1 by 1 :) """
""" MOS Data Ingestor, why not? """ # Twisted Python imports from syslog import LOG_LOCAL2 from twisted.python import syslog syslog.startLogging(prefix="pyWWA/mos_parser", facility=LOG_LOCAL2) # Twisted Python imports from twisted.internet import reactor # Standard Python modules import re import datetime import pytz # pyWWA stuff from pyldm import ldmbridge import common DBPOOL = common.get_database("mos") class myProductIngestor(ldmbridge.LDMProductReceiver): def process_data(self, buf): """ Actual ingestor """ try: real_process(buf)
Ua.lCSeq = 1 Ua.rCSeq = 1 Ua.cId = SipCallId() req = Ua.genRequest("REGISTER") Ua.changeState((UacStateTrying,)) global_config['_sip_tm'].regConsumer(Ua, str(Ua.cId)) Ua.tr = global_config['_sip_tm'].newTransaction(req, Ua.recvResponse) def recvRequest(req): if req.getMethod() in ('NOTIFY', 'INFO', 'PING'): return (req.genResponse(200, 'OK'), None, None) else: return (req.genResponse(501, 'Not Implemented'), None, None) if __name__ == '__main__': syslog.startLogging('RegAgent') # Get config file configuration = ConfigFile('/etc/registrar_agent/config.ini') try: opts, args = getopt.getopt(sys.argv[1:], 'c:') except getopt.GetoptError: print 'usage: registrar_agent.py [-c config]' sys.exit(1) for o, a in opts: if o == '-c': configuration = a continue
# Setup logging try: fn_log = cfg_central.get("logging", "log_file") except NoOptionError: fn_log = "" try: log_prefix = cfg_central.get("logging", "syslog_prefix") except NoOptionError: log_prefix = "Shiji" if fn_log: tw_log.startLogging(open(fn_log, "w")) else: tw_syslog.startLogging(prefix=log_prefix, facility=syslog.LOG_LOCAL0) # Install reactor try: if reactor_type.lower() == "epoll": from twisted.internet import epollreactor as selected_reactor from twisted.internet import reactor elif reactor_type.lower() == "select": from twisted.internet import selectreactor as selected_reactor from twisted.internet import reactor elif reactor_type.lower() == "kqueue": from twisted.internet import kqreactor as selected_reactor from twisted.internet import reactor except IOError, e: print "Error starting up reactor type '%s'. (%s)" % (reactor_type, str(e))
# Setup logging try: fn_log = cfg_central.get("logging", "log_file") except NoOptionError: fn_log = "" try: log_prefix = cfg_central.get("logging", "syslog_prefix") except NoOptionError: log_prefix = "Shiji" if fn_log: tw_log.startLogging(open(fn_log, "w")) else: tw_syslog.startLogging(prefix=log_prefix, facility=syslog.LOG_LOCAL0) # Install reactor try: if reactor_type.lower() == "epoll": from twisted.internet import epollreactor as selected_reactor from twisted.internet import reactor elif reactor_type.lower() == "select": from twisted.internet import selectreactor as selected_reactor from twisted.internet import reactor elif reactor_type.lower() == "kqueue": from twisted.internet import kqreactor as selected_reactor from twisted.internet import reactor except IOError, e: print "Error starting up reactor type '%s'. (%s)" % (reactor_type, str(e)) sys.exit(-2)