def startLogging(logfilename, sysLog, prefix, nodaemon):
if logfilename == '-':
if not nodaemon:
print 'daemons cannot log to stdout'
os._exit(1)
logFile = sys.stdout
elif sysLog:
syslog.startLogging(prefix)
elif nodaemon and not logfilename:
logFile = sys.stdout
else:
logFile = app.getLogFile(logfilename or 'twistd.log')
try:
import signal
except ImportError:
pass
else:
def rotateLog(signal, frame):
from twisted.internet import reactor
reactor.callFromThread(logFile.rotate)
signal.signal(signal.SIGUSR1, rotateLog)
if not sysLog:
log.startLogging(logFile)
sys.stdout.flush()
def startLogging(logfilename, sysLog, prefix, nodaemon):
if logfilename == "-":
if not nodaemon:
print "daemons cannot log to stdout"
os._exit(1)
logFile = sys.stdout
elif sysLog:
syslog.startLogging(prefix)
elif nodaemon and not logfilename:
logFile = sys.stdout
else:
logFile = app.getLogFile(logfilename or "twistd.log")
try:
import signal
except ImportError:
pass
else:
def rotateLog(signal, frame):
from twisted.internet import reactor
reactor.callFromThread(logFile.rotate)
signal.signal(signal.SIGUSR1, rotateLog)
if not sysLog:
log.startLogging(logFile)
sys.stdout.flush()
def setup_syslog():
"""Setup how we want syslogging to work"""
# https://stackoverflow.com/questions/13699283
frame = inspect.stack()[-1]
module = inspect.getmodule(frame[0])
filename = os.path.basename(module.__file__)
syslog.startLogging(prefix='pyWWA/%s' % (filename, ),
facility=LOG_LOCAL2)
def setup_syslog():
"""Setup how we want syslogging to work"""
# https://stackoverflow.com/questions/13699283
frame = inspect.stack()[-1]
module = inspect.getmodule(frame[0])
filename = os.path.basename(module.__file__)
syslog.startLogging(prefix="pyWWA/%s" % (filename, ), facility=LOG_LOCAL2)
# pyIEM does logging via python stdlib logging, so we need to patch those
# messages into twisted's logger.
LOG.addHandler(logging.StreamHandler(stream=log.logfile))
# Allow for more verbosity when we are running this manually.
LOG.setLevel(logging.DEBUG if sys.stdout.isatty() else logging.INFO)
def __init__(self, configFile):
global _coreInstance
if _coreInstance:
raise Exception("Instance of app already exists")
_coreInstance = self
self._deferredStopList = []
from rcore.config import config
config.reload(configFile)
try:
logDest = config()['log']['destination']
if logDest == 'syslog':
from twisted.python import syslog
try:
prefix = config()['log']['syslogprefix']
except:
prefix = os.path.basename(sys.argv[0])
syslog.startLogging(prefix)
elif logDest == 'stdout':
log.startLogging(sys.stdout)
else:
dn = os.path.dirname(logDest)
if not dn:
dn = self.get_default_log_dir()
if dn and not os.path.exists(dn):
os.makedirs(dn, 0o755)
log.startLogging(DailyLogFile(os.path.basename(logDest), dn))
except Exception as e:
log.startLogging(sys.stdout)
log.msg(
"Setting log from config file is failed. continue with logging to stdout: "
+ str(e))
from rcore.alarm import Alarm
self._rpcServices = {}
self._users = {}
self.mainContextId = makeContext(MainContext)
setCurrentContext(self.mainContextId)
def run(base_directory, usesyslog=None):
if usesyslog:
from twisted.python import log,syslog
syslog.startLogging(prefix=usesyslog)
if len(sys.argv) <= 1 or not sys.argv[1] in ("start", "stop", "tail"):
print "Huh?"
print "(start|stop|tail)"
return 1
if sys.argv[1] == "tail":
try:
return subprocess.call(["tail", "-f", os.path.join(base_directory, "twistd.log")])
except KeyboardInterrupt:
return 0
if sys.argv[1] == "start":
sys.argv[1:] = ["start", base_directory]
elif sys.argv[1] == "stop":
sys.argv[1:] = ["stop", base_directory]
base_run()
def __init__(self, configFile):
global _coreInstance
if _coreInstance:
raise Exception("Instance of app already exists")
_coreInstance = self
self._deferredStopList = []
from rcore.config import config
config.reload(configFile)
try:
logDest = config()['log']['destination']
if logDest == 'syslog':
from twisted.python import syslog
try:
prefix = config()['log']['syslogprefix']
except:
prefix = os.path.basename(sys.argv[0])
syslog.startLogging(prefix)
elif logDest == 'stdout':
log.startLogging(sys.stdout)
else:
dn = os.path.dirname(logDest)
if not dn:
dn = self.get_default_log_dir()
if dn and not os.path.exists(dn):
os.makedirs(dn, 0755)
log.startLogging(DailyLogFile(os.path.basename(logDest), dn))
except Exception as e:
log.startLogging(sys.stdout)
log.msg("Setting log from config file is failed. continue with logging to stdout: " + str(e))
from rcore.alarm import Alarm
self._rpcServices = {}
self._users = {}
self.mainContextId = makeContext(MainContext)
setCurrentContext(self.mainContextId)
# Give process it's own session under init
os.setsid()
# Final fork
try:
pid = os.fork()
if pid > 0:
pid_file = open(working_dir + '/mailrfd.pid', 'w')
pid_file.write(str(pid))
pid_file.close()
sys.exit()
except OSError, e:
print e
sys.exit(1)
if __debug__:
log.startLogging(sys.stderr)
else:
syslog.startLogging(prefix='mailrfd',
options=LOG_PID,
facility=LOG_MAIL)
if __debug__:
endpoint = TCP4ServerEndpoint(reactor, 8027, interface='localhost')
else:
endpoint = UNIXServerEndpoint(reactor, working_dir + '/socket')
endpoint.listen(MailRfFactory())
reactor.run()
import os
import re
import datetime
from twisted.python import log, syslog
from twisted.internet import stdio
from twisted.internet import reactor
from twisted.protocols import basic
from twisted.enterprise import adbapi
from applib.parser import parser
WMO_RE = re.compile(
(r"^([0-9A-Za-z]{4,6}) ([A-Z0-9]{4}) ([0-9]{6})( [A-Z]{3})?"
r"( /p[A-Z0-9]{3,6})?"))
syslog.startLogging(prefix="id3b_ingest", facility=LOG_LOCAL2)
CFGFN = "%s/settings.json" % (os.path.join(os.path.dirname(__file__),
"../config"), )
CONFIG = json.load(open(CFGFN))
DBOPTS = CONFIG["databaserw"]
DBPOOL = adbapi.ConnectionPool(
"psycopg2",
database=DBOPTS["name"],
cp_reconnect=True,
cp_max=20,
host=DBOPTS["host"],
user=DBOPTS["user"],
password=DBOPTS["password"],
)
connector.host = twisted.internet.reactor.dns_cache.get(
self.host, self.port)
connector.connect()
return
self.request.fail("Gateway Error", str(reason))
class ProxyFactory(twisted.web.http.HTTPFactory):
protocol = ConnectProxy
noisy = False
if __name__ == '__main__':
import sys
if config.LOG_TYPE is 'syslog':
syslog.startLogging(prefix='tecproxy', facility=config.LOG_FACILITY)
else:
log.startLogging(sys.stderr)
factory = ProxyFactory()
twisted.internet.reactor.dns_cache = DnsCache(ttl=config.DNS_TTL)
for ip in config.LISTEN_IP:
twisted.internet.reactor.listenTCP(
config.LISTEN_PORT,
factory,
interface=ip
)
twisted.internet.reactor.run()
Another twistd server is running, PID %s\n
This could either be a previously started instance of your application or a
different application entirely. To start a new one, either run it in some other
directory, or use my --pidfile and --logfile parameters to avoid clashes.
""" % pid)
if config['logfile'] == '-':
if not config['nodaemon']:
print 'daemons cannot log to stdout'
os._exit(1)
logFile = sys.stdout
elif config['nodaemon'] and not config['logfile']:
logFile = sys.stdout
elif config['syslog']:
from twisted.python import syslog
syslog.startLogging(config['prefix'])
else:
logPath = os.path.abspath(config['logfile'] or 'twistd.log')
logFile = logfile.LogFile(os.path.basename(logPath), os.path.dirname(logPath))
# rotate logs on SIGUSR1
if os.name == "posix":
import signal
def rotateLog(signal, frame, logFile=logFile):
from twisted.internet import reactor
reactor.callLater(0, logFile.rotate)
signal.signal(signal.SIGUSR1, rotateLog)
oldstdin = sys.stdin
oldstdout = sys.stdout
Ua.cId = SipCallId()
req = Ua.genRequest("REGISTER")
Ua.changeState((UacStateTrying, ))
global_config['_sip_tm'].regConsumer(Ua, str(Ua.cId))
Ua.tr = global_config['_sip_tm'].newTransaction(req, Ua.recvResponse)
def recvRequest(req):
if req.getMethod() in ('NOTIFY', 'INFO', 'PING'):
return (req.genResponse(200, 'OK'), None, None)
else:
return (req.genResponse(501, 'Not Implemented'), None, None)
if __name__ == '__main__':
syslog.startLogging('RegAgent')
# Get config file
configuration = ConfigFile('/etc/registrar_agent/config.ini')
try:
opts, args = getopt.getopt(sys.argv[1:], 'c:')
except getopt.GetoptError:
print 'usage: registrar_agent.py [-c config]'
sys.exit(1)
for o, a in opts:
if o == '-c':
configuration = a
continue
""" Aviation Product Parser! """
import common
import os
# Twisted Python imports
from twisted.internet import reactor
from syslog import LOG_LOCAL2
from twisted.python import syslog
syslog.startLogging(prefix='pyWWA/aviation', facility=LOG_LOCAL2)
from twisted.python import log
# pyWWA stuff
from pyldm import ldmbridge
from pyiem.nws.products.sigmet import parser
DBPOOL = common.get_database('postgis')
# Load LOCS table
LOCS = {}
_MYDIR = os.path.dirname(os.path.abspath(__file__))
TABLE_PATH = os.path.normpath(os.path.join(_MYDIR, "..", "tables"))
def load_database(txn):
txn.execute("""
SELECT id, name, ST_x(geom) as lon, ST_y(geom) as lat from stations
WHERE network ~* 'ASOS' or network ~* 'AWOS'
""")
for row in txn:
"""
import json
import os
import datetime
from syslog import LOG_LOCAL2
from twisted.internet import reactor
from twisted.enterprise import adbapi
from twisted.python import syslog
from applib import ldmbridge
# This is a hack that prevents a strange exception with datetime and threading
datetime.datetime.strptime("2017", "%Y")
# Start logging
syslog.startLogging(prefix="rtstats/app", facility=LOG_LOCAL2, setStdout=True)
def ready(_, dbpool):
"""run the server components"""
protocol = ldmbridge.RTStatsIngestor()
protocol.dbpool = dbpool
ldmbridge.LDMProductFactory(protocol)
def load_dbtables(cursor):
pass
if __name__ == "__main__":
fn = "%s/settings.json" % (os.path.join(os.path.dirname(__file__),
ua.conn_cbs = None
ua.disc_cbs = None
ua.fail_cbs = None
ua.dead_cbs = None
ua.tr = None
def recvEvent(self, event, ua):
msg("recvEvent")
# Don't think that they need to incerconnect at all
pass
def recvRequest(req):
pass
if __name__ == '__main__':
syslog.startLogging('callback')
try:
opts, args = getopt.getopt(sys.argv[1:], 'fp:c:')
except getopt.GetoptError:
print 'usage: callback.py [-f] [-p pidfile] [-c config]'
sys.exit(1)
pidfile = None
config = '/etc/sip-callback/config.ini'
foreground = False
json_port = 8000
for o, a in opts:
if o == '-p':
pidfile = a
def parse_feed(self, xmlfeed):
return feedparser.parse(xmlfeed)
class RssParserProtocol(Protocol):
def __init__(self, finished):
self.finished = finished
self.buffer = StringIO()
def dataReceived(self, data):
self.buffer.write(data)
def connectionLost(self, reason=connectionDone):
self.finished.callback(self.buffer.getvalue())
def error(traceback):
print traceback
if __name__ == "__main__":
startLogging(prefix='myrefs')
rss_feeds = RssFeedsRepository()
root = Resource()
root.putChild('rssfeeds', RssFeedsResource(rss_feeds))
root.putChild('', File('static'))
root.putChild('updatefeeds', CheckRssFeedsResource(rss_feeds))
app_reactor.listenTCP(8888, server.Site(root))
app_reactor.run()
print e
sys.exit(1)
# Give process it's own session under init
os.setsid()
# Final fork
try:
pid = os.fork()
if pid > 0:
pid_file = open(working_dir + '/mailrfd.pid', 'w')
pid_file.write(str(pid))
pid_file.close()
sys.exit()
except OSError, e:
print e
sys.exit(1)
if __debug__:
log.startLogging(sys.stderr)
else:
syslog.startLogging(prefix='mailrfd', options=LOG_PID, facility=LOG_MAIL)
if __debug__:
endpoint = TCP4ServerEndpoint(reactor, 8027, interface='localhost')
else:
endpoint = UNIXServerEndpoint(reactor, working_dir + '/socket')
endpoint.listen(MailRfFactory())
reactor.run()
""" HML parser! """
from syslog import LOG_LOCAL2
from twisted.python import syslog
syslog.startLogging(prefix='pyWWA/hml_parser', facility=LOG_LOCAL2)
# Twisted Python imports
from twisted.internet import reactor
from twisted.python import log
# Standard Python modules
import datetime
import common
import os
from pyldm import ldmbridge
from pyiem.nws.products.hml import parser as hmlparser
DBPOOL = common.get_database("hads")
# LDM Ingestor
class MyProductIngestor(ldmbridge.LDMProductReceiver):
""" I receive products from ldmbridge and process them 1 by 1 :) """
def connectionLost(self, reason):
log.msg('connectionLost')
log.err(reason)
reactor.callLater(5, self.shutdown)
def shutdown(self):
reactor.callWhenRunning(reactor.stop)
""" Twisted Way to dump data to the database """
# Twisted Python imports
from syslog import LOG_LOCAL2
from twisted.python import syslog
syslog.startLogging(prefix='pyWWA/afos_dump', facility=LOG_LOCAL2)
from twisted.python import log
from twisted.internet import reactor
import sys
from pyldm import ldmbridge
from pyiem.nws import product
from pyiem.nws.ugc import UGCParseException
import common
import datetime
import pytz
DBPOOL = common.get_database('afos')
def shutdown():
""" Down we go! """
log.msg("Stopping...")
reactor.callWhenRunning(reactor.stop)
# LDM Ingestor
class MyProductIngestor(ldmbridge.LDMProductReceiver):
""" I receive products from ldmbridge and process them 1 by 1 :) """
""" MOS Data Ingestor, why not? """
# Twisted Python imports
from syslog import LOG_LOCAL2
from twisted.python import syslog
syslog.startLogging(prefix="pyWWA/mos_parser", facility=LOG_LOCAL2)
# Twisted Python imports
from twisted.internet import reactor
# Standard Python modules
import re
import datetime
import pytz
# pyWWA stuff
from pyldm import ldmbridge
import common
DBPOOL = common.get_database("mos")
class myProductIngestor(ldmbridge.LDMProductReceiver):
def process_data(self, buf):
"""
Actual ingestor
"""
try:
real_process(buf)
Ua.lCSeq = 1
Ua.rCSeq = 1
Ua.cId = SipCallId()
req = Ua.genRequest("REGISTER")
Ua.changeState((UacStateTrying,))
global_config['_sip_tm'].regConsumer(Ua, str(Ua.cId))
Ua.tr = global_config['_sip_tm'].newTransaction(req, Ua.recvResponse)
def recvRequest(req):
if req.getMethod() in ('NOTIFY', 'INFO', 'PING'):
return (req.genResponse(200, 'OK'), None, None)
else:
return (req.genResponse(501, 'Not Implemented'), None, None)
if __name__ == '__main__':
syslog.startLogging('RegAgent')
# Get config file
configuration = ConfigFile('/etc/registrar_agent/config.ini')
try:
opts, args = getopt.getopt(sys.argv[1:], 'c:')
except getopt.GetoptError:
print 'usage: registrar_agent.py [-c config]'
sys.exit(1)
for o, a in opts:
if o == '-c':
configuration = a
continue
# Setup logging
try:
fn_log = cfg_central.get("logging", "log_file")
except NoOptionError:
fn_log = ""
try:
log_prefix = cfg_central.get("logging", "syslog_prefix")
except NoOptionError:
log_prefix = "Shiji"
if fn_log:
tw_log.startLogging(open(fn_log, "w"))
else:
tw_syslog.startLogging(prefix=log_prefix, facility=syslog.LOG_LOCAL0)
# Install reactor
try:
if reactor_type.lower() == "epoll":
from twisted.internet import epollreactor as selected_reactor
from twisted.internet import reactor
elif reactor_type.lower() == "select":
from twisted.internet import selectreactor as selected_reactor
from twisted.internet import reactor
elif reactor_type.lower() == "kqueue":
from twisted.internet import kqreactor as selected_reactor
from twisted.internet import reactor
except IOError, e:
print "Error starting up reactor type '%s'. (%s)" % (reactor_type,
str(e))
# Setup logging
try:
fn_log = cfg_central.get("logging", "log_file")
except NoOptionError:
fn_log = ""
try:
log_prefix = cfg_central.get("logging", "syslog_prefix")
except NoOptionError:
log_prefix = "Shiji"
if fn_log:
tw_log.startLogging(open(fn_log, "w"))
else:
tw_syslog.startLogging(prefix=log_prefix, facility=syslog.LOG_LOCAL0)
# Install reactor
try:
if reactor_type.lower() == "epoll":
from twisted.internet import epollreactor as selected_reactor
from twisted.internet import reactor
elif reactor_type.lower() == "select":
from twisted.internet import selectreactor as selected_reactor
from twisted.internet import reactor
elif reactor_type.lower() == "kqueue":
from twisted.internet import kqreactor as selected_reactor
from twisted.internet import reactor
except IOError, e:
print "Error starting up reactor type '%s'. (%s)" % (reactor_type, str(e))
sys.exit(-2)