def sendmail(BOUNCE_RETURN_ADDRESS,TO_ADDRESS_LIST,BODY):
global DISABLED
if DISABLED:
log ("Not sending to %s" % repr(TO_ADDRESS_LIST))
return
try:
mod_to_list = []
for address in TO_ADDRESS_LIST:
mod_to_list.append(shell_escape(address))
mod_bounce_address = shell_escape(BOUNCE_RETURN_ADDRESS)
cmd = "/usr/lib/sendmail -oi -f'%s' -- %s" % (mod_bounce_address,
string.join(mod_to_list, ' '))
fp = os.popen(cmd, "w", 16384)
fp.write(BODY)
r = fp.close()
if not r: return
if os.WIFEXITED(r):
if os.WEXITSTATUS(r):
raise "cmd '%s' returned %d" % (cmd, os.WEXITSTATUS(r))
elif os.WIFSIGNALED(r):
raise "cmd '%s' ended on signal %d" % (cmd, os.WTERMSIG(r))
elif os.WIFSTOPPED(r):
raise "cmd '%s' ended on signal %d" % (cmd, os.WSTOPSIG(r))
except IOError:
delayed_queue(BOUNCE_RETURN_ADDRESS, TO_ADDRESS_LIST, BODY)
except:
import handle_error
handle_error.handleException("Unable to send message")
delayed_queue(BOUNCE_RETURN_ADDRESS, TO_ADDRESS_LIST, BODY)
def __init__(self,
context,
pagename=None,
readDefaultHDF=1,
israwpage=0,
parseCGI=1,
makePUT=0,
**parms):
if pagename is None: pagename = self._pagename
if not pagename: raise NoPageName("missing pagename")
self.pagename = pagename
self.readDefaultHDF = readDefaultHDF
self._israwpage = israwpage
self.context = context
self._pageparms = parms
self._error_template = None
self.page_start_time = time.time()
if makePUT:
context.environ['REQUEST_METHOD'] = 'PUT'
neo_cgi.cgiWrap(context.stdin, context.stdout, context.environ)
neo_cgi.IgnoreEmptyFormVars(1)
self.ncgi = neo_cgi.CGI()
if parseCGI:
self.ncgi.parse()
self._path_num = 0
domain = self.ncgi.hdf.getValue("CGI.ServerName", "")
domain = self.ncgi.hdf.getValue("HTTP.Host", domain)
self.domain = domain
self.setPaths([self.ncgi.hdf.getValue("CGI.DocumentRoot", "")])
self._sent_headers = 0
self._reply_headers = {}
self._reply_code = 200
if self.ncgi.hdf.getValue("CGI.HTTPS", ""):
self.http = "https://"
else:
self.http = "http://"
try:
self.subclassinit()
except:
SHOULD_DISPLAY = 0
DISPLAY_ERROR = 1
import handle_error
handle_error.handleException("Display Failed!")
ERROR_MESSAGE = handle_error.exceptionString()
return
def get(self, doc_id, prefix, hdf):
try:
data = self._db.get(str(doc_id))
except db.error, reason:
if reason[0] == -7:
raise eNotCached
log("bad message cache for %d" % doc_id)
import handle_error
handle_error.handleException("bad cache")
raise eNotCached
def run_trends(self): ndb = self.ndb print "---------- TREND STAGE -----------------" try: trendsmanager = nc_trends.NCTrendsManager(ndb) trendsmanager.computeAllTrends() except: import handle_error handle_error.handleException()
def decodeQPrfc2047_re(matchobj):
charset = string.lower(matchobj.group('charset'))
encoding = string.lower(matchobj.group('encoding'))
encstring = matchobj.group('b64text')
if encoding == "b":
try:
outstring = base64.decodestring(encstring)
except binascii.Error, reason:
import handle_error
handle_error.handleException("bad base64")
outstring = encstring
def __init__(self, context, pagename=None,readDefaultHDF=1,israwpage=0,
parseCGI=1, makePUT=0, **parms):
if pagename is None: pagename = self._pagename
if not pagename: raise NoPageName("missing pagename")
self.pagename = pagename
self.readDefaultHDF = readDefaultHDF
self._israwpage = israwpage
self.context = context
self._pageparms = parms
self._error_template = None
self.page_start_time = time.time()
if makePUT:
context.environ['REQUEST_METHOD'] = 'PUT'
neo_cgi.cgiWrap(context.stdin, context.stdout, context.environ)
neo_cgi.IgnoreEmptyFormVars(1)
self.ncgi = neo_cgi.CGI()
if parseCGI:
self.ncgi.parse()
self._path_num = 0
domain = self.ncgi.hdf.getValue("CGI.ServerName","")
domain = self.ncgi.hdf.getValue("HTTP.Host", domain)
self.domain = domain
self.setPaths([self.ncgi.hdf.getValue("CGI.DocumentRoot","")])
self._sent_headers = 0
self._reply_headers = {}
self._reply_code = 200
if self.ncgi.hdf.getValue("CGI.HTTPS", ""):
self.http = "https://"
else:
self.http = "http://"
try:
self.subclassinit()
except:
SHOULD_DISPLAY = 0
DISPLAY_ERROR = 1
import handle_error
handle_error.handleException("Display Failed!")
ERROR_MESSAGE = handle_error.exceptionString()
return
def main(argv,stdout,environ):
hostname = myhost()
nccm = nc_cmgr.NCCollectionManager(hostname)
ncsrv = nc_srvrpc.NCServerRpc(hostname)
config = ncsrv.getConfig()
for module,module_config in config:
print "config: %s/%s" % (module,module_config)
try:
if module == "Disk":
mon = DiskMonitor.makeMonitor(nccm)
mon.collectData(module_config)
elif module == "Mem":
mon = MemMonitor.makeMonitor(nccm)
mon.collectData(module_config)
elif module == "Cpu":
mon = CpuMonitor.makeMonitor(nccm)
mon.collectData(module_config)
elif module == "Tcp":
mon = TcpMonitor.makeMonitor(nccm)
mon.collectData(module_config)
elif module == "DirQueue":
mon = DirQueueMonitor.makeMonitor(nccm)
mon.collectData(module_config)
elif module == "Proc":
mon = ProcMonitor.makeMonitor(nccm)
mon.collectData(module_config)
elif module == "InnoTbSpc":
mon = InnoTbSpcMonitor.makeMonitor(nccm)
mon.collectData(module_config)
elif module == "Mysql":
mon = MysqlMonitor.makeMonitor(nccm)
mon.collectData(module_config)
else:
fallback_monitor(module, nccm, module_config)
except:
import handle_error
handle_error.handleException()
print "=================\n" + nccm.postdata() + "\n==============\n"
ncsrv.checkIn(nccm)
def main(argv):
alist, args = getopt.getopt(argv[1:], "f:v:", ["help", "load=", "lang="])
one_file = None
verbose = 0
load_file = None
lang = 'en'
for (field, val) in alist:
if field == "--help":
usage(argv[0])
return -1
if field == "-f":
one_file = val
if field == "-v":
verbose = int(val)
if field == "--load":
load_file = val
if field == "--lang":
lang = val
global DONE
#signal.signal(signal.SIGTERM, handleSignal)
#signal.signal(signal.SIGINT, handleSignal)
log("trans: start")
start_time = time.time()
try:
t = Translator()
if load_file:
t.loadMap(load_file, 'S', lang)
else:
locations = t.loadStrings(one_file, verbose=verbose)
t.dumpStrings(locations)
t.dumpFiles(locations, 'hdf')
except KeyboardInterrupt:
pass
except:
import handle_error
handle_error.handleException("Translation Error")
def main(argv):
alist, args = getopt.getopt(argv[1:], "f:v:", ["help", "load=", "lang="])
one_file = None
verbose = 0
load_file = None
lang = 'en'
for (field, val) in alist:
if field == "--help":
usage(argv[0])
return -1
if field == "-f":
one_file = val
if field == "-v":
verbose = int(val)
if field == "--load":
load_file = val
if field == "--lang":
lang = val
global DONE
#signal.signal(signal.SIGTERM, handleSignal)
#signal.signal(signal.SIGINT, handleSignal)
log("trans: start")
start_time = time.time()
try:
t = Translator()
if load_file:
t.loadMap(load_file, 'S', lang)
else:
locations = t.loadStrings(one_file, verbose=verbose)
t.dumpStrings(locations)
t.dumpFiles(locations, 'hdf')
except KeyboardInterrupt:
pass
except:
import handle_error
handle_error.handleException("Translation Error")
def get(self, month, prefix, hdf,inv_key=None):
path = os.path.join(self._path, "cache", "idx_%s.hdf" % month)
if not os.path.exists(path):
raise eNotCached
cache_hdf = neo_util.HDF()
try:
cache_hdf.readFile(path)
except:
import handle_error
handle_error.handleException("bad index cache: %s" % path)
raise eNotCached
if inv_key:
if cache_hdf.getValue("cacheinfo.inv_key","") != inv_key:
log("cache invalidation (%s,%s!=%s)!" % (month,inv_key,cache_hdf.getValue("cacheinfo.inv_key","")))
raise eNotCached
hdf.setValue(prefix, "1")
hdf.copy(prefix,cache_hdf.getObj("cachedata"))
def rfc2047_decode2 (enc_string):
global rfc_2047_pat
enc_string = rfc_2047_pat.sub(decodeQPrfc2047_re, enc_string)
## return subject_string
# return truncated_rfc_2047_pat.sub("--",subject_string)
# try to decode truncated rfc2047 string for Japanese messages
# a) base64-decode as much of the message as possible;
# (length of encoded string must be multiple of 4 bytes)
# b) apply ISO-2022-JP-decoding to retrieve only whole multi-byte characters
#
# For non-japanese, just strip =?charset?q? part
#
# Note: requires kconv to work properly
m = my_truncated_rfc_2047_pat.match(enc_string)
if m == None:
return truncated_rfc_2047_pat.sub('', enc_string), None
output = m.group('start')
the_charset = string.lower(m.group('charset'))
the_encoding = string.lower(m.group('encoding'))
if the_charset != 'iso-2022-jp' or the_encoding != 'b':
return output + m.group('b64text'), the_charset
b64text = m.group('b64text')
l = len(b64text)
if l < 4:
return output, the_charset
b64text = b64text[0:((l / 4) * 4)] # only multiples of 4 bytes
output = []
in_double = 0 # double-byte mode flag
i = 0
try:
remainder = base64.decodestring(b64text)
except binascii.Error, reason:
import handle_error
handle_error.handleException()
remainder = b64text
def convert_html_to_text (html_s, charset):
if USE_LINKS and os.path.isfile('/usr/local/bin/links'):
temp_file = os.tmpnam()
try:
fp = open(temp_file, 'w')
fp.write(html_s)
fp.close()
fp = os.popen ('export HOME=/home/neo; /usr/local/bin/links -assume-codepage utf-8 -dump %s' % temp_file)
out = fp.read()
fp.close()
os.unlink(temp_file)
return out
except:
import handle_error
handle_error.handleException("Unable to launch links")
try:
os.unlink(temp_file)
except:
pass
from html2text import html2text
out = html2text (html_s, use_ansi = 0, is_latin1 = (charset == 'iso-8859-1'))
return out
def main(argv):
alist, args = getopt.getopt(argv[1:], "", ["help"])
tup = pwd.getpwnam("discuss")
DISCUSS_UID = tup[2]
r_uid = os.getuid()
if r_uid != DISCUSS_UID:
if r_uid == 0:
os.setreuid(DISCUSS_UID, DISCUSS_UID)
else:
raise "Most run archiver as discuss!"
lock = do_lock(DISCUSS_DATA_ROOT, "archiver.lock")
for (field, val) in alist:
if field == "--help":
usage(argv[0])
return -1
global DONE
signal.signal(signal.SIGTERM, handleSignal)
log("archiver: start")
while not DONE:
try:
archive_dir()
except KeyboardInterrupt:
DONE = 1
except:
import handle_error
handle_error.handleException("Archiver Error")
if DONE: break
# tlog("sleeping")
time.sleep(10)
def archive_dir():
base_dir = DISCUSS_MAILDIR
new_dir = os.path.join(base_dir,"new")
bounce_dir = os.path.join(base_dir,"bounces")
error_dir = os.path.join(base_dir,"cur")
# Always check permissions on directory first!
# process files...
global DONE
files = os.listdir(new_dir)
for a_file in files:
if DONE: break
filepath = os.path.join(new_dir,a_file)
tlog("*** archive message: %s" % filepath)
try:
archive_file(filepath)
except KeyboardInterrupt:
raise "user keyboard interrupt"
except:
log("should move to cur: %s" % filepath)
handle_error.handleException("on file %s" % filepath)
try:
os.link(filepath, os.path.join(error_dir,a_file))
os.unlink(filepath)
except OSError:
pass
global MDB_CACHE
for list_path in MDB_CACHE.keys():
(mdb, counts) = MDB_CACHE[list_path]
tlog("flushing %s:%d" % (list_path, counts))
mdb.flush()
MDB_CACHE = {}
def display(self):
hdf = self.ncgi.hdf
sets = []
nset = 0
while 1:
q_service_id = hdf.getIntValue("Query.serv_id.%s" % nset,-1)
if q_service_id == -1: break
q_source_id = hdf.getIntValue("Query.source_id.%s" % nset,-1)
sets.append((nset, q_service_id, q_source_id))
nset = nset + 1
if len(sets) == 0:
q_service_id = hdf.getIntValue("Query.serv_id",-1)
q_source_id = hdf.getIntValue("Query.source_id",-1)
sets.append((0, q_service_id, q_source_id))
q_graphType = hdf.getIntValue("Query.graphtype",0)
q_width = hdf.getIntValue("Query.width",550)
q_height = hdf.getIntValue("Query.height",150)
gi = GraphImage(q_width,q_height, q_graphType)
xTickRange, xMinTicks, xDisplayRange, xMinRange = figureGraphType(q_graphType)
now = time.time()
lasttime = int(now - (xDisplayRange))
for (nset, q_service_id, q_source_id) in sets:
startTime = time.time()
service = self.ndb.services.fetchRow( ('serv_id', q_service_id) )
source = self.ndb.monitor_sources.fetchRow(
('source_id', q_source_id) )
smach = self.ndb.machines.fetchRow(
('mach_id', source.source_mach_id) )
if xMinRange == 0:
h_data = self._fetchRows1(q_service_id, q_source_id, lasttime)
else:
h_data = self._fetchSummaryRows2(q_service_id, q_source_id, xMinRange, lasttime)
h_data.reverse()
log("len(h_data)=%s" % len(h_data))
endTime = time.time()
dt = endTime - startTime
log("fetching rows %.2f" % dt)
startTime = time.time()
incremental = 0
# setup fixed scaling
if service.type == "pct":
gi.setMinMax(0,100)
elif service.type == "state":
gi.setMinMax(0,1)
elif service.type in ("inc", "total"):
incremental = 1
if incremental:
n = h_data[0].value
for data in h_data:
p = Point(data.value-n, None, None, data.pstart, data.pend)
gi.addData(nset, p)
n = data.value
else:
for data in h_data:
gi.addData(nset, data)
endTime = time.time()
dt = endTime - startTime
log("generating point set %.2f" % dt)
sys.stdout.write("Content-Type: image/png\n\n")
startTime = time.time()
try:
im = gi.makeImage()
except:
import handle_error
handle_error.handleException("Error")
endTime = time.time()
dt = endTime - startTime
log("makeImage %.2f" % dt)
startTime = time.time()
im.save(sys.stdout,"PNG")
endTime = time.time()
dt = endTime - startTime
log("generating PNG %.2f" % dt)
raise DisplayDone
SHOULD_DISPLAY = 0
except Redirected:
# catch redirect exceptions
SHOULD_DISPLAY = 0
except DisplayError, num:
ncgi.hdf.setValue("Query.error", str(num))
if self._error_template:
ncgi.hdf.setValue("Content", self._error_template)
else:
DISPLAY_ERROR = 1
except:
SHOULD_DISPLAY = 0
DISPLAY_ERROR = 1
import handle_error
handle_error.handleException("Display Failed!")
ERROR_MESSAGE = handle_error.exceptionString()
if DISPLAY_ERROR:
#print "Content-Type: text/html\n\n"
# print the page
self['Content-Type'] = "text/html"
# print the page
self.push("<head>")
self.push('''
<script type="text/javascript">
function toggleDebugInfo() {
var tb = document.getElementById('%s');
def display(self):
hdf = self.ncgi.hdf
sets = []
nset = 0
while 1:
q_service_id = hdf.getIntValue("Query.serv_id.%s" % nset,-1)
if q_service_id == -1: break
q_source_id = hdf.getIntValue("Query.source_id.%s" % nset,-1)
sets.append((nset, q_service_id, q_source_id))
nset = nset + 1
if len(sets) == 0:
q_service_id = hdf.getIntValue("Query.serv_id",-1)
q_source_id = hdf.getIntValue("Query.source_id",-1)
sets.append((0, q_service_id, q_source_id))
q_width = hdf.getIntValue("Query.width",550)
q_height = hdf.getIntValue("Query.height",150)
gi = GraphImage(q_width,q_height)
for (nset, q_service_id, q_source_id) in sets:
service = self.ndb.services.fetchRow( ('serv_id', q_service_id) )
source = self.ndb.monitor_sources.fetchRow(
('source_id', q_source_id) )
smach = self.ndb.machines.fetchRow(
('mach_id', source.source_mach_id) )
h_data = self.ndb.monitor_history.fetchRows(
[ ('serv_id', q_service_id),
('source_id', q_source_id)] ,
limit_to=300, order_by=['pstart desc'])
try:
cur = self.ndb.monitor_state.fetchRow(
[ ('serv_id', q_service_id),
('source_id', q_source_id)] )
h_data.insert(0,cur)
except odb.eNoMatchingRows:
pass
h_data.reverse()
incremental = 0
# setup fixed scaling
if service.type == "pct":
gi.setMinMax(0,100)
elif service.type == "state":
gi.setMinMax(0,1)
elif service.type in ("inc", "total"):
incremental = 1
if incremental:
n = h_data[0].value
for data in h_data:
p = Point(data.value-n, data.pstart, data.pend)
gi.addData(nset, p)
n = data.value
else:
for data in h_data:
gi.addData(nset, data)
sys.stdout.write("Content-Type: image/png\n\n")
try:
gi.makeImage().save(sys.stdout,"PNG")
except:
import handle_error
handle_error.handleException("Error")
raise DisplayDone
def to_utf8 (s, charset):
charset = string.lower(charset)
charset = string.replace(charset, '"', '')
charset = string.strip(charset)
x = string.find(charset, ';')
if x != -1:
charset = charset[:x]
if not charset:
charset = "iso-8859-1"
if charset[:8] == "windows-":
charset = "cp" + charset[8:]
if charset[:4] == "win-":
charset = "cp" + charset[4:]
if charset[:3] == 'cp-':
charset = "cp" + charset[3:]
if charset == "us-ascii":
charset = "ascii"
# I've also seen kor-ascii... I don't know what that is, though
if charset in ["ksc5601", 'ks_c_5601-1987', 'ms949', '5601']:
charset = "cp949"
elif charset == 'euc_kr':
charset = 'euc-kr'
elif charset == 'koi8r':
charset = 'koi8-r'
elif charset in ["gb2312", "gb2312_charset", "chinese-gb2312"]:
charset = "gbk"
elif charset == "iso-8859-11": # libiconv doesn't have this yet
charset = "TIS-620"
elif charset == "unicode-1-1-utf-7":
charset = "utf-7"
elif charset == "unicode-1-1-utf-8":
charset = "utf-8"
elif charset in ["default", 'unknown', 'unknown-8bit', 'x-user-defined', '_charset', 'default_charset', 'x-unknown', '8859_1', 'ansi', 'iso08859-1', '8859-1', 'us-asci', 'asci', 'standard', 'ansi_charset', 'iso-8859-1-windows-3.1-latin-1']: # idiots
charset = "iso-8859-1"
elif charset in ["chinesebig5_charset", "chinesebig5"]:
charset = "big5"
elif charset == 'koi8-r.xlt':
charset = 'koi8-r'
elif charset == "x-sjis":
charset = 'sjis'
elif charset == 'iso-8859-8-i':
charset = 'iso-8859-8'
elif charset == 'dos-862':
charset = 'cp862'
elif charset == 'x-roman8':
charset = 'hp-roman8'
elif charset == "x-mac-cyrillic":
charset = 'maccyrillic'
elif charset.find('euc-kr') != -1:
charset = 'euc-kr'
elif charset.find('big5') != -1:
charset = 'big5'
elif charset != 'iso-8859-11' and charset.find('iso-8859-1') != -1:
charset = 'iso-8859-1'
elif charset.find('gb2312') != -1:
charset = 'gb2312'
elif charset.find('iso-2022-jp') != -1:
charset = 'iso-2022-jp'
if not charset:
charset = 'iso-8859-1'
if charset == 'utf-8':
return s
# use the kconv library, it works better with japanese...
if kconv is not None and charset in ['iso-2022-jp', 'sjis', 'euc-jp']:
kc = kconv.Kconv(kconv.UTF8)
try:
return kc.convert(s)
except:
import handle_error
handle_error.handleException("Unable to kconv decode [%s]" % charset)
try:
return unicode(s, charset, 'ignore').encode('utf-8')
except UnicodeError, reason:
if string.find(str(reason), "ASCII decoding error") != -1:
if string.find(s, chr(0x91)) != -1: charset = 'cp1252'
else: charset = 'iso-8859-1'
try:
return unicode(s, charset).encode('utf-8')
except UnicodeError, reason:
pass
SHOULD_DISPLAY = 0
except Redirected:
# catch redirect exceptions
SHOULD_DISPLAY = 0
except DisplayError, num:
ncgi.hdf.setValue("Query.error", str(num))
if self._error_template:
ncgi.hdf.setValue("Content", self._error_template)
else:
DISPLAY_ERROR = 1
except:
SHOULD_DISPLAY = 0
DISPLAY_ERROR = 1
import handle_error
handle_error.handleException("Display Failed!")
ERROR_MESSAGE = handle_error.exceptionString()
if DISPLAY_ERROR:
#print "Content-Type: text/html\n\n"
# print the page
self['Content-Type'] = "text/html"
# print the page
self.push("<head>")
self.push('''
<script type="text/javascript">
function toggleDebugInfo() {
var tb = document.getElementById('%s');
except:
import handle_error
handle_error.handleException("Unable to kconv decode [%s]" % charset)
try:
return unicode(s, charset, 'ignore').encode('utf-8')
except UnicodeError, reason:
if string.find(str(reason), "ASCII decoding error") != -1:
if string.find(s, chr(0x91)) != -1: charset = 'cp1252'
else: charset = 'iso-8859-1'
try:
return unicode(s, charset).encode('utf-8')
except UnicodeError, reason:
pass
import handle_error
handle_error.handleException("Unable to decode [%s] %s" % \
(charset,string.replace(string.replace(s[500:],"\n","\\n"),"\r","\\r")), dump=0)
except LookupError:
global UNKNOWN_CHARSETS
if not UNKNOWN_CHARSETS.has_key(charset):
import handle_error
handle_error.handleException("Unable to find decode charset '%s'" % charset)
UNKNOWN_CHARSETS[charset] = 1
return s
USE_LINKS = 0
def convert_html_to_text (html_s, charset):
if USE_LINKS and os.path.isfile('/usr/local/bin/links'):
temp_file = os.tmpnam()
try:
fp = open(temp_file, 'w')
