def install(host, src, dstdir):
if isLocal(host):
if not exists(host, src):
util.output("file does not exist: %s" % src)
return False
dst = os.path.join(dstdir, os.path.basename(src))
if exists(host, dst):
# Do not clobber existing files/dirs (this is not an error)
return True
util.debug(1, "cp %s %s" % (src, dstdir))
try:
if os.path.isfile(src):
shutil.copy2(src, dstdir)
elif os.path.isdir(src):
shutil.copytree(src, dst)
except OSError:
# Python 2.6 has a bug where this may fail on NFS. So we just
# ignore errors.
pass
else:
util.error("install() not yet supported for remote hosts")
return True
def calcHistogramTrhreshold(self, imgf):
hist = cv2.calcHist([imgf],[0],None,[256],[0,256])
threshold = 255
bins = len(hist)
index = 0
total = 0;
while( index < bins ):
if(self.traced):
util.output("hist "+ str(index)+ " value "+ str(hist[index]))
total += hist[index]
index += 1;
soglia = total * 0.005 ;
index = bins -1
cumulative = 0
while( index > 0 ):
cumulative += hist[index]
if( cumulative > soglia ):
break;
threshold -= 1
index -= 1
if(self.traced):
util.output("Threshold is "+str(threshold));
if( threshold >= 254 ):
threshold = 254
else:
threshold += 1
self.threshold = threshold
if(self.traced):
util.show('before tresh', imgf)
return threshold;
def init(app):
util.output('wmd init')
global _app
_app = app
bottle.TEMPLATE_PATH.append(
os.path.join(os.path.dirname(__file__), 'views'))
route()
def best_matches(query, nquestions):
bests = PriorityQueue()
matches = []
query_tfidf = {}
# weird case where query is exactly a sentence in the article
if query in scores:
util.output("query '%s' is a sentence in the article" % query)
return [query]
query_tokens = tokenize(query, stemmer)
for t in query_tokens:
freq = query_tokens.count(t)
q_tf = freq / float(len(query_tokens))
if t in idf:
query_tfidf[t] = q_tf * idf[t]
for sentence in scores:
s_tfidf = scores[sentence]['tf-idf']
bests.put((cosine_sim(query_tfidf, s_tfidf), sentence))
if bests.qsize() > nquestions:
_ = bests.get()
while bests.qsize() > 0:
matches.append(bests.get()[1])
return matches[::-1]
def fetch_weather(driver):
url_prefix = "http://www.tianqihoubao.com/lishi/xian/month/%d%02d.html"
header, rows = None, []
for year in range(2014, 2019):
for month in range(1, 13):
# this is web link we are gonna fetch
url = url_prefix % (year, month)
driver.get(url)
source = driver.page_source
soup = BeautifulSoup(source, "html.parser")
print(soup.prettify(), file=open("test.txt", "w"))
table = soup.find("tbody")
all_data = table.find_all("tr")
table_header = all_data[0]
all_data = all_data[1:]
header, _rows = parse_table(table_header, all_data)
rows.extend(_rows)
output(header, rows, "weather.txt")
def makeLocalNetworks():
netcfg = config.Config.localnetscfg
if not os.path.exists(netcfg):
if not config.Installing:
util.warn("list of local networks does not exist in %s" % netcfg)
return
util.output("generating local-networks.bro ...", False)
out = open(os.path.join(config.Config.policydirsiteinstallauto, "local-networks.bro"), "w")
print >>out, "# Automatically generated. Do not edit.\n"
netcfg = config.Config.localnetscfg
if os.path.exists(netcfg):
nets = readNetworks(netcfg)
print >>out, "redef local_nets = {"
for (cidr, tag) in nets:
print >>out, "\t%s," % cidr,
if tag != "":
print >>out, "\t# %s" % tag,
print >>out
print >>out, "};\n"
util.output(" done.")
def makeConfig(path, silent=False):
manager = config.Config.manager()
if not manager:
return
if (not silent):
util.output("generating broctl-config.bro ...", False)
filename = os.path.join(path, "broctl-config.bro")
out = open(filename, "w")
print >> out, "# Automatically generated. Do not edit."
print >> out, "redef Notice::mail_dest = \"%s\";" % config.Config.mailto
print >> out, "redef Notice::mail_dest_pretty_printed = \"%s\";" % config.Config.mailalarmsto
print >> out, "redef Notice::sendmail = \"%s\";" % config.Config.sendmail
print >> out, "redef Notice::mail_subject_prefix = \"%s\";" % config.Config.mailsubjectprefix
if manager.type != "standalone":
print >> out, "@if ( Cluster::local_node_type() == Cluster::MANAGER )"
print >> out, "redef Log::default_rotation_interval = %s secs;" % config.Config.logrotationinterval
if manager.type != "standalone":
print >> out, "@endif"
if config.Config.ipv6comm:
print >> out, "redef Communication::listen_ipv6 = T ;"
else:
print >> out, "redef Communication::listen_ipv6 = F ;"
out.close()
if (not silent):
util.output(" done.")
def view(string, entities):
response = requests.get(util.config("quoteData"))
data = response.json()
quoteNumber = 111
for item in entities:
if item["entity"] == "quote" or item["entity"] == "id" or item[
"entity"] == "number":
quoteNumber = item["utteranceText"].lower().strip()
quote = data["quotes"][int(quoteNumber) - 1]
if quote is None:
return util.output(
"end", "quote_doesnt_exist",
util.translate("quote_doesnt_exist", {"ID": quoteNumber}))
# TODO: quote side_text
return util.output(
"end", "quote",
util.translate("quote", {
"ID": quoteNumber,
"author": quote["user"],
"text": quote["text"]
}))
def _checkDiskSpace():
minspace = float(config.Config.mindiskspace)
if minspace == 0.0:
return
for (node, dfs) in control.getDf(config.Config.nodes()).items():
for df in dfs:
fs = df[0]
total = float(df[1])
used = float(df[2])
avail = float(df[3])
perc = used * 100.0 / (used + avail)
key = ("disk-space-%s%s" % (node, fs.replace("/", "-"))).lower()
if perc > 100 - minspace:
try:
if float(config.Config.state[key]) > 100 - minspace:
# Already reported.
continue
except KeyError:
pass
util.output("Disk space low on %s:%s - %.1f%% used." % (node, fs, perc))
config.Config.state[key] = "%.1f" % perc
def makeLocalNetworks(path, silent=False):
netcfg = config.Config.localnetscfg
if not os.path.exists(netcfg):
util.warn("list of local networks does not exist in %s" % netcfg)
return
if (not silent):
util.output("generating local-networks.bro ...", False)
out = open(os.path.join(path, "local-networks.bro"), "w")
print >> out, "# Automatically generated. Do not edit.\n"
netcfg = config.Config.localnetscfg
if os.path.exists(netcfg):
nets = readNetworks(netcfg)
print >> out, "redef Site::local_nets = {"
for (cidr, tag) in nets:
print >> out, "\t%s," % cidr,
if tag != "":
print >> out, "\t# %s" % tag,
print >> out
print >> out, "};\n"
if (not silent):
util.output(" done.")
def cleanup(nodes, cleantmp=False):
hadError = False
util.output("cleaning up nodes ...")
result = isRunning(nodes)
running = [node for (node, on) in result if on]
notrunning = [node for (node, on) in result if not on]
results1 = execute.rmdirs([(n, n.cwd()) for n in notrunning])
results2 = execute.mkdirs([(n, n.cwd()) for n in notrunning])
if nodeFailed(results1) or nodeFailed(results2):
hadError = True
for node in notrunning:
node.clearCrashed()
for node in running:
util.output(" %s is still running, not cleaning work directory" % node.name)
if cleantmp:
results3 = execute.rmdirs([(n, config.Config.tmpdir) for n in running + notrunning])
results4 = execute.mkdirs([(n, config.Config.tmpdir) for n in running + notrunning])
if nodeFailed(results3) or nodeFailed(results4):
hadError = True
return not hadError
def makeLocalNetworks(path, silent=False):
netcfg = config.Config.localnetscfg
if not os.path.exists(netcfg):
util.warn("list of local networks does not exist in %s" % netcfg)
return
if ( not silent ):
util.output("generating local-networks.bro ...", False)
out = open(os.path.join(path, "local-networks.bro"), "w")
print >>out, "# Automatically generated. Do not edit.\n"
netcfg = config.Config.localnetscfg
if os.path.exists(netcfg):
nets = readNetworks(netcfg)
print >>out, "redef Site::local_nets = {"
for (cidr, tag) in nets:
print >>out, "\t%s," % cidr,
if tag != "":
print >>out, "\t# %s" % tag,
print >>out
print >>out, "};\n"
if ( not silent ):
util.output(" done.")
def output(tag, data):
def outputOne(tag, vals):
util.output("%-20s " % tag, nl=False)
if not error:
util.output("%-10s " % vals["kpps"], nl=False)
if "mbps" in vals:
util.output("%-10s " % vals["mbps"], nl=False)
util.output()
else:
util.output("<%s> " % error)
util.output("\n%-20s %-10s %-10s (%ds average)" %
(tag, "kpps", "mbps", interval))
util.output("-" * 30)
totals = None
for (port, error, vals) in sorted(data):
if error:
util.output(error)
continue
if str(port) != "$total":
outputOne(port, vals)
else:
totals = vals
if totals:
util.output("")
outputOne("Total", totals)
util.output("")
def fetch_pollution(driver):
# this is web link we are gonna fetch.
url = "https://www.aqistudy.cn/historydata/monthdata.php?city=%E8%A5%BF%E5%AE%89"
# those steps below are how crawler get prepared.
# we use variable "driver" to simulate a Chrome browser.
driver.get(url)
while True:
# we need some time to get data loaded.
time.sleep(1)
# now all the weather data on webpage is here
source = driver.page_source
# we use tool "BeautifulSoup" to parse data from source page
soup = BeautifulSoup(source, "html.parser")
# parse data table from webpage
table = soup.find("table")
# get table_rows from table, each row represents a record.
# we select row from the 2nd ([1:]), because first
all_data = table.find_all("tr")
if len(all_data) > 1:
break
table_header = all_data[0]
all_data = all_data[1:]
header, rows = parse_table(table_header, all_data)
output(header, rows)
def _checkDiskSpace():
minspace = float(config.Config.mindiskspace)
if minspace == 0.0:
return
for (node, dfs) in control.getDf(config.Config.nodes()).items():
for df in dfs:
fs = df[0]
total = float(df[1])
used = float(df[2])
avail = float(df[3])
perc = used * 100.0 / (used + avail)
key = "disk-space-%s%s" % (node, fs.replace("/", "-"))
if perc > 100 - minspace:
try:
if float(config.Config.state[key]) > 100 - minspace:
# Already reported.
continue
except KeyError:
pass
util.output("Disk space low on %s:%s - %.1f%% used." % (node, fs, perc))
config.Config.state[key] = "%.1f" % perc
def makeConfig(path, silent=False):
manager = config.Config.manager()
if not manager:
return
if ( not silent ):
util.output("generating broctl-config.bro ...", False)
filename = os.path.join(path, "broctl-config.bro")
out = open(filename, "w")
print >>out, "# Automatically generated. Do not edit."
print >>out, "redef Notice::mail_dest = \"%s\";" % config.Config.mailto
print >>out, "redef Notice::mail_dest_pretty_printed = \"%s\";" % config.Config.mailalarmsto
print >>out, "redef Notice::sendmail = \"%s\";" % config.Config.sendmail;
print >>out, "redef Notice::mail_subject_prefix = \"%s\";" % config.Config.mailsubjectprefix;
if manager.type != "standalone":
print >>out, "@if ( Cluster::local_node_type() == Cluster::MANAGER )"
print >>out, "redef Log::default_rotation_interval = %s secs;" % config.Config.logrotationinterval
if manager.type != "standalone":
print >>out, "@endif"
if config.Config.ipv6comm:
print >>out, "redef Communication::listen_ipv6 = T ;"
else:
print >>out, "redef Communication::listen_ipv6 = F ;"
out.close()
if ( not silent ):
util.output(" done.")
def _checkDiskSpace():
minspace = float(config.Config.mindiskspace)
if minspace == 0.0:
return
results = control.getDf(config.Config.hosts())
for (nodehost, dfs) in results:
host = nodehost.split("/")[1]
for df in dfs:
if df[0] == "FAIL":
# A failure here is normally caused by a host that is down, so
# we don't need to output the error message.
continue
fs = df[0]
perc = df[4]
key = ("disk-space-%s%s" % (host, fs.replace("/", "-"))).lower()
if perc > 100 - minspace:
if key in config.Config.state:
if float(config.Config.state[key]) > 100 - minspace:
# Already reported.
continue
util.output("Disk space low on %s:%s - %.1f%% used." % (host, fs, perc))
config.Config.state[key] = "%.1f" % perc
def install(host, src, dst):
if isLocal(host):
if not exists(host, src):
util.output("file does not exist: %s" % src)
return False
if os.path.isfile(dst):
try:
os.remove(dst)
except OSError, e:
print 'install: os.remove(%s): %s' % (dst, e.strerror)
sys.exit(1)
util.debug(1, "cp %s %s" % (src, dst))
try:
if os.path.isfile(src):
shutil.copy2(src, dst)
elif os.path.isdir(src):
shutil.copytree(src, dst)
except OSError:
# Python 2.6 has a bug where this may fail on NFS. So we just
# ignore errors.
pass
return True
def output(tag, data):
def outputOne(tag, vals):
util.output("%-20s " % tag, nl=False)
if not error:
util.output("%-10s " % vals["kpps"], nl=False)
if "mbps" in vals:
util.output("%-10s " % vals["mbps"], nl=False)
util.output()
else:
util.output("<%s> " % error)
util.output("\n%-20s %-10s %-10s (%ds average)" % (tag, "kpps", "mbps", interval))
util.output("-" * 30)
totals = None
for (port, error, vals) in sorted(data):
if error:
util.output(error)
continue
if str(port) != "$total":
outputOne(port, vals)
else:
totals = vals
if totals:
util.output("")
outputOne("Total", totals)
util.output("")
def executeCmd(nodes, cmd):
hadError = False
for (node, success, output) in execute.executeCmdsParallel([(n, cmd) for n in nodes]):
out = output and "\n> ".join(output) or ""
util.output("[%s] %s\n> %s" % (node.name, (success and " " or "error"), out))
if not success:
hadError = True
return not hadError
def get_model(corpus, num_topics, kwargs):
output(f"running num_topics: {num_topics}.")
try:
model = LdaModel(corpus, num_topics, **kwargs)
topic_ids = save_and_inference(model, corpus, num_topics)
except RuntimeError as e:
logging.error(f"PID: {os.getpid()}, num_topics: {num_topics} error")
print(e)
return model, topic_ids
def toggleAnalysis(types, enable=True):
ana = config.Config.analysis()
for t in types:
if ana.isValid(t.lower()):
ana.toggle(t.lower(), enable)
else:
util.output("unknown analysis type '%s'" % t)
def getDf(nodes):
hadError = False
dirs = (
"logdir",
"bindir",
"helperdir",
"cfgdir",
"spooldir",
"policydir",
"libdir",
"tmpdir",
"staticdir",
"scriptsdir",
)
df = {}
for node in nodes:
df[node.name] = {}
for dir in dirs:
path = config.Config.config[dir]
cmds = []
for node in nodes:
if dir == "logdir" and node.type != "manager":
# Don't need this on the workers/proxies.
continue
cmds += [(node, "df", [path])]
results = execute.runHelperParallel(cmds)
for (node, success, output) in results:
if success:
if output:
fields = output[0].split()
# Ignore NFS mounted volumes.
if fields[0].find(":") < 0:
df[node.name][fields[0]] = fields
else:
util.output("error checking disk space on node '%s': no df output" % node)
hadError = True
else:
if output:
msg = output[0]
else:
msg = "unknown failure"
util.output("error checking disk space on node '%s': %s" % (node, msg))
hadError = True
result = {}
for node in df:
result[node] = df[node].values()
return (hadError, result)
def outputOne(tag, vals):
util.output("%-20s " % tag, nl=False)
if not error:
util.output("%-10s " % vals["kpps"], nl=False)
if "mbps" in vals:
util.output("%-10s " % vals["mbps"], nl=False)
util.output()
else:
util.output("<%s> " % error)
def _updateHTTPStats():
# Create meta file.
if not os.path.exists(config.Config.statsdir):
try:
os.makedirs(config.Config.statsdir)
except OSError, err:
util.output("error creating directory: %s" % err)
return
util.warn("creating directory for stats file: %s" % config.Config.statsdir)
def outputOne(tag, vals):
util.output("%-20s " % tag, nl=False)
if not error:
util.output("%-10s " % vals["kpps"], nl=False)
if "mbps" in vals:
util.output("%-10s " % vals["mbps"], nl=False)
util.output()
else:
util.output("<%s> " % error)
def bench_function(f):
samples = util.get_sample_count()
start_time = time.time()
pi = f(samples)
end_time = time.time()
util.output(samples, pi, start_time, end_time)
def executeCmd(nodes, cmd):
for special in "|'\"":
cmd = cmd.replace(special, "\\" + special)
cmds = [(n, "run-cmd", [cmd]) for n in nodes]
for (node, success, output) in execute.runHelperParallel(cmds):
util.output("[%s] %s\n> %s" %
(node.host,
(success and " " or "error"), "\n> ".join(output)))
def _expireLogs():
i = int(config.Config.logexpireinterval)
if not i:
return
(success, output) = execute.runLocalCmd(os.path.join(config.Config.scriptsdir, "expire-logs"))
if not success:
util.output("error running expire-logs\n\n")
util.output(output)
def _expireLogs():
i = int(config.Config.logexpireinterval)
if not i:
return
(success, output) = execute.runLocalCmd(os.path.join(config.Config.scriptsdir, "expire-logs"))
if not success:
util.output("error running expire-logs\n\n")
util.output(output)
def doCron(watch):
if config.Config.cronenabled == "0":
return
if not os.path.exists(os.path.join(config.Config.scriptsdir, "broctl-config.sh")):
util.output("error: broctl-config.sh not found (try 'broctl install')")
return
config.Config.config["cron"] = "1" # Flag to indicate that we're running from cron.
if not util.lock():
return
util.bufferOutput()
if watch:
# Check whether nodes are still running an restart if neccessary.
for (node, isrunning) in control.isRunning(config.Config.nodes()):
if not isrunning and node.hasCrashed():
control.start([node])
# Check for dead hosts.
_checkHosts()
# Generate statistics.
_logStats(5)
# Check available disk space.
_checkDiskSpace()
# Expire old log files.
_expireLogs()
# Update the HTTP stats directory.
_updateHTTPStats()
# Run external command if we have one.
if config.Config.croncmd:
(success, output) = execute.runLocalCmd(config.Config.croncmd)
if not success:
util.output("error running croncmd: %s" % config.Config.croncmd)
# Mail potential output.
output = util.getBufferedOutput()
if output:
util.sendMail("cron: " + output.split("\n")[0], output)
util.unlock()
config.Config.config["cron"] = "0"
util.debug(1, "cron done")
def _getProfLogs():
cmds = []
for node in config.Config.hosts():
if not execute.isAlive(node.addr):
continue
cmd = os.path.join(config.Config.scriptsdir, "get-prof-log") + " %s %s %s/prof.log" % (node.name, node.host, node.cwd())
cmds += [(node, cmd, [], None)]
for (node, success, output) in execute.runLocalCmdsParallel(cmds):
if not success:
util.output("cannot get prof.log from %s" % node.name)
def _checkHosts():
for node in config.Config.hosts():
tag = "alive-%s" % node.host
alive = execute.isAlive(node.addr) and "1" or "0"
if tag in config.Config.state:
previous = config.Config.state[tag]
if alive != previous:
util.output("host %s %s" % (node.host, alive == "1" and "up" or "down"))
config.Config._setState(tag, alive)
def restart(nodes, clean):
if len(nodes) > 0:
all_nodes = nodes
else:
all_nodes = config.Config.nodes()
util.output("stopping ...")
if len(nodes) > 0:
# User picked nodes to restart.
_stopNodes(nodes)
else:
stop([])
if clean:
# Can't delete the tmp here because log archival might still be going on there in the background.
cleanup(all_nodes, False)
util.output("checking configuration ...")
if not checkConfigs(all_nodes):
return
util.output("installing ...")
install.install(False, False)
util.output("starting ...")
if len(nodes) > 0:
_startNodes(nodes)
else:
start([])
def attachGdb(nodes):
running = isRunning(nodes)
cmds = []
for (node, isrunning) in running:
if isrunning:
cmds += [(node, "gdb-attach", ["gdb-%s" % node.name, config.Config.bro, str(node.getPID())])]
results = execute.runHelperParallel(cmds)
for (node, success, output) in results:
if success:
util.output("gdb attached on %s" % node.name)
else:
util.output("cannot attach gdb on %s: %s" % node.name, output)
def df(nodes):
util.output("%10s %15s %-5s %-5s %-5s" % ("", "", "total", "avail", "capacity"))
for (node, dfs) in getDf(nodes).items():
for df in dfs:
total = float(df[1])
used = float(df[2])
avail = float(df[3])
perc = used * 100.0 / (used + avail)
util.output("%10s %15s %-5s %-5s %-5.1f%%" % (node, df[0],
prettyPrintVal(total),
prettyPrintVal(avail), perc))
def scaleTentativeImageToDimensionsOfReference1(self, imgReference,
imgTentative):
print imgReference.shape
pass
heightReference, widthReference, _ = imgReference.shape
heightTentative, widthTentative, _ = imgTentative.shape
if ((heightReference == heightTentative)
and (widthReference == widthTentative)):
return imgTentative
imgResized = cv2.resize(imgTentative,
(widthReference, heightReference))
if (self.traced):
util.output('Resized to: ' + str(imgResized.shape[1]) + '/' +
str(imgResized.shape[0]))
return imgResized
def install(host, src, dst):
if isLocal(host):
if not exists(host, src):
util.output("file does not exist: %s" % src)
return False
if os.path.isfile(dst):
os.remove(dst)
util.debug(1, "cp %s %s" % (src, dst))
shutil.copy2(src, dst)
return True
else:
util.error("install() not yet supported for remote hosts")
return False
def install(host, src, dst):
if isLocal(host):
if not exists(host, src):
util.output("file does not exist: %s" % src)
return False
if os.path.isfile(dst):
os.remove(dst)
util.debug(1, "cp %s %s" % (src, dst))
shutil.copy2(src, dst)
return True
else:
util.error("install() not yet supported for remote hosts")
return False
def df(nodes):
util.output("%10s %15s %-5s %-5s %-5s" %
("", "", "total", "avail", "capacity"))
for (node, dfs) in getDf(nodes).items():
for df in dfs:
total = float(df[1])
used = float(df[2])
avail = float(df[3])
perc = used * 100.0 / (used + avail)
util.output("%10s %15s %-5s %-5s %-5.1f%%" %
(node, df[0], prettyPrintVal(total),
prettyPrintVal(avail), perc))
def parseNodes(self, names):
"""Returns `Node`_ objects for a string of space-separated node names.
If a name does not correspond to a known node, an error message is
printed and the node is skipped from the returned list. If no names
are known, an empty list is returned."""
nodes = []
for arg in names.split():
h = config.Config.nodes(arg, True)
if not h:
util.output("unknown node '%s'" % arg)
else:
nodes += [h]
return nodes
def save_and_inference(model: LdaModel, corpus, num_topics, chunksize=0):
path = f"./dev/model/{_ARGS.name}_{num_topics}.pkl"
try:
model.save(path)
output(f"model saved at <{path}>")
if chunksize > 0:
gammas = [model.inference(chunk)[0] for chunk in grouper(corpus, chunksize)]
gamma = concatenate(gammas)
else:
gamma, _ = model.inference(corpus)
except RuntimeError as e:
logging.error(f"PID: {os.getpid()}, num_topics: {num_topics} error")
print(e)
output(f"num_topics {num_topics} inference compete.")
return gamma.argmax(axis=1)
def outputOne(tag, vals):
util.output("%-21s " % tag, nl=False)
if not error:
util.output("%-10s " % vals.get("kpps", ""), nl=False)
util.output("%s" % vals.get("mbps", ""))
else:
util.output("<%s>" % error)
def _getProfLogs():
cmds = []
for node in config.Config.hosts():
if not execute.isAlive(node.addr):
continue
cmd = os.path.join(config.Config.scriptsdir,
"get-prof-log") + " %s %s %s/prof.log" % (
node.name, node.host, node.cwd())
cmds += [(node, cmd, [], None)]
for (node, success, output) in execute.runLocalCmdsParallel(cmds):
if not success:
util.output("cannot get prof.log from %s" % node.name)
def _makeCrashReports(nodes):
cmds = []
for node in nodes:
cmds += [(node, "run-cmd", [
os.path.join(config.Config.scriptsdir, "post-terminate"),
node.cwd(), "crash"
])]
for (node, success, output) in execute.runHelperParallel(cmds):
if not success:
util.output("cannot run post-terminate for %s" % node.tag)
else:
util.sendMail("Crash report from %s" % node.tag, "\n".join(output))
node.clearCrashed()
def parseNodes(self, names):
"""Returns `Node`_ objects for a string of space-separated node names.
If a name does not correspond to a known node, an error message is
printed and the node is skipped from the returned list. If no names
are known, an empty list is returned."""
nodes = []
for arg in names.split():
h = config.Config.nodes(arg, True)
if not h:
util.output("unknown node '%s'" % arg)
else:
nodes += [h]
return nodes
def attachGdb(nodes):
running = isRunning(nodes)
cmds = []
for (node, isrunning) in running:
if isrunning:
cmds += [(node, "gdb-attach",
["gdb-%s" % node.tag, config.Config.bro,
node.getPID()])]
results = execute.runHelperParallel(cmds)
for (node, success, output) in results:
if success:
util.output("gdb attached on %s" % node.tag)
else:
util.output("cannot attach gdb on %s: %s" % node.tag, output)
def _makeCrashReports(nodes):
for n in nodes:
plugin.Registry.broProcessDied(n)
cmds = []
for node in nodes:
cmds += [(node, "run-cmd", [os.path.join(config.Config.scriptsdir, "post-terminate"), node.cwd(), "crash"])]
for (node, success, output) in execute.runHelperParallel(cmds):
if not success:
util.output("cannot run post-terminate for %s" % node.name)
else:
util.sendMail("Crash report from %s" % node.name, "\n".join(output))
node.clearCrashed()
def cleanWithExpectedCount(self, img, id, target):
if(self.writeOp):
util.show('orig',img)
imgf = self.prepare(img, 'orig');
if(self.writeOp):
util.show('prepared',imgf)
num_points = 0;
target_value = int(target*1.1)
nuova_soglia = 256
while((num_points<target_value) and (nuova_soglia>=0) ):
nuova_soglia = nuova_soglia -1 ;
num_points = self.calculatePoints(imgf, nuova_soglia)
if(self.writeOp):
util.output("Test for threshold "+ str(nuova_soglia)+" Points #: "+ str(num_points));
self.soglia = nuova_soglia+1;
return self.applyThreshold(imgf, self.soglia)
def extract(self): """ 提取有效数据 """ print '\n\t in extract' files = util.get_dir_list(self.config['requests_dir']) output_path = self.config['dump_dir'] util.check_path(output_path) files.sort(key=lambda x:int(x[:-4])) content = "" for file in files: path = self.config['requests_dir'] + '/' + file f = open(path) text = f.readlines() f.close() print path content += self._filter(text) util.output(output_path, 'total', content)
def extract(self):
""" 提取有效数据 """
print '\n\t in extract'
files = util.get_dir_list(self.config['requests_dir'])
output_path = self.config['dump_dir']
util.check_path(output_path)
files.sort(key=lambda x: int(x[:-4]))
content = ""
for file in files:
path = self.config['requests_dir'] + '/' + file
f = open(path)
text = f.readlines()
f.close()
print path
content += self._filter(text)
util.output(output_path, 'total', content)
def getWeather(entities, owm):
for item in entities:
if item["entity"] == "city":
util.output(
"intermediate", "fetching",
util.translate("fetching", {"city": item["sourceText"]}))
city = item["sourceText"]
report = owm.weather_manager().weather_at_place(city)
weather = report.weather
val = {"city": city, "report": report, "weather": weather}
return val
return util.output("end", "error", util.translate("error"))
def _checkHosts():
for node in config.Config.hosts():
if execute.isLocal(node):
continue
tag = "alive-%s" % node.host.lower()
alive = execute.isAlive(node.addr) and "1" or "0"
if tag in config.Config.state:
previous = config.Config.state[tag]
if alive != previous:
plugin.Registry.hostStatusChanged(node.host, alive == "1")
util.output("host %s %s" % (node.host, alive == "1" and "up" or "down"))
config.Config._setState(tag, alive)
def get_data(self):
""" 抓取数据
如果响应成功：检查是否需要继续请求，存储response文本
如果响应失败：次数超过10次，直接退出
Note：返回false不一定就没有更多，可能服务器繁忙
"""
fail = 0 # 请求失败次数
index = 0 # 已经加载的数据个数
page = 1 # 请求次数
more = True # 循环条件，初始为 True
header = {
'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1'
}
if not self.config.has_key('cookie'):
self.config['cookie'] = ""
header['Cookie'] = self.config['cookie']
while more:
print '\n no.' + str(page)
params = self._get_params(index, page)
if self.config.has_key('type') and self.config['type'] == 'post':
url = self.config['base_url']
response = requests.post(url, data=params, headers=header)
else:
url = self.config['base_url'] + params
response = requests.get(url, headers=header)
print '\t\t status: ' + str(response.status_code)
if response.status_code == 200:
util.output(self.config['requests_dir'], page, response.text)
more = self._check_more(response.text)
print '\t\t\t has_more: ' + str(more)
if more is False and fail < 10:
more = True
fail += 1
print '\n\tfail: ' + str(fail) + '\ttry again'
else:
fail = 0
page += 1
index += 10
else:
if fail > 10:
break
fail += 1
print '\n\t\t something wrong, fail ' + str(fail) + '\n\n'
print '\n'
def _makeCrashReports(nodes):
for n in nodes:
plugin.Registry.broProcessDied(n)
msg = "If you want to help us debug this problem, then please forward\nthis mail to [email protected]\n"
cmds = []
for node in nodes:
cmds += [(node, "run-cmd", [os.path.join(config.Config.scriptsdir, "post-terminate"), node.cwd(), "crash"])]
for (node, success, output) in execute.runHelperParallel(cmds):
if not success:
util.output("cannot run post-terminate for %s" % node.name)
else:
util.sendMail("Crash report from %s" % node.name, msg + "\n".join(output))
node.clearCrashed()
def cleanup(nodes, cleantmp=False):
util.output("cleaning up nodes ...")
result = isRunning(nodes)
running = [node for (node, on) in result if on]
notrunning = [node for (node, on) in result if not on]
execute.rmdirs([(n, n.cwd()) for n in notrunning])
execute.mkdirs([(n, n.cwd()) for n in notrunning])
for node in notrunning:
node.clearCrashed();
for node in running:
util.output(" %s is still running, not cleaning work directory" % node.name)
if cleantmp:
execute.rmdirs([(n, config.Config.tmpdir) for n in running + notrunning])
execute.mkdirs([(n, config.Config.tmpdir) for n in running + notrunning])
def _updateHTTPStats():
# Get the prof.logs.
# FIXME: Disabled for now. This currently copies the complete prof.log
# each time. As these can get huge, that can take a while. We should
# change that to only copy the most recent chunk and then also expire old
# prof logs on the manager.
# _getProfLogs()
# Create meta file.
if not os.path.exists(config.Config.statsdir):
try:
os.makedirs(config.Config.statsdir)
except OSError, err:
util.output("error creating directory: %s" % err)
return
util.warn("creating directory for stats file: %s" % config.Config.statsdir)
def makeLayout():
def nextPort(node):
global port
port += 1
node.setPort(port)
return port
global port
port = 47759
manager = config.Config.manager()
if not manager:
return
util.output("generating broctl-layout.bro ...", False)
out = open(os.path.join(config.Config.policydirsiteinstallauto, "broctl-layout.bro"), "w")
print >>out, "# Automatically generated. Do not edit.\n"
print >>out, "redef BroCtl::manager = [$ip = %s, $p=%s/tcp, $tag=\"%s\"];\n" % (manager.addr, nextPort(manager), manager.tag);
proxies = config.Config.nodes("proxy")
print >>out, "redef BroCtl::proxies = {"
for p in proxies:
tag = p.tag.replace("proxy-", "p")
print >>out, "\t[%d] = [$ip = %s, $p=%s/tcp, $tag=\"%s\"]," % (p.count, p.addr, nextPort(p), tag)
print >>out, "};\n"
workers = config.Config.nodes("worker")
print >>out, "redef BroCtl::workers = {"
for s in workers:
tag = s.tag.replace("worker-", "w")
p = s.count % len(proxies) + 1
print >>out, "\t[%d] = [$ip = %s, $p=%s/tcp, $tag=\"%s\", $interface=\"%s\", $proxy=BroCtl::proxies[%d]]," % (s.count, s.addr, nextPort(s), tag, s.interface, p)
print >>out, "};\n"
print >>out, "redef BroCtl::log_dir = \"%s\";\n" % config.Config.subst(config.Config.logdir, make_dest=False)
# Activate time-machine support if configured.
if config.Config.timemachinehost:
print >>out, "redef BroCtl::tm_host = %s;\n" % config.Config.timemachinehost
print >>out, "redef BroCtl::tm_port = %s;\n" % config.Config.timemachineport
print >>out
util.output(" done.")