def init_phoenix_jdbc(self, drop=False):
log("Initializing Phoenix", 2)
self.p_conn = jaydebeapi.connect(
"org.apache.phoenix.jdbc.PhoenixDriver",
["jdbc:phoenix:localhost"], self.PHOENIX_JAR_PATH)
log("Finished initializing Phoenix", 2)
def create_results_table(self):
# Even though we're making this multi-tenant, we still keep the
# various customers seperated by namespace. It's for security
# reasons that we use multi-tenant as it isolates the current
# tenant when querying from the web frontend (using the TenantID
# JDBC param
# TODO: figure out where we put this
fh = open(
"/Users/mark/Documents/SCM/metly/data/create_results_table.sql",
"r")
sql_lines = []
for line in fh:
line = line.strip()
if line.startswith("#") or len(line) == 0:
continue
sql_lines.append(line)
sql = "\n".join(sql_lines)
log("Create SQL:\n%s" % sql, 5)
cursor = self.p_conn.cursor()
cursor.execute(sql)
def reload_cache(self):
session = db.Session()
self.log_sources = {}
try:
for log_source in session.query(LogSource).all():
log(2, "Initialising log source:")
log(2, "Customer: %s, Source Name: %s, Format: %s" % \
(log_source.customer.name, log_source.name, \
log_source.parameters["format"].value))
self.log_sources[log_source.id] = log_source
self.parsers[log_source.id] = self.new_parser(log_source)
finally:
session.close()
def init_defaults(self, drop):
log("Initialising defaults", 3)
if drop == True:
log("Dropping database", 3)
Base.metadata.drop_all(db.engine)
Base.metadata.create_all(db.engine)
session = db.Session()
# make sure all our constants exist etc
if session.query(Requestor).first() != None:
log("Defaults already initialised", 3)
session.close()
return
session.add(LogSourceType(0, "syslog", "Syslog"))
session.add(LogSourceType(1, "filesystem", "Filesystem Path"))
session.add(Requestor("customer"))
session.add(Requestor("AFP (Australian Federal Police)"))
session.add(Requestor("NSW Police"))
session.add(Requestor("Victorian Police"))
session.add(Requestor("ACC (Australian Crime Comission)"))
session.add(Requestor("ASIO"))
session.add(Requestor("DSD"))
customer = Customer("test", "test")
network = Network()
network.name = "Default"
network.description = "Default"
session.add(network)
customer.networks.append(network)
network = Network()
network.name = "Intranet"
network.description = "Intranet"
session.add(network)
customer.networks.append(network)
session.add(customer)
session.add(Customer("internode", "internode"))
session.add(Customer("telstra", "telstra"))
user = User()
user.username = "******"
user.firstname = "Mark"
user.lastname = "Knowles"
user.email = "*****@*****.**"
user.su = True
user.customer = customer
user.enabled = True
user.salt = hash.new_salt()
user.password = hash.hash_password("test", user.salt)
session.add(user)
session.commit()
session.close()
log("Done initialising defaults", 3)
def main(self):
self.args = Args()
self.parse_arguments()
if os.path.exists(self.args.config_path) == False:
log(0, "Couldn't find config file:", self.args.config_path)
sys.exit(1)
self.config = ConfigParser.RawConfigParser()
self.config.read(self.args.config_path)
self.cert_path = self.config.get("main", "cert_path")
self.key_path = self.config.get("main", "key_path")
server_pki_path = self.config.get("server", "pki_path")
server_rpc_host = self.config.get("server", "host")
server_rpc_port = self.config.get("server", "port")
self.listen_address = self.config.get("listen", "address")
self.listen_port = int(self.config.get("listen", "port"))
db_uri = self.config.get("database", "uri")
self.open_db(db_uri)
self.server_rpc = ServerRPC(server_rpc_host, server_rpc_port, \
server_pki_path)
try:
while True:
try:
self.server_rpc.connect()
break
except urllib3.exceptions.MaxRetryError:
log(0, "Couldn't connect to Metly Daemon, retrying")
time.sleep(10)
except NotAuthorizedException, ex:
self.server_rpc.register()
sys.stderr.write(("This webserver isn't authorized with Metly" \
"Daemon. UUID: %s\n") % ex.uuid)
sys.exit(1)
def do_POST(self):
try:
content_type_header = self.headers.getheader("content-type")
content_type, post_dict = cgi.parse_header(content_type_header)
length = int(self.headers.getheader("content-length"))
if content_type == "multipart/form-data":
environ = {"REQUEST_METHOD":"POST", \
"CONTENT_TYPE":self.headers["Content-Type"]}
fs = cgi.FieldStorage(fp=self.rfile, headers=self.headers, \
environ=environ)
self.rpc.post_binary_part(self, fs)
return
if content_type != "application/json":
self.wfile.write("Expected JSON data")
# TODO: do we need a timeout
post_body = self.rfile.read(length)
post_body = "".join(post_body)
response = self.rpc.render_json(post_body, self)
self.send_headers(200)
self.end_headers()
#TODO: check that this actually writes all the bytes
if response != None:
self.wfile.write(response)
except NotAuthorizedException, ex:
log("Unauthorized connection from ... %s:%d" % self.client_address,
0)
self.send_headers(403)
self.send_header("UUID", ex.uuid)
self.end_headers()
def refresh_authorized_web_servers(self):
log("Populating authorized web_server uuids, uuids follow", 5)
try:
session = db.Session()
web_servers = session.query(WebServer)\
.filter(WebServer.authorized==True).all()
for web_server in web_servers:
self.authorized_web_servers[web_server.uuid] = web_server
log(web_server.uuid, 5)
log("EOL", 5)
finally:
session.close()
def insert_event(self, event, customer_uuid, customer_short_name):
# VERY IMPORTANT: jpype needs to be initialized for threads.
# If we don't do this, we start getting mystery segfaults
if type(self.p_conn) != phoenixdb.connection.Connection:
if not jpype.isThreadAttachedToJVM():
log("Configured jpype for threading", 6)
jpype.attachThreadToJVM()
cursor = self.p_conn.cursor()
if re.search("^\w+$", customer_short_name) == None:
raise Exception(( \
"Invalid customer short name: %s. " \
"Avoiding SQL injection") % customer_short_name)
# sql = ( "UPSERT INTO %s_events(id, tenant_id, raw, receiptTime, " \
# "startTime, device_id, collector_id) VALUES ( " \
# "NEXT VALUE FOR %s_events_sequence, ?, ?, ?, ?, ?, ?)") % \
# (customer_short_name, customer_short_name)
# params = (customer_uuid, event.raw, event.create_dt, event.create_dt, \
# event.device_id, event.collector_id)
sql = ( "UPSERT INTO %s_events(id, tenant_id, raw, startTime, " \
"device_id, collector_id) VALUES ( " \
"NEXT VALUE FOR %s_events_sequence, ?, ?, to_date(?, 'yyyy-mm-dd'), ?, ?)") % \
(customer_short_name, customer_short_name)
params = (customer_uuid, event.raw, str(event.create_dt.date()), \
event.device_id, event.collector_id)
log("Tables exist, logging normally", 5)
cursor.execute(sql, params)
self.p_conn.commit()
log("Done", 5)
def init(self, drop=False, drop_phoenix=False):
log("Starting db.init", 2)
self.init_defaults(drop)
self.init_phoenix_jsonrpc()
self.init_tables(drop_phoenix)
log("Finished db.init", 2)
def init(self, drop=False, drop_phoenix=False):
log("Starting db.init", 2)
self.init_defaults(drop)
log("Finished db.init", 2)
class MetlyWeb(object):
DEFAULT_CONFIG_PATH = "/etc/metly/web.conf"
DEFAULT_WEB_SUBJECT = ("/C=AU/ST=New South Wales/L=Sydney/O=Metly/"
"OU=MetlyWeb Self Signed/CN=localhost/"
"[email protected]")
def __init__(self):
self.cert_path = None
self.key_path = None
self.listen_address = None
self.listen_port = None
self.auto_cert = True
def main(self):
self.args = Args()
self.parse_arguments()
if os.path.exists(self.args.config_path) == False:
log(0, "Couldn't find config file:", self.args.config_path)
sys.exit(1)
self.config = ConfigParser.RawConfigParser()
self.config.read(self.args.config_path)
self.cert_path = self.config.get("main", "cert_path")
self.key_path = self.config.get("main", "key_path")
server_pki_path = self.config.get("server", "pki_path")
server_rpc_host = self.config.get("server", "host")
server_rpc_port = self.config.get("server", "port")
self.listen_address = self.config.get("listen", "address")
self.listen_port = int(self.config.get("listen", "port"))
db_uri = self.config.get("database", "uri")
self.open_db(db_uri)
self.server_rpc = ServerRPC(server_rpc_host, server_rpc_port, \
server_pki_path)
try:
while True:
try:
self.server_rpc.connect()
break
except urllib3.exceptions.MaxRetryError:
log(0, "Couldn't connect to Metly Daemon, retrying")
time.sleep(10)
except NotAuthorizedException, ex:
self.server_rpc.register()
sys.stderr.write(("This webserver isn't authorized with Metly" \
"Daemon. UUID: %s\n") % ex.uuid)
sys.exit(1)
try:
self.auto_cert = self.config.get("main", "auto_cert")
self.auto_cert = self.auto_cert.upper() == "TRUE"
except ConfigParser.NoOptionError:
pass
# check to see if our server certificate exists
if os.path.exists(self.cert_path) == False or \
os.path.exists(self.key_path) == False:
if self.auto_cert == True:
# create the keypair automatically
CertUtil.create_self_signed_cert(self.DEFAULT_WEB_SUBJECT, \
self.key_path, self.cert_path)
else:
log(0, "Server cert or key cert doesn't exist:", \
self.cert_path, self.key_path)
sys.exit(1)
self.start_ui()