def callback(self, msg):
'''
:param msg:
:return:
'''
# TODO 识别访问src ip是否为外网地址， 若为外网地址， 则统一归属为netmapcloud
try:
access_data = {
"uuid": msg.get("id"),
"src": msg.get("src"),
"dest": msg.get("dest"),
"port": msg.get("port")
}
self.mongoClient.insert(access_data)
ip_data = IpaddressApi().get(primiry_id=msg.get("src"))
if ip_data:
HostGraphApi().create_relation(
primary_key1=msg.get("id"),
primary_key2=ip_data.get("host_id"),
re_type="access",
port=msg.get("port"),
access="%s_%s" % (msg.get("src"), msg.get("dest")))
except:
logger.info(traceback.format_exc())
logger.info("Access Drawer failed: %s" % to_str(msg))
def report_local(cls, uuid, ip):
data = {"host_id": uuid, "ip": ip}
request = urllib2.Request("http://%s" % api_service +
localipaddress_url)
request.add_header('content-type', 'application/json')
response = urllib2.urlopen(request, urllib.urlencode(data), timeout=5)
logger.info(response.read())
if response.getcode() == 201:
return True
return False
def unregister(uuid):
status = __check_register(uuid)
if not status:
print("主机 %s 未注册" % uuid)
request = urllib2.Request("http://%s" % api_service + register_url +
"/%s" % uuid)
request.get_method = lambda: 'DELETE'
request.add_header('content-type', 'application/json')
response = urllib2.urlopen(request)
logger.info(response.read())
if response.getcode() == 200:
__remove_register()
def __check_register(uuid):
if os.path.exists("/usr/local/netmap/.register.info"):
with open("/usr/local/netmap/.register.info", "r") as f:
lines = f.readlines()
_uuid = lines[0]
_uuid = _uuid.strip("\n")
if _uuid == uuid:
return True
else:
logger.info("OLD ID: %s, NEW ID:%s, clear cache info" %
(_uuid, uuid))
__remove_register()
return False
def register(uuid):
status = __check_register(uuid)
if status:
print("主机已注册，注册信息： %s" % uuid)
sys.exit(0)
request = urllib2.Request("http://%s" % api_service + register_url)
request.add_header('content-type', 'application/json')
data = {"id": uuid}
response = urllib2.urlopen(request, urllib.urlencode(data))
logger.info(response.read())
if response.getcode() == 201:
__write_register(uuid)
print("注册成功 %s" % uuid)
print("注册成功 %s" % uuid)
def report_access(cls, uuid, request_data):
data = {
"id": uuid,
"src": request_data[1],
"dest": request_data[2],
"port": request_data[5],
"url": request_data[-1]
}
request = urllib2.Request("http://%s" % api_service + access_url)
request.add_header('content-type', 'application/json')
response = urllib2.urlopen(request, urllib.urlencode(data), timeout=5)
logger.info(response.read())
if response.getcode() == 201:
return True
return False
def converting_file_into_chunks(self, df, chunk_size=100):
"""
Large pyspark dataframe is converted into chunks and converted to pandas to
convert it into pandas df
:param df: orig dataframe
:param chunk_size: size of each dataframe chunk
:return:
"""
# created window using first column
window = Window.orderBy(funct.col(df.columns[0]))
df = df.withColumn('row_number', funct.row_number().over(window))
for i in range(0, df.count(), chunk_size):
chunk = df.where((funct.col('row_number') >= i) & (funct.col('row_number') < (i + chunk_size)))
logger.info(chunk.count())
if chunk.count() != 0:
pd_df = chunk.toPandas()
### you can do what ever you want to with the pandas ddataframe
pd_df.to_csv("{}_file.csv".format(i))
print("############")
def on_patch(self, req, resp, **kwargs):
'''
Handles patch requests
:param req:
:param resp:
:param kwargs:
:return: 返回json
'''
self.url = req.path
self._validate_method(req)
self._validate_header(req)
data = self._fetch_data(req)
self.trace_req_log(req, data)
result = {}
try:
self._before_update(req, data=data, **kwargs)
num, update_data = self.update(req=req,
data=data,
resp=resp,
**kwargs)
if update_data:
result = {"num": num, "id": update_data}
resp.status = falcon.HTTP_200
else:
resp.status = falcon.HTTP_404
except (ValueError, TypeError, IndexError, UnicodeError) as e:
logger.info(traceback.format_exc())
resp.status = falcon.HTTP_400
result = {
"title": e.__class__.__name__,
"description": e.args[0],
"code": 400
}
except (ResourceNotFound) as e:
resp.status = falcon.HTTP_404
except (MethodNotAllowed) as e:
logger.info(traceback.format_exc())
resp.status = falcon.HTTP_405
result = {
"title": "method error",
"description": "method not allowed",
"code": 405
}
except Exception as e:
logger.info(traceback.format_exc())
resp.status = falcon.HTTP_500
result = {
"title": "service error",
"description": "服务器异常",
"code": 500
}
finally:
resp.body = to_str(result)
resp.set_headers(header)
resp.set_header("ReqId", self.req_id)
self.trace_resp_log(req, result=result, resp=resp)
def trace_resp_log(self, req, result, resp, **kwargs):
status = resp.status or 000
msg = "[ %s ][URL:%s] - [METHOD: %s] [%s] - [RESULT: %s]" % (
self.req_id, req.path, req.method.lower(), status,
to_str(result)[:LOG_MSG_MAX_LEN])
logger.info(msg)
def trace_req_log(self, req, data, **kwargs):
self.req_id = "req-%s" % (allocate_uuid())
msg = "[ %s ][URL:%s] - [METHOD: %s] [000] - [DATA: %s]" % (
self.req_id, req.path, req.method.lower(),
to_str(data)[:LOG_MSG_MAX_LEN])
logger.info(msg)
def on_get(self, req, resp, **kwargs):
'''
Handles get requests
:param req:
:param resp:
:param kwargs:
:return: 返回json
'''
self.url = req.path
self._validate_method(req)
self._validate_header(req)
data = self._fetch_data(req)
self.trace_req_log(req, data)
result = {}
try:
if kwargs:
self._before_detail(req, data, **kwargs)
result = self.detail(req, data, resp, **kwargs)
else:
order_by = data.pop("order_by", None)
if order_by:
if order_by.endswith("__-1"):
order_by = "-%s" % order_by.strip("__-1")
if order_by.endswith("__1"):
order_by = order_by.strip("__1")
limit = data.pop("limit", None)
offset = data.pop("offset", None)
self._before_list(req, data=data)
count, _data = self.list(req,
data,
resp,
offset=offset,
limit=limit,
order_by=order_by)
result = {"count": count, "data": _data}
if result:
resp.status = falcon.HTTP_200
else:
resp.status = falcon.HTTP_404
except (ValueError, TypeError, IndexError, UnicodeError) as e:
logger.info(traceback.format_exc())
resp.status = falcon.HTTP_400
result = {
"title": e.__class__.__name__,
"description": e.args[0],
"code": 400
}
except (ResourceNotFound) as e:
resp.status = falcon.HTTP_404
except (MethodNotAllowed) as e:
logger.info(traceback.format_exc())
resp.status = falcon.HTTP_405
result = {
"title": "method error",
"description": "method not allowed",
"code": 405
}
except Exception as e:
logger.info(traceback.format_exc())
resp.status = falcon.HTTP_500
result = {
"title": "service error",
"description": "服务器异常",
"code": 500
}
finally:
resp.body = to_str(result)
resp.set_headers(header)
resp.set_header("ReqId", self.req_id)
self.trace_resp_log(req, result=result, resp=resp)
# coding: utf-8
from wsgiref import simple_server
from lib.logs import logger
from core.conf import PORT
from core.conf import IPPADDRESS
from core.base import APP
if __name__ == '__main__':
print("Starting ...")
httpd = simple_server.make_server(IPPADDRESS, PORT, APP)
logger.info("service listen at %s:%s" % (IPPADDRESS, PORT))
httpd.serve_forever()