class YcyReplier(object):
def __init__(self):
self.log = Logger()
'''训练QA样本，获得QA映射表'''
csv_file = csv.reader(open(RESOURCES_PATH, 'r', encoding='UTF-8'))
for QA in csv_file:
tags = jieba.analyse.extract_tags(QA[0], topK=3)
key = ''
for tag in tags:
key += tag
if (len(key)):
QA_dict[key] = QA[1]
else:
QA_dict[QA[0]] = QA[1]
self.log.info("Dict:{}".format(QA_dict))
def reply_text(self, msg):
tags = jieba.analyse.extract_tags(msg, topK=3)
key = ''
for tag in tags:
key += tag
self.log.info("KeyWords:{}".format(key))
if (len(key)):
if key in QA_dict:
return (QA_dict[key])
else:
return
else:
if msg in QA_dict:
return (QA_dict[msg])
else:
return
def get_gsv_gvi_list_by_way_id(
log: Logger, edge_gdf: GeoDataFrame,
gsv_gvi_gdf: GeoDataFrame) -> Dict[int, List[float]]:
"""Returns a dictionary of lists of GSV point GVI values by edge id.
Only point GVI values within 30m from edge geometry are included in the lists.
"""
edges = edge_gdf[[E.id_way.name, 'geometry']].copy()
gvi_points = gsv_gvi_gdf[['geometry', 'GVI']].copy()
edges['geom_b_30'] = [geom.buffer(30) for geom in edges['geometry']]
edges = edges.set_geometry('geom_b_30')
edge_gvi_points = gpd.sjoin(edges,
gvi_points,
how='inner',
op='intersects')
gvi_points_by_way_id = edge_gvi_points.groupby(E.id_way.name)
gvi_list_by_way_id = {}
for way_id, g_points in gvi_points_by_way_id:
gvi_list_by_way_id[way_id] = list(g_points['GVI'])
log.info(
f'Found GVI point samples for {sample_ratio(len(edges), len(gvi_list_by_way_id))} % edges'
)
return gvi_list_by_way_id
def sjoin_noise_values(gdf,
noise_layers: dict,
log: Logger = None) -> gpd.GeoDataFrame:
sample_gdf = gdf.copy()
sample_gdf['sample_idx'] = sample_gdf.index
for name, noise_gdf in noise_layers.items():
log.debug(f'joining noise layer [{name}] to sampling points')
sample_gdf = gpd.sjoin(sample_gdf, noise_gdf, how='left',
op='within').drop(['index_right'], axis=1)
if (len(sample_gdf.index) > len(gdf.index)):
log.warning(
f'joined multiple noise values for one or more sampling points ({len(sample_gdf.index)} != {len(gdf.index)})'
)
distinct_samples = remove_duplicate_samples(sample_gdf, 'sample_idx',
noise_layers)
if (len(distinct_samples.index) == len(gdf.index)):
log.info('successfully removed duplicate samples')
else:
log.error('error in removing duplicate samples')
if (list(sample_gdf.columns).sort() != list(
distinct_samples.columns).sort()):
log.error(
'schema of the dataframe was altered during removing duplicate samples'
)
return distinct_samples.drop(columns=['sample_idx'])
def save_ckpt(cfg, state):
#ckpt_best_method_year_month_day_time
ckpt_name = cfg.get_project_name() + '_' + cfg.get_method(
) + '_' + cfg.get_start_time() + '.pth.tar'
file_dir = os.path.join(cfg.get_backup_path(), ckpt_name)
torch.save(state, file_dir)
log.info('Saved at %s' % file_dir)
class Context:
"""
上下文关联参数读写
"""
def __init__(self):
self._cg = configparser.ConfigParser()
self._cg.read(constant.context_data_path.encode('utf-8'))
self._log = Logger('关联参数').get_logger()
def get(self, option):
"""
获取参数值
:param option: key
:return: value
"""
if option in self._cg.options(section='DATA'):
value = self._cg.get('DATA', option=option)
self._log.info('成功获取关联参数：{} = {}'.format(option, value))
return value
else:
self._log.error('不存在关联参数：{}'.format(option))
def write(self, option, value):
"""
把传入的数据写入到文件
:param option: key
:param value: value
:return: None
"""
self._cg.set('DATA', option, value)
with open(constant.context_data_path, 'w') as fp:
self._cg.write(fp)
self._log.info('成功写回关联参数：{} = {}'.format(option, value))
def __init__(self):
"""
Manager server connection object
"""
self.server = settings.config["tool"]["address"].encode("utf8")
self.port = settings.config["tool"]["port"]
self.token = settings.config["tool"]["token"].encode("utf8")
self.uri_base = "/rest_api/v2"
# Manager server connection object
if settings.config["tool"]["proxy_connect"] == "1":
self.manager_server = NetworkConnection(
settings.config["tool"]["address"],
settings.config["tool"]["port"],
is_https=settings.config["tool"]["using_https"],
proxies=settings.config["tool"]["http_proxy"])
else:
self.manager_server = NetworkConnection(
settings.config["tool"]["address"],
settings.config["tool"]["port"],
is_https=settings.config["tool"]["using_https"])
self.headers = {
"token": self.token,
"Content-Type": "application/json"
}
if self.check_service() is True:
self.client = MsfRpcClient(self,
server=self.server,
port=self.port,
token=self.token)
Logger.info("Metasploit service ready!")
else:
Logger.error(
">> MetasploitAPI >> CAN NOT LOGIN TO METASPLOIT SERVICE")
sys.exit()
def set_default_and_na_edge_noises(graph: ig.Graph, data_extent: Polygon,
log: Logger) -> None:
"""Sets noise attributes of edges to their default values and None outside the extent of the noise data.
"""
# first set noise attributes of all edges as nodata
graph.es[E.noises.value] = None
graph.es[E.noise_source.value] = None
edge_gdf = ig_utils.get_edge_gdf(graph, attrs=[E.id_ig])
data_extent_gdf = gpd.GeoDataFrame(data=[{
'has_noise_data': 1
}],
geometry=[data_extent],
crs=CRS.from_epsg(3879))
joined = gpd.sjoin(edge_gdf, data_extent_gdf, how='left',
op='within').drop(['index_right'], axis=1)
edges_within = joined[joined['has_noise_data'] == 1]
real_edge_count = len([
geom for geom in list(edge_gdf['geometry'])
if isinstance(geom, LineString)
])
log.info(
f'found {real_edge_count - len(edges_within)} edges of {real_edge_count} outside noise data extent'
)
# set noise attributes of edges within the data extent to default values (no noise)
for edge in edges_within.itertuples():
graph.es[getattr(edge, E.id_ig.name)][E.noises.value] = {}
graph.es[getattr(edge, E.id_ig.name)][E.noise_source.value] = ''
def start_export_process(args):
args.body = read_request_body()
export_filename = get_filename(args.index)
line_number = 0
csv_writer = None
first_line = True
with open(export_filename, 'wb') as json_file:
if args.out == 'csv':
csv_writer = csv.writer(json_file)
for source_data in export_data_from_elasticsearch(
args.http, args.index, args.body, args.size, args.skip,
args.limit):
if source_data:
line_number += 1
if args.out == 'json':
json_file.write(json.dumps(source_data) + '\n')
elif args.out == 'csv':
if first_line:
first_line = False
csv_writer.writerow(tuple(source_data.keys()))
try:
csv_writer.writerow(tuple(source_data.values()))
except Exception as error:
Logger.error(u"CSV文件写入出错%s" % error)
print u"当前导出数据量%s\r" % line_number,
Logger.info(u"导出数据量%d" % (line_number))
out_filename = 'data_list'
if args.out == 'csv':
out_filename = "%s.csv" % export_filename
elif args.out == 'json':
out_filename = "%s.json" % export_filename
if rename_to_outfile(export_filename, out_filename):
os.rename(export_filename, out_filename)
Logger.info(u"文件生成成功 %s" % out_filename)
def save_file(json_dict, json_file):
dir_name = os.path.dirname(json_file)
if not os.path.exists(dir_name):
Log.info('Json Dir: {} not exists.'.format(dir_name))
os.makedirs(dir_name)
with open(json_file, 'w') as write_stream:
write_stream.write(json.dumps(json_dict))
def json2xml(json_file, xml_file):
if not os.path.exists(json_file):
Log.error('Json file: {} not exists.'.format(json_file))
exit(1)
xml_dir_name = os.path.dirname(xml_file)
if not os.path.exists(xml_dir_name):
Log.info('Xml Dir: {} not exists.'.format(xml_dir_name))
os.makedirs(xml_dir_name)
def xml2json(xml_file, json_file):
if not os.path.exists(xml_file):
Log.error('Xml file: {} not exists.'.format(xml_file))
exit(1)
json_dir_name = os.path.dirname(json_file)
if not os.path.exists(json_dir_name):
Log.info('Json Dir: {} not exists.'.format(json_dir_name))
os.makedirs(json_dir_name)
class OperateConfig:
"""
配置文件读写
:param file_path: 配置文件路径
"""
def __init__(self, file_path):
self._file_name_path = file_path
self._cg = configparser.ConfigParser()
self._cg.read(self._file_name_path, encoding='utf-8')
self._log = Logger('读取配置').get_logger()
def get_str(self, section, option):
"""
读取字符型参数
:param section: 类名
:param option: key
:return: value
"""
value = self._cg.get(section, option)
self._log.debug('获取到：{} = {}'.format(option, value))
return value
def get_int(self, section, option):
"""
读取数值型参数
:param section: 类名
:param option: key
:return: value
"""
value = self._cg.getint(section, option)
self._log.debug('获取到：{} = {}'.format(option, value))
return value
def get_bool(self, section, option):
"""
读取布尔型参数
:param section: 类名
:param option: key
:return: value
"""
value = self._cg.getboolean(section, option)
self._log.debug('获取到：{} = {}'.format(option, value))
return value
def write_data(self, section, option, value):
"""
把传入的参数写入到文件
:param section: 类名
:param option: key
:param value: value
:return: None
"""
self._cg.set(section, option, value)
with open(self._file_name_path, 'w') as fp:
self._cg.write(fp)
self._log.info('成功写入参数：{} = {}'.format(option, value))
def add_unique_geom_id(point_gdf: gpd.GeoDataFrame,
log: Logger = None) -> gpd.GeoDataFrame:
"""Adds an unique identifier (string) to GeoDataFrame of points based on point locations (x/y).
"""
point_gdf[S.xy_id] = [
f'{str(round(geom.x, 1))}_{str(round(geom.y, 1))}'
for geom in point_gdf[S.geometry]
]
unique_count = point_gdf[S.xy_id].nunique()
unique_share = round(100 * unique_count / len(point_gdf.index), 2)
log.info(f'found {unique_count} unique sampling points ({unique_share} %)')
return point_gdf
def get_ckpt(self):
ckpt_dir = glob.glob(
os.path.join(self.get_backup_path(), self.net_info['ckpt'] + '.*'))
if not len(ckpt_dir):
log.error("{} is not found".format(ckpt_dir))
sys.exit()
if len(ckpt_dir) != 1:
log.error("File {} conflict!".format(self.net_info['ckpt']))
sys.exit()
ckpt_dir = ckpt_dir[0]
log.info(">>> loading ckpt from '{}'".format(ckpt_dir))
return ckpt_dir
class PortManager:
# Port Manager: calls necessary managers and utilities to generate parameters for sql.
# List of valid ports it can receive is taken from the Configuration setup.
#
validPortNumbers = ()
def __init__(self):
self.g_config = GlobalConfig()
self.validPortNumbers = self.g_config.get_ports()
self.date_time_field = self.g_config.get_db_datetime_name()
self.log = Logger().get('reportserver.manager.PortManager.PortManager')
def isPortValid(self, port_number):
if (port_number in self.validPortNumbers):
return True
else:
return False
def getPort(self, port_number, uom, unit):
self.log.info("Retrieving port:" + str(port_number) + "uom:" + uom + " size: " + str(unit))
items = []
if self.isPortValid(port_number):
results = DatabaseHandler().get_json_by_time(port_number, uom, unit)
items = utilities.process_data(results)
port_json = {
'port': str(port_number),
'timespan': uom + "=" + str(unit),
'items':items
}
return port_json
def get_port_attack_count(self, tablename, unit, uom):
fromDate = dateTimeUtility.get_begin_date_iso(unit, uom)
sql = "select count(distinct session) as total_attacks from %s where %s >= '%s' " %(tablename, self.date_time_field, fromDate)
self.log.debug("sql is:" + sql)
result = DatabaseHandler().query_db(sql)[0]
return int(result['total_attacks'])
def get_unique_ips(self, tablename, unit, uom):
fromDate = dateTimeUtility.get_begin_date_iso(unit, uom)
sql = "select count(distinct peerAddress) as unique_ips from %s where %s >= '%s' " % (tablename, self.date_time_field, fromDate)
self.log.debug("sql is:" + sql)
result = DatabaseHandler().query_db(sql)[0]
return int(result['unique_ips'])
class Dictionary(customdict_trait):
def __init__(self, default_params_filename='params.yaml', *args, **kwargs):
# Extend the dictionary with the values passed in arguments.
# Call the Dictionary constructor once the parameters file is set.
arguments = Arguments(args, kwargs)
if arguments.args.config_file is not None:
parameters_file = arguments.args.config_file[0]
else:
parameters_file = default_params_filename
super().__init__(parameters_file, *args)
for possible_action in arguments.possible_actions:
setattr(self, possible_action, False)
setattr(self, arguments.args.action, True)
# setattr(self, 'do_plot', arguments.args.plot)
self.do_plot = arguments.args.plot
self.save_predictions = arguments.args.save
self.input_file = arguments.args.file[0]
if arguments.args.window is not None:
self.window_size = arguments.args.window[0]
else:
self.window_size = 10
if arguments.args.epochs is not None:
self.epochs = arguments.args.epochs[0]
else:
self.epochs = 1
# Output filename specified
if arguments.args.output is not None:
self.output = arguments.args.output[0]
else:
self.output = None
#
# Extend the dictionary with custom meta-parameters
#
self.ohlc_tags = list(list(self.csv_dict.keys())[1:])
#
# Set log_level and start the logger
#
setattr(self, 'log_level',
arguments.args.debug[0] if arguments.args.debug is not None \
else 3)
if 'log_level' not in self:
self.log_level = 3 # default value = WARNING
self.log = Logger(self.log_level)
self.log.info(
'Using configuration parameters from: {}'.format(parameters_file))
class IpsServiceHandler:
def __init__(self):
self.log = Logger().get("reportserver.manager.IpsServiceHandler.py")
def process(self, rqst, path_tokens, query_tokens):
uom = None
units = None
self.log.info("processing ipaddress request:" + str(path_tokens) + str(query_tokens))
try:
time_period = utilities.validate_time_period(query_tokens)
uom = time_period[0]
units = time_period[1]
except ValueError:
rqst.badRequest(units)
return
if len(path_tokens) == 5:
ipaddress = path_tokens[4].strip()
self.log.debug("requested: " + str(ipaddress))
if ipaddress is not None or ipaddress is not "":
try:
ipaddress = utilities.validate_ipaddress(ipaddress)
self.get_ips_data_by_time(rqst, ipaddress, uom, units)
except ValueError:
rqst.badRequest(badIpAddress)
return
elif ipaddress == None or ipaddress == "":
self.get_ips_data_by_time(rqst, "", uom, units)
else:
rqst.badRequest()
return
elif len(path_tokens) == 4:
self.get_ips_list_json(rqst, uom, units)
else:
rqst.badRequest()
return
def get_ips_data_by_time(self, rqst, ipaddress, uom, units):
ips_manager = IpsManager()
addressjsondata = ips_manager.get_data(ipaddress, uom, units)
if addressjsondata is not None:
# send response:
rqst.sendJsonResponse(addressjsondata, 200)
else:
rqst.notFound()
def get_ips_list_json(self, rqst, uom, units):
response = "{not implemented yet.}"
rqst.sendJsonResponse(response, 200)
class DatabaseHandler:
def __init__(self):
self.global_config = GlobalConfig()
self.db_path = self.global_config['Database']['path']
self.log = Logger().get('reportserver.dao.DatabaseHandler.DatabaseHandler')
# Connect to given database.
# Defaults to the honeypot db, but another path can be passed in (mainly for testing).
# Database needs to exist first.
def connect(self, database_name):
if (database_name == None):
database_name = self.db_path
if not os.path.exists(database_name):
self.log.error("Database does not exist in path: " + database_name)
return None
try:
conn = sqlite3.connect(database_name)
except sqlite3.OperationalError as oe:
self.log.error("****Problem connecting to database*** at: " + database_name)
self.log.error(oe)
else:
return conn
# Query DB and return JSON
def query_db(self, query, args=(), one=False, db=None):
#print ("#debug args are: " +str(args))
cur = self.connect(db).cursor()
cur.execute(query, args)
r = [dict((cur.description[i][0], value) \
for i, value in enumerate(row)) for row in cur.fetchall()]
cur.connection.close()
return (r[0] if r else None) if one else r
# Unit of Measure could be "weeks", "days", "hours", "minutes".
# Return all data from the DB within that measure of time as JSON.
def get_json_by_time(self, portnumber, uom, units):
begin_date_iso = dateTimeUtility.get_begin_date_iso(uom, units)
tableName = self.global_config.get_plugin_config(portnumber)['table']
date_time_field = self.global_config.get_db_datetime_name()
# query = query_db("SELECT * FROM %s where (datetime > '%s')" % (tableName, query_date_iso))
queryString = "SELECT * FROM %s where %s >= '%s' order by id, %s" % (tableName, date_time_field, begin_date_iso, date_time_field)
#args = (tableName, date_time_field, begin_date_iso)
self.log.info("queryString is: " + str(queryString))
#print ("args to use: " + str(args))
results = self.query_db(queryString)
self.log.debug("results: " + str(results))
return results
def create_test_suite(directory: str):
"""
构建测试集
:param directory: 测试用例目录
:return: testSuite
"""
# 引用日志类
log = Logger('构建测试集').get_logger()
log.info('递归 {} 目录下所有测试用例'.format(directory))
discover = unittest.defaultTestLoader.discover(
directory,
pattern='test_*.py',
top_level_dir=constant.all_case_top_path
)
log.info('搜索完毕，共加载测试用例计{}个'.format(discover.countTestCases()))
return discover
class IpsManager:
# Ips Manager: calls necessary managers and utilities to generate parameters for sql.
#
validPortNumbers = ()
def __init__(self):
self.g_config = GlobalConfig()
self.valid_port_numbers = self.g_config.get_ports()
self.date_time_field = self.g_config.get_db_datetime_name()
self.log = Logger().get('reportserver.manager.IpsManager.py')
def get_data(self, ipaddress, uom, unit):
self.log.info("Retrieving ipaddress data: " + str(ipaddress) + " uom: " + uom + " size: " + str(unit))
port_data = []
for port in self.valid_port_numbers:
results = self.get_json_by_ip(port, ipaddress, uom, unit)
items = utilities.process_data(results)
port_data.append({port:items})
port_json = {
'ipaddress': str(ipaddress),
'timespan': uom + "=" + str(unit),
'ports':port_data
}
return port_json
def get_json_by_ip(self, portnumber, ipaddress, uom, units):
begin_date_iso = dateTimeUtility.get_begin_date_iso(uom, units)
table_name = self.g_config.get_plugin_config(portnumber)['table']
date_time_field = self.g_config.get_db_datetime_name()
# query = query_db("SELECT * FROM %s where (datetime > '%s')" % (tableName, query_date_iso))
queryString = "SELECT * FROM %s where %s >= '%s' and peerAddress = '%s' order by id, %s" % (
table_name, date_time_field, begin_date_iso, ipaddress, date_time_field)
# args = (tableName, date_time_field, begin_date_iso)
self.log.info("queryString is: " + str(queryString))
# print ("args to use: " + str(args))
results = DatabaseHandler().query_db(queryString)
self.log.debug("results: " + str(results))
return results
def add_inside_nodata_zone_column(gdf,
nodata_zone: gpd.GeoDataFrame,
log: Logger = None) -> gpd.GeoDataFrame:
"""Adds boolean column (nodata_zone) indicating whether the points in the gdf are within the given nodata_zone polygon.
Args:
gdf: A GeoDataFrame object of sampling points.
nodata_zone: A GeoDataFrame object with one feature in it. It must have one attribute (nodata_zone) with value 1.
"""
joined = gpd.sjoin(gdf, nodata_zone, how='left',
op='within').drop(['index_right'], axis=1)
if (log != None):
nodata_zone_count = len(joined[joined[S.nodata_zone] == 1])
nodata_zone_share = round(100 * nodata_zone_count / len(gdf.index), 2)
log.info(
f'found {nodata_zone_count} ({nodata_zone_share} %) sampling points inside potential nodata zone'
)
return joined
def noise_graph_update(graph: ig.Graph, noise_csv_dir: str,
log: Logger) -> None:
"""Updates attributes noises and noise_source to graph.
"""
noise_csvs = os.listdir(noise_csv_dir)
for csv_file in noise_csvs:
edge_noises = pd.read_csv(noise_csv_dir + csv_file)
edge_noises[E.noise_source.name] = edge_noises[
E.noise_source.name].replace({np.nan: ''})
log.info(f'updating {len(edge_noises)} edge noises from ' + csv_file)
for edge in edge_noises.itertuples():
graph.es[getattr(edge, E.id_ig.name)][E.noises.value] = getattr(
edge, E.noises.name)
graph.es[getattr(edge,
E.id_ig.name)][E.noise_source.value] = getattr(
edge, E.noise_source.name)
def export_data_from_elasticsearch(http,
index='*',
json_body='',
size_data=100,
from_data=0,
limit=10000000):
elastic_search_connect = Elasticsearch(http,
retry_on_timeout=True,
max_retries=3,
timeout=3600000)
try:
body = json.loads(json_body)
except Exception as error:
Logger.error(u"查询请求body出错%s" % error)
body = {}
if limit < size_data:
size_data = limit
Logger.info(u'查询body为 %s' % json.dumps(body))
total_number = elastic_search_connect.count(index=index, body=body)
Logger.info(u"数据总量为 %d" % int(total_number['count']))
if not body.has_key('size'):
body['size'] = size_data
if not body.has_key('from'):
body['from'] = from_data
else:
body['from'] = 0
data_number = 0
while True:
response = elastic_search_connect.search(index=index, body=body)
if len(response['hits']['hits']) > 0:
data = response['hits']['hits']
for _source in data:
yield _source['_source']
data_number += 1
if data_number == limit:
break
else:
body['from'] += size_data
continue
break
else:
break
yield None
def test_UpdateLogin(self):
u"""修改登录密码"""
print("*************执行修改登录密码用例**************")
params = Params()
self.driver.find_element_by_id("iv_head").click() #点击进入个人中心页
self.driver.find_element_by_id("rl_pwd_manage").click() #点击密码管理
try:
self.driver.find_element_by_id("rl_reset_loginpwd").click() #点击修改登录密码
#异常情况
self.driver.find_element_by_id("tv_confimchange").click() #不输入,点击确认修改
time.sleep(2)
self.driver.find_element_by_id("et_originpwd").send_keys(params['loginPWD']) # #输入原密码
self.driver.find_element_by_id("et_pwd").send_keys(params['loginPWD']) #输入新密码
self.driver.find_element_by_id("et_queren_pwd").send_keys("111111") #再次输入 不一致新密码
self.driver.find_element_by_id("tv_confimchange").click() #点击确认修改
time.sleep(2)
#正常情况
self.driver.find_element_by_id("et_originpwd").send_keys(params['loginPWD']) # #输入原密码
time.sleep(2)
self.driver.find_element_by_id("et_pwd").send_keys(params['loginPWD']) #输入新密码
time.sleep(2)
self.driver.find_element_by_id("et_queren_pwd").send_keys(params['loginPWD']) #再次输入新密码
self.driver.find_element_by_id("tv_confimchange").click() #点击确认修改
time.sleep(2)
exist = False
try:
self.driver.find_element_by_id("tv_confimchange").click() #点击确认修改
exist = True
except:
pass
except:
exist = True
pic_name = get_current_function_name()
#调用截图方法
getScreen(self,pic_name)
#写入日志
logger=Logger(logname='log.txt',loglevel="INFO",logger=pic_name).getlog()
logger.info("*************执行修改登录密码用例**************")
logger.error(traceback.format_exc())
self.assertEqual(exist,False) #已跳转,不在修改页
class LogMain():
def __init__(self, title):
self.flag = conf.logger_config
print(self.flag)
print(title)
#self.logger = Logger(logger="FSD TEST").getlog()
self.logger = Logger(logger=title).getlog()
def run_info(self, message):
"""当为True时打印日志
Usge:
run_info('ceshi','message')
"""
if self.flag:
#logger = Logger(logger=title).getlog()
self.logger.info(message)
else:
#
pass
def get_mean_gsv_gvi_by_way_id(log: Logger,
gvi_list_by_way_id: Dict[int, List[float]],
edge_gdf: GeoDataFrame) -> Dict[int, float]:
"""Calculate mean GSV GVI for edges. Only edges (way IDs) for which enough
GSV GVI point samples are found are included in the returned dictionary.
"""
edge_length_by_way_id = get_col_by_col_dict(edge_gdf, E.id_way.name,
E.length.name)
mean_gsv_gvi_by_way_id = {
way_id: get_mean_edge_gsv_gvi(edge_length_by_way_id[way_id],
gvi_list_by_way_id[way_id])
for way_id in gvi_list_by_way_id
}
na_filtered = {
way_id: gvi
for way_id, gvi in mean_gsv_gvi_by_way_id.items() if gvi is not None
}
log.info(
f'Got mean point GVI for {sample_ratio(len(edge_gdf), len(na_filtered))} % edges'
)
return na_filtered
def test_UpdatePay(self):
u"""修改支付密码"""
print("*************执行修改支付密码用例**************")
params = Params()
self.driver.find_element_by_id("iv_head").click()
self.driver.find_element_by_id("rl_pwd_manage").click() #点击密码管理
try:
self.driver.find_element_by_id('tv_changeTranPwd').click() #点击修改支付密码
self.driver.find_element_by_id('tv_getAuthCode').click() #点击获取支付密码
zf_pwd = input("请输入短信验证码:") #需要手动输入
time.sleep(8)
self.driver.find_element_by_id('et_authcode').send_keys(zf_pwd)
self.driver.find_element_by_id('tv_next').click() #点击下一步
time.sleep(2)
try:
#输入支付密码
inputText(self,params['applyPWD']) #延时输入
time.sleep(2)
inputText(self,params['applyPWD']) #再次确认输入
time.sleep(2)
except:
pass
exist = False
try:
self.driver.find_element_by_id("inputView").click() #输入框是否隐藏
exist = True
except:
print("两次输入不一致")
pass
except:
exist = True
pic_name = get_current_function_name() #获取方法名
getScreen(self,pic_name) #调用截图方法
#写入日志
logger=Logger(logname='log.txt',loglevel="INFO",logger=pic_name).getlog()
logger.info("*************执行修改支付密码用例**************")
logger.error(traceback.format_exc())
self.assertEqual(exist,False) #已跳转,不在修改页
def login(self):
data = {
"email": settings.config["user"]["email"],
"password": settings.config["user"]["password"]
}
try:
r = self.manager_server.connect("POST",
"/auth/login/",
data=json.dumps(data))
Logger.info("Before login")
if r.status_code == 200:
user_info = r.json()
if "one-token" in user_info:
self.token = user_info["one-token"]
self.headers = {
"one-token": self.token,
"Content-Type": "application/json"
}
Logger.info("Login Successful!")
return r
except Exception, ex:
Logger.error(" web_service_api >> login >> error " + str(ex))
class KeyWordConsumer:
def __init__(self, log_file, log_name):
self.logger = Logger(log_file, log_name, 10*1024*1024, 2)
self._db_conn = None
def start(self):
rabbit_topic = RabbitTopic.init_rabbitmq_consumer(EXCHANGE_NAME, QUEUE_NAME, QUEUE_LIMIT,
[ROUTING_KEY], self.logger)
if not rabbit_topic:
self.logger.debug('Construct key word consumer error')
return
self._conn_db()
if not self._db_conn:
self.logger.exception('Connect to database error')
return
while 1:
try:
rabbit_topic.start_consuming(self._callback, QUEUE_NAME)
except ConnectionClosed:
self.logger.debug('Connection to rabbitmq server closed, re-connecting...')
rabbit_topic = RabbitTopic.init_rabbitmq_consumer(EXCHANGE_NAME, QUEUE_NAME, QUEUE_LIMIT,
[ROUTING_KEY], self.logger)
def _callback(self, channel, method, properties, key_word):
self.logger.info(os.linesep)
self.logger.info('----> Get body message %s and start searching this key word...<----' % key_word)
try:
url = 'https://play.google.com/store/search?q=%s&c=apps' % key_word
search_web_driver = SearchWebDriver(url, self._db_conn, self.logger)
search_web_driver.search()
except Exception:
self.logger.exception('Search key word %s error' % key_word)
channel.basic_ack(delivery_tag=method.delivery_tag)
self.logger.info('Set key word %s as consumed' % key_word)
self._set_key_word_consumed(key_word)
def _conn_db(self):
try:
self._db_conn = util.conn_mysql_db()
except Exception:
self.logger.exception('Connect to database error')
def _set_key_word_consumed(self, key_word):
query = 'UPDATE key_word SET status=%s WHERE key_word=%s'
try:
MySQLDBUtil.update(query, (CONSUMED, key_word), self._db_conn)
except Exception:
self.logger.exception('Set key word %s as consumed error' % key_word)
def exec_test_to_generate_report(present_title: str, test_suite_or_case):
"""
执行测试，生成测试报告
:param present_title: 测试主题
:param test_suite_or_case: 测试套件或用例
:return: None
"""
# 引用日志类
log = Logger('执行测试，生成测试报告').get_logger()
repo_name, repo_path_name = _create_report_path_and_name(present_title)
# 调用第三方模块HTMLTestRunner执行测试，生成报告
fp = open(repo_path_name, 'wb')
runner = HTMLTestReportCN(stream=fp,
title=present_title,
description='用例执行情况：')
log.info('开始执行测试')
runner.run(test_suite_or_case)
fp.close()
log.info('测试执行完毕.')
log.info('测试报告文件：%s' % repo_name)
log.info('测试报告路径：%s' % repo_path_name)
def encrypt_by_hashlib(en_string: str, en_mode: str):
"""
封装hashlib哈希算法
:param en_string: 需要加密的字符串
:param en_mode: 加密算法名
:return: 加密后的值
"""
log = Logger('哈希算法加密').get_logger()
mode = str(en_mode).lower().strip()
log.info('执行{}加密：{}'.format(mode, en_string))
if mode == 'md5':
m = hashlib.md5()
m.update(str(en_string).encode('utf-8'))
value = m.hexdigest()
elif mode == 'sha1':
m = hashlib.sha1()
m.update(str(en_string).encode('utf-8'))
value = m.hexdigest()
elif mode == 'sha224':
m = hashlib.sha224()
m.update(str(en_string).encode('utf-8'))
value = m.hexdigest()
elif mode == 'sha256':
m = hashlib.sha256()
m.update(str(en_string).encode('utf-8'))
value = m.hexdigest()
elif mode == 'sha384':
m = hashlib.sha384()
m.update(str(en_string).encode('utf-8'))
value = m.hexdigest()
elif mode == 'sha512':
m = hashlib.sha512()
m.update(str(en_string).encode('utf-8'))
value = m.hexdigest()
else:
log.info('此函数目前还不支持{}算法，赶快去动手加入它吧！'.format(mode))
sys.exit()
log.info('{}加密成功，值为：{}'.format(mode, value))
return value
class Database:
def __init__(self):
self.global_config = GlobalConfig()
self.log = Logger().get('database.database.Database')
def create_default_database(self):
"""
Calls methods needed to create the database.
"""
self.create_db_dir()
self.create_db()
# Execute scripts BEFORE updating schema
run_db_scripts(self.global_config)
self.update_schema()
def create_db_dir(self):
"""
Creates the database directory if it doesn't already exist.
"""
# if database directory does not exist create it
db_path = self.global_config['Database']['path']
(db_dir, db_name) = ntpath.split(db_path)
if not os.path.isdir(db_dir):
self.log.info("Database directory not found, "
"creating database directory...")
os.mkdir(db_dir)
def create_db(self):
"""
Creates the database if it doesn't already exist.
"""
# if database file does not exist in the directory, create it
(db_dir, db_name) = ntpath.split(self.global_config['Database']['path'])
if not os.path.exists(self.global_config['Database']['path']):
self.log.info("Database file not found, creating database file...")
# this actually creates the database file
connection = sqlite3.connect(self.global_config['Database']['path'])
connection.close()
def update_schema(self):
"""
Updates the database when columns have been added to, or
removed from, the schema.
"""
# Create any new tables that have been added to the plugin
# config schema.
db_tables = DataValidator().get_tables()
cfg_tables = get_config_table_list(
self.global_config.get_ports(),
self.global_config.get_plugin_dictionary())
table_diff = list(set(cfg_tables) - set(db_tables))
self.create_non_exist_tables(table_diff)
# Populate the newly created tables with their column
# definitions.
DataValidator().update_tables_and_schema()
self.update_table_structure()
def create_non_exist_tables(self, table_diff):
"""
create tables that do not exist from the table difference between the current database and the configuration
"""
if len(table_diff) > 0:
for table in table_diff:
Table_Init.create_table(table, self.global_config)
self.log.info('Updated database schema, table names now match configuration.')
else:
self.log.info('Database Schema and Configuration table names already match.')
def create_dict_config_column_list(self):
"""
get a dictionary of tables and corresponding columns from the config
"""
config_column_lists = {}
for port in self.global_config.get_ports():
value = self.global_config.get_plugin_dictionary().get(port)
config_column_lists[value.get('table')] = value.get('tableColumns')
return config_column_lists
def create_dict_transformed_column_list(self, database_column_lists):
"""
returns only custom plugin defined columns from database schema i.e.
ignores default columns
"""
transformed_db_column_list = {}
for table in database_column_lists:
col_list = database_column_lists[table]
transformed_db_column_list[table] = []
# default column ids to ignore
default_list = []
for default in default_columns:
default_list.append(default[0])
for column in col_list:
# ignores the default columns
if column[1] in default_list:
continue
transformed_db_column_list[table].append([column[0],column[1],column[2]])
return transformed_db_column_list
def update_table_structure(self):
cfg_schema = self.create_dict_config_column_list()
db_schema = DataValidator().get_schema()
db_schema_sans_defaults = self.create_dict_transformed_column_list(db_schema)
for table in cfg_schema:
if not [(x[1], x[2]) for x in cfg_schema[table]] == \
[(x[1], x[2]) for x in db_schema_sans_defaults[table]]:
Table_Init.change_table_structure(
table, cfg_schema[table], db_schema[table],
self.global_config)
start_time = time.time()
_datas, _labels = data_gen.__next__()
##print(np.mean(_datas)," , ",np.max(_datas)," , ",np.min(_datas))
r_lab, r_log, _alfa, _, _dis_total_loss, _lr, _acc = sess.run(
[
real_labels, real_logits, alfa, dis_optimize_ops, dis_total_loss,
lr, accuracy
],
feed_dict={
real_datas: _datas,
real_labels: _labels
})
##print(np.mean(_datas), " , ", np.max(_datas), " , ", np.min(_datas)," : ")
##print(r_lab)
##.print(r_log)
logger.tick(iter)
logger.info('klr', _lr * 1000)
logger.info('acc', _acc * 100)
logger.info('time', time.time() - start_time)
logger.info('loss_dis_gan', _dis_total_loss)
if (iter + 1) % 1000 == 0:
logger.save()
save_model(saver, sess, sCheckpointDir, step=iter)
last_save_time = time.time()
logger.log('Model Saved\n\n')
logger.flush()
last_log_time = time.time()
from common.logger import Logger
from bs4 import BeautifulSoup
from time import sleep
import sys
import os
sys.path.append(os.getcwd())
HOST = 'localhost'
PORT = 3306
USER = '******'
PASS = '******'
DBNAME = 'selenium'
CHARSET = 'utf8'
# global db
logger = Logger(logger="test_by12306.py").getlog()
logger.info("初始化日志模块完成")
class TestCase(unittest.TestCase):
"""测试用例模块"""
def setUp(self):
"""初始化unittest前置条件"""
pass
def tearDown(self):
"""初始化unittest后置条件"""
logger.info("准备退出chromedriver")
# def dbinit(self):
def insert_data_sql(self, a, b):
"""将爬取到的数据插入到数据库中"""
class AndroidEngine:
"""
安卓引擎，自动化启动appium服务，自动连接设备，真机可自动切换WiFi连接，获取driver
1、实例化前请确保模拟器已开启，真机已连接电脑，并且打开调试模式
2、暂时只支持单机连接启动，若本机同时连接多个设备，则默认连接已连接设备列表第一个设备
"""
def __init__(self):
self._log = Logger('安卓引擎').get_logger()
# 读取配置
self._reader = OperateConfig(constant.config_pro_app)
self._if_wifi = self._reader.get_bool('server', 'if_wifi')
self._android_mode = self._reader.get_str('server', 'android_mode')
def get_driver(self):
"""
根据配置获取driver
:return: driver对象
"""
self._start_server()
devices = self._get_device_names()
version = self._get_android_version(devices[0])
app_path = publicFunctions.get_apk_path()
ports = eval(self._reader.get_str('connected', 'server_ports'))
if self._android_mode == 'simulator':
desired_caps = DesiredCaps.caps_android_simulator
desired_caps['platformVersion'] = version
desired_caps['deviceName'] = devices[0]
desired_caps['app'] = app_path
driver = self._get_driver(desired_caps, ports[0])
return driver
elif self._android_mode == 'machine':
desired_caps = DesiredCaps.caps_android_machine
desired_caps['platformVersion'] = version
desired_caps['deviceName'] = devices[0]
desired_caps['app'] = app_path
driver = self._get_driver(desired_caps, ports[0])
return driver
else:
self._log.error('启动模式配置有误，请确认：{}'.format(self._android_mode))
self._kill_server()
sys.exit()
def quit_driver(self, driver):
"""
退出驱动程序，断开模拟器连接，杀掉node进程
:param driver: driver对象
:return: None
"""
if self._android_mode == 'simulator':
self._disconnect_simulators()
driver.quit()
self._kill_server()
sleep(3)
self._log.info('已退出驱动')
def _start_server(self):
"""
使用命令行自动化启动appium server
:return: driver对象
"""
if self._android_mode == 'simulator':
self._connect_simulators()
devices = self._get_device_names()
if self._if_wifi is True and self._android_mode == 'machine':
self._switch_to_wifi()
devices = [device for device in self._get_device_names() if ':' in device]
commands = self._create_appium_commands(devices)
for cmd in commands:
cmd = r"start {}".format(cmd)
os.system(cmd)
sleep(3)
self._log.info('appium server已启动：{}'.format(cmd))
def _get_driver(self, desired_caps: dict, port: str):
"""
获取driver
:param desired_caps: 连接参数
:param port: 服务端口
:return: driver对象
"""
try:
driver = webdriver.Remote(command_executor='http://127.0.0.1:{}/wd/hub'.format(port),
desired_capabilities=desired_caps)
sleep(1)
self._log.info('appium server已连接')
return driver
except WebDriverException as e:
self._log.error('appium server连接失败：{}'.format(e))
sys.exit()
def _connect_simulators(self):
"""
对于模拟器，在启动后可以调用此方法实现自动连接电脑
:return: None
"""
simulators = self._reader.get_str('server', 'simulator').split(';')
for simulator in simulators:
cmd = 'adb connect {}'.format(simulator)
os.system(cmd)
self._log.debug('模拟器（{}）已连接'.format(simulator))
def _disconnect_simulators(self):
"""
断开全部已连接模拟器设备
:return: None
"""
devices = self._reader.get_str('server', 'simulator').split(';')
for device in devices:
cmd = 'adb disconnect {}'.format(device)
os.system(cmd)
self._log.debug('设备（{}）已断开'.format(device))
def _switch_to_wifi(self):
"""
对于真机，若需要使用WiFi连接模式，在手机用USB线连接到电脑打开调试模式后，调用此方法可切换至WIFI连接
:return: None
"""
devices = self._get_device_names()
simulators = self._reader.get_str('server', 'simulator').split(';')
machines = list(set(devices) - set(simulators))
ports = self._create_useful_ports(5555, machines)
for machine, port in zip(machines, ports):
if str(port) in '|'.join(self._get_device_names()):
cmd_1 = 'adb -s {} shell ip -f inet addr show wlan0'.format(machine)
result_1 = self._execute_command(cmd_1)
ip = re.search(r"inet\s(\d+\.\d+\.\d+\.\d+)", result_1).group(1)
cmd_2 = 'adb -s {} tcpip {}'.format(machine, port)
os.system(cmd_2)
cmd_3 = 'adb connect {}:{}'.format(ip, port)
result_2 = self._execute_command(cmd_3)
if 'connected' in result_2:
self._log.debug('设备（{}）成功切换至WIFI连接：{}'.format(machine, result_2.strip()))
self._log.warning('请拔掉设备（{}）USB线！！'.format(machine))
else:
self._log.error('设备（{}）切换至WIFI连接失败：{}'.format(machine, result_2.strip()))
def _get_device_names(self):
"""
获取已连接安卓设备名
:return: 安卓设备名列表
"""
cmd = 'adb devices'
result = self._execute_command(cmd)
devices = re.findall(r"(.*[^\s])\s*device", result)
devices.pop(0)
if devices:
self._log.debug('获取到已连接设备列表：{}'.format(devices))
return devices
else:
self._log.error('未检测到安卓设备。')
sys.exit()
def _get_android_version(self, device: str):
"""
获取已连接安卓设备版本号
:param device: 设备名
:return: 版本号
"""
cmd = f'adb -s {device} shell getprop ro.build.version.release'
result = self._execute_command(cmd)
self._log.debug('获取到设备版本号：{}'.format(result))
return result.strip()
def _get_package_and_activity(self, apk_path=publicFunctions.get_apk_path()):
"""
通过'aapt'命令自动获取appPackage和appActivity
:param apk_path: apk路径
:return: appPackage和appActivity
"""
sdk_path = self._get_sdk_path()
adb_disk = sdk_path.split(':')[0]
build_tools_path = os.path.join(sdk_path, 'build-tools')
aapt_path = os.path.join(build_tools_path, os.listdir(build_tools_path)[0])
cmd = f'{adb_disk}:&cd {aapt_path}&aapt dump badging {apk_path}'
result = self._execute_command(cmd)
package = re.search(r"package: name='([\w\\.]+)'", result).group(1)
activity = re.search(r"launch.*activity: name='([\w\\.]+)'", result).group(1)
return package, activity
def _get_sdk_path(self):
"""
从PATH环境变量中提取Android SDK路径
:return: Android SDK路径
"""
path_env = os.environ['PATH']
sdk_search = re.search(r'(.+?)\\platform-tools', path_env)
if sdk_search:
sdk_path = sdk_search.group(1).split(';')[-1]
if '%' in sdk_path:
sdk_path = os.environ[sdk_path.strip('%')]
return sdk_path
else:
self._log.error('Android SDK环境变量未配置！！')
exit()
@staticmethod
def _execute_command(cmd: str):
"""
执行cmd命令
:param cmd: cmd命令
:return: 命令行输出
"""
with os.popen(cmd) as f:
result = f.read()
return result
def _kill_server(self):
"""
用于每次执行完毕，杀掉进程
:return: None
"""
cmd1 = 'tasklist | find "node.exe"'
if self._execute_command(cmd1):
cmd2 = 'taskkill -F -PID node.exe'
self._execute_command(cmd2)
self._log.info('杀掉appium server进程')
def _create_appium_commands(self, devices_list: list):
"""
创建Appium命令行模式启动命令
:param devices_list: 设备名列表
:return: cmd命令列表
"""
p_port_list = self._create_useful_ports(4723, devices_list)
bp_port_list = self._create_useful_ports(4900, devices_list)
self._reader.write_data('connected', 'server_ports', str(p_port_list))
cmd_list = ['appium -a 127.0.0.1 -p {} -bp {}'.format(
p_port_list[i], bp_port_list[i]
) for i in range(len(devices_list))
]
self._log.debug('已生成启动命令：{}'.format(cmd_list))
return cmd_list
def _create_useful_ports(self, start_port: int, devices_list: list):
"""
根据获取的已连接设备创建指定数量的可用端口
:param start_port: 起始端口
:param devices_list: 从命令行自动获取的设备列表
:return: 可用端口列表
"""
port_list = []
cmd = 'netstat -ano | findstr {}'.format(start_port)
while len(port_list) != len(devices_list):
if not self._execute_command(cmd):
port_list.append(start_port)
start_port += 1
self._log.debug('已生成可用端口：{}'.format(port_list))
return port_list
class WorldmapServiceHandler():
def __init__(self):
self.log = Logger().get('reportserver.manager.WorldmapServiceManager.py')
self.global_config = GlobalConfig()
self.global_config.read_plugin_config()
self.global_config.read_global_config()
def process(self, rqst, path_tokens, query_tokens):
global have_basemap
if not have_basemap:
err_msg = \
('<html><head><title>WorldMap</title></head><body>'
'To enable WorldMap generation, please visit '
'<a href="https://recce7.github.io/">the documentation</a> and '
'follow the directions for installing the Basemap library.'
'</body></html>')
rqst.send_response(200)
#todo make this configurable for allow-origin
rqst.send_header("Access-Control-Allow-Origin","http://localhost:8000")
rqst.send_header('Content-Type', 'text/html')
rqst.send_header('Content-Length', len(err_msg))
rqst.end_headers()
rqst.flush_headers()
rqst.wfile.write(bytes(err_msg, "utf-8"))
rqst.wfile.flush()
return
uom = None
units = None
self.log.info("processing ipaddress request:" + str(path_tokens) + str(query_tokens))
try:
time_period = utilities.validate_time_period(query_tokens)
uom = time_period[0]
units = time_period[1]
except ValueError:
rqst.badRequest(units)
return
if len(path_tokens) >= 5:
rqst.badRequest()
return
else:
self.construct_worldmap(rqst, uom, units)
def construct_worldmap(self, rqst, uom, units):
#call to construct port list
#find unique ips by port
#merge the results togoether
#build the map
#probably want to look at the PortsServiceHandler.py or IpsServiceHandler.py to follow those patterns.
ip_map = pickle.loads(pickle_bytes)
pts = self.get_point_list(uom, units)
for pt in pts:
srclat, srclong = pt
x, y = ip_map(srclong, srclat)
plt.plot(x, y, 'o', color='#ff0000', ms=2.7, markeredgewidth=1.0)
plt.savefig('reportserver/worldmap.png', dpi=600)
img = Image.open('reportserver/worldmap.png')
draw = ImageDraw.Draw(img)
font = ImageFont.truetype(
"/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", 175)
draw.text((50, 50), "Unique IP addresses: last %s %s" % (units, uom),
(0, 0, 0), font=font)
font = ImageFont.truetype(
"/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", 125)
draw.text((50, 325), "Total: %s" % (len(pts)),
(0, 0, 0), font=font)
# draw = ImageDraw.Draw(img)
# draw = ImageDraw.Draw(img)
img.save("reportserver/worldmap.png")
rqst.sendPngResponse("reportserver/worldmap.png", 200)
def get_point_list(self, uom, units):
begin_date = dateTimeUtility.get_begin_date_iso(uom, units)
query_string = ('select lat,long '
'from ('
'select distinct lat,long,timestamp, ip '
'from ipInfo '
'where lat is not null '
'and long is not null '
'and datetime(timestamp) > datetime(\'' + begin_date + '\')'
');')
connection = sqlite3.connect(self.global_config['Database']['path'])
cursor = connection.cursor()
return cursor.execute(query_string).fetchall()
class PortsServiceHandler():
def __init__(self):
self.log = Logger().get('reportserver.server.PortServiceHandler.PortServiceHandler')
def process(self, rqst, path_tokens, query_tokens):
uom = None
units = None
self.log.info("processing ports request:" + str(path_tokens) + str(query_tokens))
if len(query_tokens) > 0:
try:
time_period = utilities.validate_time_period(query_tokens)
uom = time_period[0]
units = time_period[1]
except ValueError:
rqst.badRequest(units)
return
# default if we aren't given valid uom and units
if uom is None or units is None:
uom = "days"
units = 1
if len(path_tokens) == 5:
portNbr = utilities.validate_port_number(path_tokens[4])
self.log.debug("requested: " + str(portNbr))
if portNbr is not None and 0 < portNbr < 9000:
self.get_port_data_by_time(rqst, portNbr, uom, units)
elif portNbr == None or "":
self.get_port_list_json(rqst, uom, units)
else:
rqst.badRequest()
return
elif len(path_tokens) == 4:
self.get_port_list_json(rqst, uom, units)
else:
rqst.badRequest()
return
def get_port_list_json(self,rqst, uom, units):
jsondata = self.construct_port_summary_list(rqst, uom, units)
rqst.sendJsonResponse(jsondata, 200)
def get_port_data_by_time(self, rqst, portnumber, uom, units):
portmgr = PortManager()
portjsondata = portmgr.getPort(portnumber, uom, units)
if portjsondata is not None:
# send response:
rqst.sendJsonResponse(portjsondata, 200)
else:
rqst.notFound()
def construct_port_summary_list(self, rqst, uom, units):
g_config = GlobalConfig()
plugins_dictionary = g_config.get_plugin_dictionary()
json_list = []
for key, val in plugins_dictionary.items():
json_list.append(self.construct_port_summary(rqst, val['port'], val['table'], uom, units))
return json_list
def construct_port_summary(self, rqst, portnumber, tablename, uom, units):
portmgr = PortManager()
port_attacks = portmgr.get_port_attack_count(tablename, uom, units)
unique_ips = portmgr.get_unique_ips(tablename, uom, units)
timespan = uom+"="+str(units)
response_json = {
'port': str(portnumber),
'total_attacks': str(port_attacks),
'unique_ipaddresses': str(unique_ips),
'timespan':timespan,
'rel_link': rqst.get_full_url_path() + "/ports/" + str(portnumber)+"?" + timespan
}
return response_json
class _Framework:
def __init__(self, plugin_cfg_path, global_cfg_path):
self._global_config = GlobalConfig(plugin_cfg_path, global_cfg_path)
self._plugin_imports = {}
self._listener_list= {}
self._running_plugins_list = []
self._data_manager = None
self._shutting_down = False
self._log = None
self._pid = os.getpid()
def start(self):
self.set_shutdown_hook()
print('Press Ctrl+C to exit.')
if not self.drop_permissions():
return
self._global_config.read_global_config()
self.start_logging()
self._global_config.read_plugin_config()
self._data_manager = DataManager()
self._data_manager.start()
self.start_listeners()
def start_logging(self):
log_path = self._global_config['Framework']['logName']
log_level = self._global_config['Framework']['logLevel']
self._log = Logger(log_path, log_level).get('framework.frmwork.Framework')
self._log.info('RECCE7 started (PID %d)' % self._pid)
@staticmethod
def drop_permissions():
if os.getuid() != 0:
return True
dist_name = os.getenv('RECCE7_OS_DIST')
users_dict = {
'centos': ('nobody', 'nobody'),
'debian': ('nobody', 'nogroup')
}
if dist_name not in users_dict:
print(
'Unable to lower permission level - not continuing as\n'
'superuser. Please set the environment variable\n'
'RECCE7_OS_DIST to one of:\n\tcentos\n\tdebian\n'
'or rerun as a non-superuser.')
return False
lowperm_user = users_dict[dist_name]
nobody_uid = pwd.getpwnam(lowperm_user[0]).pw_uid
nogroup_gid = grp.getgrnam(lowperm_user[1]).gr_gid
os.setgroups([])
os.setgid(nogroup_gid)
os.setuid(nobody_uid)
os.umask(0o077)
return True
def create_import_entry(self, port, name, clsname):
imp = import_module('plugins.' + name)
self._plugin_imports[port] = getattr(imp, clsname)
def start_listeners(self):
ports = self._global_config.get_ports()
for port in ports:
plugin_config = self._global_config.get_plugin_config(port)
module = plugin_config['module']
clsname = plugin_config['moduleClass']
self.create_import_entry(port, module, clsname)
address = self._global_config['Framework']['listeningAddress']
listener = NetworkListener(address, plugin_config, self)
listener.start()
self._listener_list[port] = listener
def set_shutdown_hook(self):
signal.signal(signal.SIGINT, self.shutdown)
def shutdown(self, *args):
self._shutting_down = True
self._log.debug('Shutting down network listeners')
for listener in self._listener_list.values():
listener.shutdown()
self._log.debug('Shutting down plugins')
for plugin in self._running_plugins_list:
plugin.shutdown()
self._log.debug('Shutting down data manager')
self._data_manager.shutdown()
print('Goodbye!')
#
# Framework API
#
def get_config(self, port):
"""
Returns the configuration dictionary for the plugin
running on the specified port.
:param port: a port number associated with a loaded plugin
:return: a plugin configuration dictionary
"""
return self._global_config.get_plugin_config(port)
def spawn(self, socket, config):
"""
Spawns the plugin configured by 'config' with the provided
(accepted) socket.
:param socket: an open, accepted socket returned by
socket.accept()
:param config: the plugin configuration dictionary describing
the plugin to spawn
:return: a reference to the plugin that was spawned
"""
# ToDo Throw exception if plugin class not found
plugin_class = self._plugin_imports[config['port']]
plugin = plugin_class(socket, config, self)
plugin.start()
self._running_plugins_list.append(plugin)
return plugin
def insert_data(self, data):
"""
Inserts the provided data into the data queue so that it can
be pushed to the database.
:param data: data object to add to the database
"""
self._data_manager.insert_data(data)
def plugin_stopped(self, plugin):
"""
Tells the framework that the specified plugin has stopped
running and doesn't need to be shutdown explicitly on program
exit.
:param plugin: a reference to a plugin
"""
if self._shutting_down:
return
self._running_plugins_list.remove(plugin)
class NetworkListener(Thread):
def __init__(self, listening_address, config, framework):
super().__init__()
self._config = config
self._listening_address = listening_address
self._port = config['port']
self._framework = framework
self._session_socket = None
self._lock = Lock()
self._running = False
self.__connection_count = 0
self._logger = Logger().get('framework.networklistener.NetworkListener')
@property
def connection_count(self):
with self._lock:
return self.__connection_count
@connection_count.setter
def connection_count(self, val):
with self._lock:
self.__connection_count = val
# Override
def run(self):
self._running = True
self._logger.info('%s plugin listener started on port %d'
% (self._config['moduleClass'], self._port))
while self._running:
self._session_socket = socket.socket(
socket.AF_INET, socket.SOCK_STREAM)
self._session_socket.setsockopt(
socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self._session_socket.bind((self._listening_address, self._port))
self._session_socket.listen(1)
self.start_listening(self._session_socket)
self._session_socket.close()
self.connection_count += 1
self._logger.info('%s plugin listener on port %d shutting down'
% (self._config['moduleClass'], self._port))
self._session_socket = None
def start_listening(self, local_socket):
try:
(new_socket, addr) = local_socket.accept()
if self._running:
self._logger.info('New connection from %s on port %d'
% (addr, self._port))
self._framework.spawn(new_socket, self._config)
except ConnectionAbortedError as e:
if not self._running:
return
raise e
except OSError as e:
if e.errno == 22 and not self._running:
return
raise e
except Exception as e:
self._logger.error('Error on connection: %s' % e)
raise e
def shutdown(self):
self._running = False
if self._session_socket:
if platform.system() == 'Linux':
self._session_socket.shutdown(socket.SHUT_RDWR)
else:
self._session_socket.close()
self.join()