def validate_time_period(query_tokens):
log = Logger().get('reportserver.manager.utilities')
log.debug("given query_tokens:" + str(query_tokens))
uom = None
units = None
for token in query_tokens:
if '=' in token:
uom,units = token.split('=')
if uom in UnitOfMeasure.get_values(UnitOfMeasure):
units = int(units)
break
else:
uom = None
units = None
# default if we aren't given valid uom and units
#TODO: get this from a config file.
if uom is None or units is None:
uom = "days"
units = 1
log.debug("validate_time_period: " + str(uom) + ": " + str(units))
return (uom, units)
def sjoin_noise_values(gdf,
noise_layers: dict,
log: Logger = None) -> gpd.GeoDataFrame:
sample_gdf = gdf.copy()
sample_gdf['sample_idx'] = sample_gdf.index
for name, noise_gdf in noise_layers.items():
log.debug(f'joining noise layer [{name}] to sampling points')
sample_gdf = gpd.sjoin(sample_gdf, noise_gdf, how='left',
op='within').drop(['index_right'], axis=1)
if (len(sample_gdf.index) > len(gdf.index)):
log.warning(
f'joined multiple noise values for one or more sampling points ({len(sample_gdf.index)} != {len(gdf.index)})'
)
distinct_samples = remove_duplicate_samples(sample_gdf, 'sample_idx',
noise_layers)
if (len(distinct_samples.index) == len(gdf.index)):
log.info('successfully removed duplicate samples')
else:
log.error('error in removing duplicate samples')
if (list(sample_gdf.columns).sort() != list(
distinct_samples.columns).sort()):
log.error(
'schema of the dataframe was altered during removing duplicate samples'
)
return distinct_samples.drop(columns=['sample_idx'])
def read_excel_sheet(self):
Logger.debug(
"TestDataReaderCommonUtil.read_excel_sheet file name is : {0}, sheet_name is : {1}"
.format(self.file_name, self.sheet_name), self.__class__.__name__)
excel = load_workbook(self.file_name)
sheet = excel[self.sheet_name]
return sheet
def validate_port_number(givenStr):
log = Logger().get('reportserver.manager.utilities')
log.debug("given str is: " + givenStr)
try:
return int(givenStr)
except Exception as e:
log.error("Error: Received invalid string to convert to int: " + givenStr)
log.error (str(e))
return None
def filter_out_features_outside_mask(log: Logger, gdf, mask_poly):
gdf['inside'] = [
True if mask_poly.intersects(geom.boundary) else False
for geom in gdf['geometry']
]
filtered = gdf[gdf['inside'] == True]
log.debug(
f'Filtered out {len(gdf)-len(filtered)} rows outside the mask of total {len(gdf)} rows'
)
return filtered
class KeyWordConsumer:
def __init__(self, log_file, log_name):
self.logger = Logger(log_file, log_name, 10*1024*1024, 2)
self._db_conn = None
def start(self):
rabbit_topic = RabbitTopic.init_rabbitmq_consumer(EXCHANGE_NAME, QUEUE_NAME, QUEUE_LIMIT,
[ROUTING_KEY], self.logger)
if not rabbit_topic:
self.logger.debug('Construct key word consumer error')
return
self._conn_db()
if not self._db_conn:
self.logger.exception('Connect to database error')
return
while 1:
try:
rabbit_topic.start_consuming(self._callback, QUEUE_NAME)
except ConnectionClosed:
self.logger.debug('Connection to rabbitmq server closed, re-connecting...')
rabbit_topic = RabbitTopic.init_rabbitmq_consumer(EXCHANGE_NAME, QUEUE_NAME, QUEUE_LIMIT,
[ROUTING_KEY], self.logger)
def _callback(self, channel, method, properties, key_word):
self.logger.info(os.linesep)
self.logger.info('----> Get body message %s and start searching this key word...<----' % key_word)
try:
url = 'https://play.google.com/store/search?q=%s&c=apps' % key_word
search_web_driver = SearchWebDriver(url, self._db_conn, self.logger)
search_web_driver.search()
except Exception:
self.logger.exception('Search key word %s error' % key_word)
channel.basic_ack(delivery_tag=method.delivery_tag)
self.logger.info('Set key word %s as consumed' % key_word)
self._set_key_word_consumed(key_word)
def _conn_db(self):
try:
self._db_conn = util.conn_mysql_db()
except Exception:
self.logger.exception('Connect to database error')
def _set_key_word_consumed(self, key_word):
query = 'UPDATE key_word SET status=%s WHERE key_word=%s'
try:
MySQLDBUtil.update(query, (CONSUMED, key_word), self._db_conn)
except Exception:
self.logger.exception('Set key word %s as consumed error' % key_word)
class PortManager:
# Port Manager: calls necessary managers and utilities to generate parameters for sql.
# List of valid ports it can receive is taken from the Configuration setup.
#
validPortNumbers = ()
def __init__(self):
self.g_config = GlobalConfig()
self.validPortNumbers = self.g_config.get_ports()
self.date_time_field = self.g_config.get_db_datetime_name()
self.log = Logger().get('reportserver.manager.PortManager.PortManager')
def isPortValid(self, port_number):
if (port_number in self.validPortNumbers):
return True
else:
return False
def getPort(self, port_number, uom, unit):
self.log.info("Retrieving port:" + str(port_number) + "uom:" + uom + " size: " + str(unit))
items = []
if self.isPortValid(port_number):
results = DatabaseHandler().get_json_by_time(port_number, uom, unit)
items = utilities.process_data(results)
port_json = {
'port': str(port_number),
'timespan': uom + "=" + str(unit),
'items':items
}
return port_json
def get_port_attack_count(self, tablename, unit, uom):
fromDate = dateTimeUtility.get_begin_date_iso(unit, uom)
sql = "select count(distinct session) as total_attacks from %s where %s >= '%s' " %(tablename, self.date_time_field, fromDate)
self.log.debug("sql is:" + sql)
result = DatabaseHandler().query_db(sql)[0]
return int(result['total_attacks'])
def get_unique_ips(self, tablename, unit, uom):
fromDate = dateTimeUtility.get_begin_date_iso(unit, uom)
sql = "select count(distinct peerAddress) as unique_ips from %s where %s >= '%s' " % (tablename, self.date_time_field, fromDate)
self.log.debug("sql is:" + sql)
result = DatabaseHandler().query_db(sql)[0]
return int(result['unique_ips'])
class DrawLots(object):
def __init__(self):
self.log = Logger()
'''获取抽签列表'''
csv_file = csv.reader(open(DRAW_LOTS_WORDS_PATH, 'r', encoding='UTF-8'))
for word in csv_file:
lots_dict[word[0]] = word[1]
words_dict[word[0]] = word[2]
def play(self):
draw_random_num = random.randint(1, len(lots_dict)) - 1
self.log.debug('draw_random_num:{}'.format(draw_random_num))
return lots_dict[str(draw_random_num)], words_dict[str(draw_random_num)]
class IpsServiceHandler:
def __init__(self):
self.log = Logger().get("reportserver.manager.IpsServiceHandler.py")
def process(self, rqst, path_tokens, query_tokens):
uom = None
units = None
self.log.info("processing ipaddress request:" + str(path_tokens) + str(query_tokens))
try:
time_period = utilities.validate_time_period(query_tokens)
uom = time_period[0]
units = time_period[1]
except ValueError:
rqst.badRequest(units)
return
if len(path_tokens) == 5:
ipaddress = path_tokens[4].strip()
self.log.debug("requested: " + str(ipaddress))
if ipaddress is not None or ipaddress is not "":
try:
ipaddress = utilities.validate_ipaddress(ipaddress)
self.get_ips_data_by_time(rqst, ipaddress, uom, units)
except ValueError:
rqst.badRequest(badIpAddress)
return
elif ipaddress == None or ipaddress == "":
self.get_ips_data_by_time(rqst, "", uom, units)
else:
rqst.badRequest()
return
elif len(path_tokens) == 4:
self.get_ips_list_json(rqst, uom, units)
else:
rqst.badRequest()
return
def get_ips_data_by_time(self, rqst, ipaddress, uom, units):
ips_manager = IpsManager()
addressjsondata = ips_manager.get_data(ipaddress, uom, units)
if addressjsondata is not None:
# send response:
rqst.sendJsonResponse(addressjsondata, 200)
else:
rqst.notFound()
def get_ips_list_json(self, rqst, uom, units):
response = "{not implemented yet.}"
rqst.sendJsonResponse(response, 200)
class DatabaseHandler:
def __init__(self):
self.global_config = GlobalConfig()
self.db_path = self.global_config['Database']['path']
self.log = Logger().get('reportserver.dao.DatabaseHandler.DatabaseHandler')
# Connect to given database.
# Defaults to the honeypot db, but another path can be passed in (mainly for testing).
# Database needs to exist first.
def connect(self, database_name):
if (database_name == None):
database_name = self.db_path
if not os.path.exists(database_name):
self.log.error("Database does not exist in path: " + database_name)
return None
try:
conn = sqlite3.connect(database_name)
except sqlite3.OperationalError as oe:
self.log.error("****Problem connecting to database*** at: " + database_name)
self.log.error(oe)
else:
return conn
# Query DB and return JSON
def query_db(self, query, args=(), one=False, db=None):
#print ("#debug args are: " +str(args))
cur = self.connect(db).cursor()
cur.execute(query, args)
r = [dict((cur.description[i][0], value) \
for i, value in enumerate(row)) for row in cur.fetchall()]
cur.connection.close()
return (r[0] if r else None) if one else r
# Unit of Measure could be "weeks", "days", "hours", "minutes".
# Return all data from the DB within that measure of time as JSON.
def get_json_by_time(self, portnumber, uom, units):
begin_date_iso = dateTimeUtility.get_begin_date_iso(uom, units)
tableName = self.global_config.get_plugin_config(portnumber)['table']
date_time_field = self.global_config.get_db_datetime_name()
# query = query_db("SELECT * FROM %s where (datetime > '%s')" % (tableName, query_date_iso))
queryString = "SELECT * FROM %s where %s >= '%s' order by id, %s" % (tableName, date_time_field, begin_date_iso, date_time_field)
#args = (tableName, date_time_field, begin_date_iso)
self.log.info("queryString is: " + str(queryString))
#print ("args to use: " + str(args))
results = self.query_db(queryString)
self.log.debug("results: " + str(results))
return results
class DataManager(Thread):
"""
This is the DataManager class, it creates the database, data queue and
the condition variable for synchronization between it, the framework and
the plugins
"""
def __init__(self):
super().__init__()
self.db = Database()
self.db.create_default_database()
self.q = DataQueue()
self.condition = Condition()
self.kill = False
self.logger = Logger().get('database.datamanager.DataManager')
def run(self):
"""
This will insert all data in the queue and then once finished give up
control of the condition variable
"""
while not self.kill:
self.condition.acquire()
if self.q.check_empty():
self.condition.wait()
while not self.q.check_empty():
value = self.q.get_next_item()
Table_Insert.prepare_data_for_insertion(
self.q.dv.table_schema, value)
self.condition.notify()
self.condition.release()
def insert_data(self, data):
"""
Synchronously inserts data into the database.
:param data: A dictionary with a table name as its key and a dictionary
of column names and corresponding values as its value.
"""
self.condition.acquire()
if self.q.insert_into_data_queue(data):
self.condition.notify()
self.condition.release()
def shutdown(self):
self.kill = True
self.condition.acquire()
self.condition.notify()
self.condition.release()
self.join()
self.logger.debug('Data manager has shut down.')
class IpsManager:
# Ips Manager: calls necessary managers and utilities to generate parameters for sql.
#
validPortNumbers = ()
def __init__(self):
self.g_config = GlobalConfig()
self.valid_port_numbers = self.g_config.get_ports()
self.date_time_field = self.g_config.get_db_datetime_name()
self.log = Logger().get('reportserver.manager.IpsManager.py')
def get_data(self, ipaddress, uom, unit):
self.log.info("Retrieving ipaddress data: " + str(ipaddress) + " uom: " + uom + " size: " + str(unit))
port_data = []
for port in self.valid_port_numbers:
results = self.get_json_by_ip(port, ipaddress, uom, unit)
items = utilities.process_data(results)
port_data.append({port:items})
port_json = {
'ipaddress': str(ipaddress),
'timespan': uom + "=" + str(unit),
'ports':port_data
}
return port_json
def get_json_by_ip(self, portnumber, ipaddress, uom, units):
begin_date_iso = dateTimeUtility.get_begin_date_iso(uom, units)
table_name = self.g_config.get_plugin_config(portnumber)['table']
date_time_field = self.g_config.get_db_datetime_name()
# query = query_db("SELECT * FROM %s where (datetime > '%s')" % (tableName, query_date_iso))
queryString = "SELECT * FROM %s where %s >= '%s' and peerAddress = '%s' order by id, %s" % (
table_name, date_time_field, begin_date_iso, ipaddress, date_time_field)
# args = (tableName, date_time_field, begin_date_iso)
self.log.info("queryString is: " + str(queryString))
# print ("args to use: " + str(args))
results = DatabaseHandler().query_db(queryString)
self.log.debug("results: " + str(results))
return results
def get_cel_value_by_row_col(self, row, col):
"""
获取excel中单元格内容
:param row:
:param col:
:return:
"""
Logger.debug(
"TestDataReaderCommonUtil.get_cel_value_by_row_col 获取excel中单元格内容 "
"file name is : {0}, row:{1}, col:{2}".format(
self.file_name, row, col), self.__class__.__name__)
cell_type = type(self.read_excel_sheet().cell(row=row,
column=col).value)
cell_value = self.read_excel_sheet().cell(row=row, column=col).value
# 整数类型去除小数点，如dept_code，读取出来是3203.0格式的，去除小数点后为3203格式
if cell_type == float and cell_value % 1 == 0:
cell_value = int(cell_value)
return cell_value
def validate_time_period(query_tokens):
log = Logger().get('reportserver.manager.utilities')
log.debug("given query_tokens:" + str(query_tokens))
uom = None
units = None
for token in query_tokens:
if '=' in token:
uom,units = token.split('=')
if uom in UnitOfMeasure.get_values(UnitOfMeasure):
units = int(units)
break
else:
uom = None
units = None
#print("\n#debug validate_time_period: " + str(uom) + ": " + str(units))
return (uom, units)
def explode_multipolygons_to_polygons(
log: Logger, polygon_gdf: gpd.GeoDataFrame) -> gpd.GeoDataFrame:
row_accumulator = []
def explode_multipolygons(row):
if (row['geometry'].type == 'MultiPolygon'):
for geom in row['geometry'].geoms:
new_row = row.to_dict()
new_row['geometry'] = geom
row_accumulator.append(new_row)
else:
row_accumulator.append(row.to_dict())
polygon_gdf.apply(explode_multipolygons, axis=1)
gdf = gpd.GeoDataFrame(row_accumulator, crs=CRS.from_epsg(3879))
if (len(polygon_gdf) != len(gdf)):
log.debug(
f'Exploaded {len(gdf)} polygons from {len(polygon_gdf)} multipolygons'
)
return gdf
class ReadLocator:
"""
所有定位器写在yaml文件中，实例化此类调用read()方法读取
:param filename: yaml文件名
:param page_number: 页数，yaml文件使用‘---’分割页数
"""
def __init__(self, filename: str, page_number: int):
self._filename = filename
if not self._filename.endswith('.yaml'):
self._filename = filename + '.yaml'
self._page_number = page_number
self._all_result = self._read_all_page()
self._log = Logger('读取定位器').get_logger()
def _read_all_page(self):
"""
读取指定页全部数据
:return: 指定页全部数据
"""
path = os.path.join(constant.yaml_path, self._filename)
with open(path, encoding='utf-8') as fp:
reader = fp.read()
all_result = yaml.load_all(reader, Loader=yaml.FullLoader)
return list(all_result)[self._page_number - 1]
def read(self, locator_name: str):
"""
根据给定名称获取定位器
:param locator_name: 定位器名称
:return: 定位器值
"""
locator = self._all_result.get(locator_name)
if locator:
self._log.debug(f'获取到定位器（{locator_name}）：{locator}')
return locator
else:
self._log.error('定位器不存在（{}）！！'.format(locator_name))
class OperateConfig:
"""
配置文件读写
:param file_path: 配置文件路径
"""
def __init__(self, file_path):
self._file_name_path = file_path
self._cg = configparser.ConfigParser()
self._cg.read(self._file_name_path, encoding='utf-8')
self._log = Logger('读取配置').get_logger()
def get_str(self, section, option):
"""
读取字符型参数
:param section: 类名
:param option: key
:return: value
"""
value = self._cg.get(section, option)
self._log.debug('获取到：{} = {}'.format(option, value))
return value
def get_int(self, section, option):
"""
读取数值型参数
:param section: 类名
:param option: key
:return: value
"""
value = self._cg.getint(section, option)
self._log.debug('获取到：{} = {}'.format(option, value))
return value
def get_bool(self, section, option):
"""
读取布尔型参数
:param section: 类名
:param option: key
:return: value
"""
value = self._cg.getboolean(section, option)
self._log.debug('获取到：{} = {}'.format(option, value))
return value
def write_data(self, section, option, value):
"""
把传入的参数写入到文件
:param section: 类名
:param option: key
:param value: value
:return: None
"""
self._cg.set(section, option, value)
with open(self._file_name_path, 'w') as fp:
self._cg.write(fp)
self._log.info('成功写入参数：{} = {}'.format(option, value))
class ReadApi:
"""
读取接口信息
:param filename: yaml文件名
"""
def __init__(self, filename: str):
self._filename = filename
if not self._filename.endswith('.yaml'):
self._filename = filename + '.yaml'
self._log = Logger('读取API信息').get_logger()
def read(self, api_name: str):
"""
读取指定接口信息
:param api_name: 接口名
:return: 返回字典，包含此接口所有信息
"""
path = os.path.join(constant.api_data_path, self._filename)
with open(path) as fp:
reader = fp.read()
all_ = yaml.load(reader, Loader=yaml.FullLoader)
single = all_.get(api_name)
self._log.debug('获取到接口{}：{}'.format(api_name, single))
return single
class IPInfoAgent(Thread):
def __init__(self, peer_address, framework, instance_name):
super().__init__()
self.log = Logger().get('recon.ipinfoagent.IPInfoAgent')
self.peer_address = peer_address
self.cache = IPInfoCache()
self.framework = framework
self.instance_name = instance_name
def run(self):
peer_info = self.cache[self.peer_address]
if not peer_info:
request_str = 'http://ipinfo.io/' + self.peer_address
self.log.debug('making REST request to ' + request_str)
response = requests.get(request_str, timeout=10)
peer_info = response.json()
peer_info['timestamp'] = datetime.now()
if 'loc' in peer_info:
lat, long = peer_info['loc'].split(',')
peer_info['lat'] = float(lat)
peer_info['long'] = float(long)
del(peer_info['loc'])
self.cache[self.peer_address] = peer_info
else:
self.log.debug('ipinfo.io data for ' +
self.peer_address +
' is still cached')
peer_info['plugin_instance'] = self.instance_name
clean_dict = {}
known_cols = ['ip', 'hostname', 'city', 'region', 'country', 'lat',
'long', 'org', 'postal', 'timestamp', 'plugin_instance']
for col in known_cols:
if col in peer_info:
clean_dict[col] = peer_info[col]
self.framework.insert_data({'ipInfo': clean_dict})
self.log.debug('ipinfo.io data for ' +
self.peer_address +
': ' +
str(peer_info))
class P0fAgent(Thread):
def __init__(self, peer_address, framework, session):
super().__init__()
self.config = GlobalConfig()
self.framework = framework
self.fs_sock = self.config['Framework']['p0f.fs_sock']
self.log = Logger().get('recon.p0fagent.P0fAgent')
self.peer_address = peer_address
self.session = session
def run(self):
p0f = P0f(self.fs_sock)
peer_info = None
try:
peer_info = p0f.get_info(self.peer_address)
except P0fException as e:
self.log.warn('p0f request failed for ' +
self.peer_address +
': ' + str(e))
return
except KeyError as e:
self.log.warn('p0f couldn\'t find any info for ' +
self.peer_address)
return
except ValueError as e:
self.log.warn('p0f returned bad data for ' +
self.peer_address)
return
except FileNotFoundError as e:
self.log.error('p0f filesystem socket not found')
return
except Exception as e:
self.log.error('unknown p0f error occurred on address ' +
self.peer_address + ': ' + str(e))
return
# prettify C/null-terminated byte arrays in p0f info dict
for key in peer_info.keys():
if type(peer_info[key]) == bytes:
decoded = peer_info[key].decode('utf-8')
peer_info[key] = decoded.partition('\x00')[0]
elif type(peer_info[key]) == datetime:
peer_info[key] = peer_info[key].isoformat()
elif type(peer_info[key]) == timedelta:
peer_info[key] = str(int(peer_info[key].total_seconds()))
elif type(peer_info[key]) == int:
peer_info[key] = str(peer_info[key])
data = { 'p0f': {
'session': self.session,
'first_seen': peer_info['first_seen'],
'last_seen': peer_info['last_seen'],
'uptime': peer_info['uptime'],
'last_nat': peer_info['last_nat'],
'last_chg': peer_info['last_chg'],
'distance': peer_info['distance'],
'bad_sw': peer_info['bad_sw'],
'os_name': peer_info['os_name'],
'os_flavor': peer_info['os_flavor'],
'os_match_q': peer_info['os_match_q'],
'http_name': peer_info['http_name'],
'http_flavor': peer_info['http_flavor'],
'total_conn': peer_info['total_conn'],
'link_type': peer_info['link_type'],
'language': peer_info['language']
}
}
self.framework.insert_data(data)
self.log.debug('p0f info for ' +
self.peer_address +
': ' +
str(peer_info))
def convert_otp_graph_to_igraph(
node_csv_file: str,
edge_csv_file: str,
hma_poly_file: str,
igraph_out_file: str,
b_export_otp_data_to_gpkg: bool = False,
b_export_decomposed_igraphs_to_gpkg: bool = False,
b_export_final_graph_to_gpkg: bool = False,
debug_otp_graph_gpkg: str = 'debug/otp_graph_features.gpkg',
debug_igraph_gpkg: str = 'debug/otp2igraph_features.gpkg',
log: Logger = Logger(printing=True)
) -> dict:
hma_poly = geom_utils.project_geom(
gpd.read_file(hma_poly_file)['geometry'][0])
# 1) read nodes nodes from CSV
n = pd.read_csv(node_csv_file, sep=';')
log.info(f'read {len(n.index)} nodes')
log.debug(f'node column types: {n.dtypes}')
log.debug(f'nodes head: {n.head()}')
log.info('creating node gdf')
n[Node.geometry.name] = [
shapely.wkt.loads(geom) if isinstance(geom, str) else Point()
for geom in n[Node.geometry.name]
]
n[Node.geom_wgs.name] = n[Node.geometry.name]
n = gpd.GeoDataFrame(n,
geometry=Node.geometry.name,
crs=CRS.from_epsg(4326))
log.info('reprojecting nodes to etrs')
n = n.to_crs(epsg=3879)
log.debug(f'nodes head: {n.head()}')
# 2) read edges from CSV
e = pd.read_csv(edge_csv_file, sep=';')
log.info(f'read {len(e.index)} edges')
log.debug(f'edge column types: {e.dtypes}')
log.debug(f'edges head: {e.head()}')
log.info('creating edge gdf')
e[Edge.geometry.name] = [
shapely.wkt.loads(geom) if isinstance(geom, str) else LineString()
for geom in e[Edge.geometry.name]
]
e[Edge.geom_wgs.name] = e[Edge.geometry.name]
e = gpd.GeoDataFrame(e,
geometry=Edge.geometry.name,
crs=CRS.from_epsg(4326))
log.info('reprojecting edges to etrs')
e = e.to_crs(epsg=3879)
log.debug(f'edges head: {e.head()}')
# 3) export graph data to gpkg
if (b_export_otp_data_to_gpkg == True):
log.info('writing otp graph data to gpkg')
e.drop(columns=[Edge.geom_wgs.name]).to_file(debug_otp_graph_gpkg,
layer='edges',
driver='GPKG')
log.info(f'exported edges to {debug_otp_graph_gpkg} (layer=edges)')
n.drop(columns=[Edge.geom_wgs.name]).to_file(debug_otp_graph_gpkg,
layer='nodes',
driver='GPKG')
log.info(f'exported nodes to {debug_otp_graph_gpkg} (layer=nodes)')
# 4) filter out edges that are unsuitable for both walking and cycling
def filter_df_by_query(df: pd.DataFrame, query: str, name: str = 'rows'):
count_before = len(df.index)
df_filt = df.query(query).copy()
filt_ratio = (count_before - len(df_filt.index)) / count_before
log.info(
f'filtered out {count_before-len(df_filt.index)} {name} ({round(filt_ratio * 100, 1)} %) by {query}'
)
return df_filt
e_filt = filter_df_by_query(
e,
f'{Edge.allows_walking.name} == True or {Edge.allows_biking.name} == True',
name='edges')
e_filt = filter_df_by_query(e_filt,
f'{Edge.is_no_thru_traffic.name} == False',
name='edges')
# 5) create a dictionaries for converting otp ids to ig ids and vice versa
log.debug('create maps for converting otp ids to ig ids')
n[Node.id_ig.name] = np.arange(len(n.index))
ids_otp_ig = {}
ids_ig_otp = {}
for node in n.itertuples():
ids_otp_ig[getattr(node,
Node.id_otp.name)] = getattr(node, Node.id_ig.name)
ids_ig_otp[getattr(node,
Node.id_ig.name)] = getattr(node, Node.id_otp.name)
# 6) add nodes to graph
log.info('adding nodes to graph')
G = ig.Graph(directed=True)
G.add_vertices(len(n.index))
for attr in Node:
if (attr.name in n.columns):
G.vs[attr.value] = list(n[attr.name])
else:
log.warning(f'node column {attr.name} not present in dataframe')
# 7) add edges to graph
log.info('adding edges to graph')
# get edge lengths by projected geometry
e_filt[Edge.length.name] = [
round(geom.length, 4) if isinstance(geom, LineString) else 0.0
for geom in e_filt[Edge.geometry.name]
]
def get_ig_uv(edge):
return (ids_otp_ig[edge['node_orig_id']],
ids_otp_ig[edge['node_dest_id']])
e_filt['uv_ig'] = e_filt.apply(lambda row: get_ig_uv(row), axis=1)
e_filt[Edge.id_ig.name] = np.arange(len(e_filt.index))
G.add_edges(list(e_filt['uv_ig']))
for attr in Edge:
if (attr.name in e_filt.columns):
G.es[attr.value] = list(e_filt[attr.name])
else:
log.warning(f'edge column {attr.name} not present in dataframe')
# 8) delete edges outside Helsinki Metropolitan Area (HMA)
hma_buffered = hma_poly.buffer(100)
def intersects_hma(geom: LineString):
if (geom.is_empty == True): return True
return True if geom.intersects(hma_buffered) else False
e_gdf = ig_utils.get_edge_gdf(G)
log.info('finding edges that intersect with HMA')
e_gdf['in_hma'] = [
intersects_hma(line) for line in e_gdf[Edge.geometry.name]
]
e_gdf_del = e_gdf.query('in_hma == False').copy()
out_ratio = round(100 * len(e_gdf_del.index) / len(e_gdf.index), 1)
log.info(f'found {len(e_gdf_del.index)} ({out_ratio} %) edges outside HMA')
log.info('deleting edges')
before_count = G.ecount()
G.delete_edges(e_gdf_del.index.tolist())
after_count = G.ecount()
log.info(f'deleted {before_count-after_count} edges')
# check if id_ig:s need to be updated to edge attributes
mismatch_count = len([
edge.index for edge in G.es
if edge.attributes()[Edge.id_ig.value] != edge.index
])
log.info(f'invalid edge ids: {mismatch_count}')
# reassign igraph indexes to edge and node attributes
G.es[Edge.id_ig.value] = [e.index for e in G.es]
G.vs[Node.id_ig.value] = [v.index for v in G.vs]
# check if id_ig:s need to be updated to edge attributes
mismatch_count = len([
edge.index for edge in G.es
if edge.attributes()[Edge.id_ig.value] != edge.index
])
log.info(f'invalid edge ids: {mismatch_count} (after re-indexing)')
# 9) find and inspect subgraphs by decomposing the graph
sub_graphs = G.decompose(mode='STRONG')
log.info(f'found {len(sub_graphs)} subgraphs')
graph_sizes = [graph.ecount() for graph in sub_graphs]
log.info(
f'subgraphs with more than 10 edges: {len([s for s in graph_sizes if s > 10])}'
)
log.info(
f'subgraphs with more than 50 edges: {len([s for s in graph_sizes if s > 50])}'
)
log.info(
f'subgraphs with more than 100 edges: {len([s for s in graph_sizes if s > 100])}'
)
log.info(
f'subgraphs with more than 500 edges: {len([s for s in graph_sizes if s > 500])}'
)
log.info(
f'subgraphs with more than 10000 edges: {len([s for s in graph_sizes if s > 10000])}'
)
small_graphs = [graph for graph in sub_graphs if graph.ecount() <= 15]
medium_graphs = [
graph for graph in sub_graphs
if (graph.ecount() > 15 and graph.ecount() <= 500)
]
big_graphs = [graph for graph in sub_graphs if graph.ecount() > 500]
small_graph_edges = []
for graph_id, graph in enumerate(small_graphs):
edges = ig_utils.get_edge_dicts(
graph, attrs=[Edge.id_otp, Edge.id_ig, Edge.geometry])
for edge in edges:
edge['graph_id'] = graph_id
small_graph_edges.extend(edges)
medium_graph_edges = []
for graph_id, graph in enumerate(medium_graphs):
edges = ig_utils.get_edge_dicts(
graph, attrs=[Edge.id_otp, Edge.id_ig, Edge.geometry])
for edge in edges:
edge['graph_id'] = graph_id
medium_graph_edges.extend(edges)
big_graph_edges = []
for graph_id, graph in enumerate(big_graphs):
edges = ig_utils.get_edge_dicts(
graph, attrs=[Edge.id_otp, Edge.id_ig, Edge.geometry])
for edge in edges:
edge['graph_id'] = graph_id
big_graph_edges.extend(edges)
if (b_export_decomposed_igraphs_to_gpkg == True):
log.info('exporting subgraphs to gpkg')
# graphs with <= 15 edges
small_graph_edges_gdf = gpd.GeoDataFrame(small_graph_edges,
crs=CRS.from_epsg(3879))
small_graph_edges_gdf.to_file(debug_igraph_gpkg,
layer='small_graph_edges',
driver='GPKG')
# graphs with 15–500 edges
medium_graph_edges_gdf = gpd.GeoDataFrame(medium_graph_edges,
crs=CRS.from_epsg(3879))
medium_graph_edges_gdf.to_file(debug_igraph_gpkg,
layer='medium_graph_edges',
driver='GPKG')
# graphs with > 500 edges
big_graph_edges_gdf = gpd.GeoDataFrame(big_graph_edges,
crs=CRS.from_epsg(3879))
big_graph_edges_gdf.to_file(debug_igraph_gpkg,
layer='big_graph_edges',
driver='GPKG')
log.info(f'graphs exported')
# 10) delete smallest subgraphs from the graph
del_edge_ids = [edge[Edge.id_ig.name] for edge in small_graph_edges]
log.info(f'deleting {len(del_edge_ids)} isolated edges')
before_count = G.ecount()
G.delete_edges(del_edge_ids)
after_count = G.ecount()
del_ratio = round(100 * (before_count - after_count) / before_count, 1)
log.info(f'deleted {before_count-after_count} ({del_ratio} %) edges')
# 11) delete isolated nodes from the graph
del_node_ids = [v.index for v in G.vs.select(_degree_eq=0)]
log.info(f'deleting {len(del_node_ids)} isolated nodes')
before_count = G.vcount()
G.delete_vertices(del_node_ids)
after_count = G.vcount()
del_ratio = round(100 * (before_count - after_count) / before_count, 1)
log.info(f'deleted {before_count-after_count} ({del_ratio} %) nodes')
# check if id_ig:s need to be updated to edge attributes
mismatch_count = len([
edge.index for edge in G.es
if edge.attributes()[Edge.id_ig.value] != edge.index
])
log.info(f'invalid edge ids: {mismatch_count}')
# reassign igraph indexes to edge and node attributes
G.es[Edge.id_ig.value] = [e.index for e in G.es]
G.vs[Node.id_ig.value] = [v.index for v in G.vs]
# check if id_ig:s need to be updated to edge attributes
mismatch_count = len([
edge.index for edge in G.es
if edge.attributes()[Edge.id_ig.value] != edge.index
])
log.info(f'invalid edge ids: {mismatch_count} (after re-indexing)')
# 12) export graph data to GeoDataFrames fro debugging
if (b_export_final_graph_to_gpkg == True):
log.info(f'exporting final graph to {debug_igraph_gpkg} for debugging')
e_gdf = ig_utils.get_edge_gdf(G,
attrs=[Edge.id_otp, Edge.id_ig],
ig_attrs=['source', 'target'])
n_gdf = ig_utils.get_node_gdf(G, ig_attrs=['index'])
e_gdf.to_file(debug_igraph_gpkg,
layer='final_graph_edges',
driver='GPKG')
n_gdf.to_file(debug_igraph_gpkg,
layer='final_graph_nodes',
driver='GPKG')
if (igraph_out_file != None and igraph_out_file != ''):
ig_utils.export_to_graphml(G, igraph_out_file)
return G
class DeveloperProducer:
def __init__(self):
self.logger = Logger(LOG_FILE, LOG_NAME, 10 * 1024 * 1024, 2)
self._db_conn = None
def start(self):
rabbit_topic = RabbitTopic.init_rabbitmq_producer(
EXCHANGE_NAME, self.logger)
if not rabbit_topic:
return
self._conn_db()
if not self._db_conn:
self.logger.exception('Connect database error')
return
while 1:
try:
if self._is_no_more_records():
self.logger.info('There are no more records, wait...')
time.sleep(PRODUCE_WAIT_TIME)
developer_list = self._fetch_developer_list()
for developer in developer_list:
if RabbitTopic.is_queue_full(QUEUE_NAME, QUEUE_LIMIT,
self.logger):
self.logger.info('Queue %s is full, wait...' %
QUEUE_NAME)
time.sleep(PRODUCE_WAIT_TIME)
continue
try:
rabbit_topic.publish(ROUTING_KEY, developer)
self.logger.info(
'Publish developer %s and update status' %
developer)
self._update_status(PUBLISHED, developer)
except ConnectionClosed:
self.logger.debug(
'Connection to rabbitmq server closed, re-connecting...'
)
rabbit_topic = RabbitTopic.init_rabbitmq_producer(
EXCHANGE_NAME, self.logger)
except Exception:
self.logger.exception('Publish developer error')
def _conn_db(self):
try:
self._db_conn = util.conn_mysql_db()
except Exception:
self.logger.exception('Connect database error')
def _fetch_developer_list(self):
developer_list = []
self.logger.info('Get un-published developer list...')
query = 'SELECT name FROM developer WHERE status=%s LIMIT %s'
try:
results = MySQLDBUtil.fetch_multiple_rows(
query, (UN_PUBLISHED, QUEUE_LIMIT), self._db_conn)
for result in results:
(developer, ) = result
developer_list.append(developer)
except Exception:
self.logger.exception('Query un-published developer error')
return developer_list
def _is_no_more_records(self):
query = 'SELECT COUNT(*) FROM developer WHERE status=%s'
try:
result = MySQLDBUtil.fetch_single_row(query, (UN_PUBLISHED, ),
self._db_conn)
if result:
(count, ) = result
if count == 0:
return True
except Exception:
self.logger.exception('Check if there is no more developers error')
return False
def _update_status(self, status, developer=None):
if not developer:
query = 'UPDATE developer SET status=%s'
else:
query = 'UPDATE developer SET status=%s WHERE name=%s'
try:
if not developer:
MySQLDBUtil.update(query, (status, ), self._db_conn)
else:
MySQLDBUtil.update(query, (status, developer), self._db_conn)
except Exception:
self.logger.exception('Update record status')
class PortsServiceHandler():
def __init__(self):
self.log = Logger().get('reportserver.server.PortServiceHandler.PortServiceHandler')
def process(self, rqst, path_tokens, query_tokens):
uom = None
units = None
self.log.info("processing ports request:" + str(path_tokens) + str(query_tokens))
if len(query_tokens) > 0:
try:
time_period = utilities.validate_time_period(query_tokens)
uom = time_period[0]
units = time_period[1]
except ValueError:
rqst.badRequest(units)
return
# default if we aren't given valid uom and units
if uom is None or units is None:
uom = "days"
units = 1
if len(path_tokens) == 5:
portNbr = utilities.validate_port_number(path_tokens[4])
self.log.debug("requested: " + str(portNbr))
if portNbr is not None and 0 < portNbr < 9000:
self.get_port_data_by_time(rqst, portNbr, uom, units)
elif portNbr == None or "":
self.get_port_list_json(rqst, uom, units)
else:
rqst.badRequest()
return
elif len(path_tokens) == 4:
self.get_port_list_json(rqst, uom, units)
else:
rqst.badRequest()
return
def get_port_list_json(self,rqst, uom, units):
jsondata = self.construct_port_summary_list(rqst, uom, units)
rqst.sendJsonResponse(jsondata, 200)
def get_port_data_by_time(self, rqst, portnumber, uom, units):
portmgr = PortManager()
portjsondata = portmgr.getPort(portnumber, uom, units)
if portjsondata is not None:
# send response:
rqst.sendJsonResponse(portjsondata, 200)
else:
rqst.notFound()
def construct_port_summary_list(self, rqst, uom, units):
g_config = GlobalConfig()
plugins_dictionary = g_config.get_plugin_dictionary()
json_list = []
for key, val in plugins_dictionary.items():
json_list.append(self.construct_port_summary(rqst, val['port'], val['table'], uom, units))
return json_list
def construct_port_summary(self, rqst, portnumber, tablename, uom, units):
portmgr = PortManager()
port_attacks = portmgr.get_port_attack_count(tablename, uom, units)
unique_ips = portmgr.get_unique_ips(tablename, uom, units)
timespan = uom+"="+str(units)
response_json = {
'port': str(portnumber),
'total_attacks': str(port_attacks),
'unique_ipaddresses': str(unique_ips),
'timespan':timespan,
'rel_link': rqst.get_full_url_path() + "/ports/" + str(portnumber)+"?" + timespan
}
return response_json
class BackFillToExcel:
"""
测试结果回写
"""
def __init__(self):
# 引用日志类
self._log = Logger('API测试结果回写').get_logger()
# 获取excel绝对路径
self._path = constant.api_result_excel_path
# 初始化判断，若excel文件不存在，就创建
if not os.path.exists(self._path):
BackupOrNewFile().create_result_file()
# 加载excel，并转到指定的sheet页，返回WorkSheet对象
self._wb = openpyxl.load_workbook(self._path)
self._ws = self._wb[self._wb.sheetnames[0]]
self._log.debug('成功加载测试结果文件：{}'.format(os.path.basename(self._path)))
self._log.debug('成功定位到回写数据表：{}'.format(self._wb.sheetnames[0]))
def save_excel(self):
"""
保存当前用例所有回填结果
:return: None
"""
try:
# self._merge_cells_before_save()
self._wb.save(self._path)
except PermissionError as e:
self._log.error('保存失败！！EXCEL文件被打开，请关闭后重新执行测试：{}'.format(e))
else:
self._log.debug('成功保存当前用例测试结果。')
def fill_api_name(self, api_name):
"""
回写接口名称
:param api_name: 接口名
:return: None
"""
self._ws.cell(self._ws.max_row, API_NAME_COL).value = api_name
self._log.debug('成功回写当前接口名称：{}'.format(api_name))
def fill_api_url(self, api_url):
"""
回写接口URL
:param api_url: URL
:return: None
"""
self._ws.cell(self._ws.max_row, API_URL_COL).value = api_url
self._log.debug('成功回写当前接口URL：{}'.format(api_url))
def fill_case_number(self, case_number):
"""
回写用例编号
注：此函数在基类的setUp方法内第一个调用，先确定回写行数，其他回写函数就可以最大行数为准写入
:param case_number: 用例编号
:return: None
"""
# 第一个调用，所以行数是 max_row + 1
self._ws.cell(self._ws.max_row + 1,
CASE_NUMBER_COL).value = case_number
self._log.debug('成功回写当前用例编号为{}，并确定当前用例回写行是第{}行'.format(
case_number, self._ws.max_row))
def fill_case_name(self, case_name):
"""
回写用例名称
:param case_name: 用例名
:return: None
"""
self._ws.cell(self._ws.max_row, CASE_NAME_COL).value = case_name
self._log.debug('成功回写当前用例名称：{}'.format(case_name))
def fill_judgement_result(self, result=1):
"""
回写判定结果
:param result: 0 失败，1 成功
:return: None
"""
if result:
self._ws.cell(self._ws.max_row,
JUDGEMENT_RESULT_COL).value = 'SUCCESS'
self._log.debug('默认回写当前用例执行判定结果为：SUCCESS')
else:
self._ws.cell(self._ws.max_row,
JUDGEMENT_RESULT_COL).value = 'FAILURE'
self._log.debug('当前用例断言失败或执行失败，改写判定结果为：FAILURE')
self._set_color_if_failure(self._ws.max_row)
def fill_excepted(self, excepted_result):
"""
回写预期结果
:param excepted_result: 预期结果
:return: None
"""
self._ws.cell(self._ws.max_row,
EXPECTED_RESULT_COL).value = str(excepted_result)
self._log.debug('成功回写当前用例预期结果：{}'.format(excepted_result))
def fill_compare_result(self, compare_result):
"""
回写比对结果
:param compare_result: 比对结果列表
:return: None
"""
if not compare_result:
self._ws.cell(self._ws.max_row,
COMPARE_RESULT_COL).value = '比对结果一致'
else:
self._ws.cell(self._ws.max_row,
COMPARE_RESULT_COL).value = str(compare_result)
self._log.debug('成功回写当前用例比对结果：{}'.format(compare_result))
def fill_response(self, response):
"""
回写返回JSON
:param response: JSON
:return: None
"""
self._ws.cell(self._ws.max_row, RESPONSE_COL).value = str(response)
self._log.debug('成功回写当前用例返回结果：{}'.format(response))
def fill_test_data(self, curr_case_data):
"""
回写本条用例测试数据
:param curr_case_data: 测试数据
:return: None
"""
self._ws.cell(self._ws.max_row,
CASE_DATA_COL).value = str(curr_case_data)
self._log.debug('成功回写当前用例测试数据：{}'.format(curr_case_data))
def _set_color_if_failure(self, row):
"""
把执行失败的用例背景色填充为红色
:param row: 行数
:return: None
"""
fill = PatternFill(fill_type='solid', fgColor="FF0000")
for col in range(API_NAME_COL, CASE_DATA_COL + 1):
self._ws.cell(row, col).fill = fill
self._log.debug('当前用例执行失败标记为红色！')
def _merge_cells_before_save(self):
"""
在保存之前调用，用于合并单元格（用例名和URL）
:return: None
"""
# 确定最小合并行
min_row_ = self._ws.max_row - self._ws.cell(self._ws.max_row,
CASE_NUMBER_COL).value + 1
# 合并单元格
self._ws.merge_cells(start_row=min_row_,
start_column=API_NAME_COL,
end_row=self._ws.max_row,
end_column=API_NAME_COL)
self._ws.merge_cells(start_row=min_row_,
start_column=API_URL_COL,
end_row=self._ws.max_row,
end_column=API_URL_COL)
# 设置垂直居中
align = Alignment(vertical='center')
self._ws.cell(min_row_, API_NAME_COL).alignment = align
self._ws.cell(min_row_, API_URL_COL).alignment = align
class _Framework:
def __init__(self, plugin_cfg_path, global_cfg_path):
self._global_config = GlobalConfig(plugin_cfg_path, global_cfg_path)
self._plugin_imports = {}
self._listener_list= {}
self._running_plugins_list = []
self._data_manager = None
self._shutting_down = False
self._log = None
self._pid = os.getpid()
def start(self):
self.set_shutdown_hook()
print('Press Ctrl+C to exit.')
if not self.drop_permissions():
return
self._global_config.read_global_config()
self.start_logging()
self._global_config.read_plugin_config()
self._data_manager = DataManager()
self._data_manager.start()
self.start_listeners()
def start_logging(self):
log_path = self._global_config['Framework']['logName']
log_level = self._global_config['Framework']['logLevel']
self._log = Logger(log_path, log_level).get('framework.frmwork.Framework')
self._log.info('RECCE7 started (PID %d)' % self._pid)
@staticmethod
def drop_permissions():
if os.getuid() != 0:
return True
dist_name = os.getenv('RECCE7_OS_DIST')
users_dict = {
'centos': ('nobody', 'nobody'),
'debian': ('nobody', 'nogroup')
}
if dist_name not in users_dict:
print(
'Unable to lower permission level - not continuing as\n'
'superuser. Please set the environment variable\n'
'RECCE7_OS_DIST to one of:\n\tcentos\n\tdebian\n'
'or rerun as a non-superuser.')
return False
lowperm_user = users_dict[dist_name]
nobody_uid = pwd.getpwnam(lowperm_user[0]).pw_uid
nogroup_gid = grp.getgrnam(lowperm_user[1]).gr_gid
os.setgroups([])
os.setgid(nogroup_gid)
os.setuid(nobody_uid)
os.umask(0o077)
return True
def create_import_entry(self, port, name, clsname):
imp = import_module('plugins.' + name)
self._plugin_imports[port] = getattr(imp, clsname)
def start_listeners(self):
ports = self._global_config.get_ports()
for port in ports:
plugin_config = self._global_config.get_plugin_config(port)
module = plugin_config['module']
clsname = plugin_config['moduleClass']
self.create_import_entry(port, module, clsname)
address = self._global_config['Framework']['listeningAddress']
listener = NetworkListener(address, plugin_config, self)
listener.start()
self._listener_list[port] = listener
def set_shutdown_hook(self):
signal.signal(signal.SIGINT, self.shutdown)
def shutdown(self, *args):
self._shutting_down = True
self._log.debug('Shutting down network listeners')
for listener in self._listener_list.values():
listener.shutdown()
self._log.debug('Shutting down plugins')
for plugin in self._running_plugins_list:
plugin.shutdown()
self._log.debug('Shutting down data manager')
self._data_manager.shutdown()
print('Goodbye!')
#
# Framework API
#
def get_config(self, port):
"""
Returns the configuration dictionary for the plugin
running on the specified port.
:param port: a port number associated with a loaded plugin
:return: a plugin configuration dictionary
"""
return self._global_config.get_plugin_config(port)
def spawn(self, socket, config):
"""
Spawns the plugin configured by 'config' with the provided
(accepted) socket.
:param socket: an open, accepted socket returned by
socket.accept()
:param config: the plugin configuration dictionary describing
the plugin to spawn
:return: a reference to the plugin that was spawned
"""
# ToDo Throw exception if plugin class not found
plugin_class = self._plugin_imports[config['port']]
plugin = plugin_class(socket, config, self)
plugin.start()
self._running_plugins_list.append(plugin)
return plugin
def insert_data(self, data):
"""
Inserts the provided data into the data queue so that it can
be pushed to the database.
:param data: data object to add to the database
"""
self._data_manager.insert_data(data)
def plugin_stopped(self, plugin):
"""
Tells the framework that the specified plugin has stopped
running and doesn't need to be shutdown explicitly on program
exit.
:param plugin: a reference to a plugin
"""
if self._shutting_down:
return
self._running_plugins_list.remove(plugin)
class CategoryProducer:
def __init__(self):
self.logger = Logger(LOG_FILE, LOG_NAME, 10*1024*1024, 2)
self._db_conn = None
def start(self):
rabbit_topic = RabbitTopic.init_rabbitmq_producer(EXCHANGE_NAME, self.logger)
if not rabbit_topic:
return
self._conn_db()
if not self._db_conn:
self.logger.exception('Connect database error')
return
while 1:
try:
if self._is_no_more_records():
self.logger.info('There are no more records, wait...')
time.sleep(PRODUCE_WAIT_TIME)
# Still no more available records, then reset...
if self._is_no_more_records():
self.logger.info('Still there are no more records, reset all records...')
self._reset_category()
category_list = self._fetch_category_list()
for category in category_list:
if RabbitTopic.is_queue_full(QUEUE_NAME, QUEUE_LIMIT, self.logger):
self.logger.info('Queue %s is full, wait...' % QUEUE_NAME)
time.sleep(PRODUCE_WAIT_TIME)
continue
try:
rabbit_topic.publish(ROUTING_KEY, category)
self.logger.info('Publish category %s and update the status' % category)
self._update_status(PUBLISHED, category)
except ConnectionClosed:
self.logger.debug('Connection to rabbitmq server closed, re-connecting...')
rabbit_topic = RabbitTopic.init_rabbitmq_producer(EXCHANGE_NAME, self.logger)
except Exception:
self.logger.exception('Publish category error')
def _conn_db(self):
try:
self._db_conn = util.conn_mysql_db()
except Exception:
self.logger.exception('Connect database error')
def _fetch_category_list(self):
category_list = []
self.logger.info('Get un-published category list...')
query = 'SELECT category FROM category WHERE status=%s LIMIT %s'
try:
results = MySQLDBUtil.fetch_multiple_rows(query, (UN_PUBLISHED, QUEUE_LIMIT), self._db_conn)
for result in results:
(category,) = result
category_list.append(category)
except Exception:
self.logger.exception('Query un-published category error')
return category_list
def _is_no_more_records(self):
query = 'SELECT COUNT(*) FROM category WHERE status=%s'
try:
result = MySQLDBUtil.fetch_single_row(query, (UN_PUBLISHED,), self._db_conn)
if result:
(count,) = result
if count == 0:
return True
except Exception:
self.logger.exception('Check available records error')
return False
def _reset_category(self):
try:
self.logger.info('All category have been published, reset all to un-published status')
self._update_status(UN_PUBLISHED)
except Exception:
self.logger.exception('Reset category table error')
return
def _update_status(self, status, category=None):
if not category:
query = 'UPDATE category SET status=%s'
else:
query = 'UPDATE category SET status=%s WHERE category=%s'
try:
if not category:
MySQLDBUtil.update(query, (status,), self._db_conn)
else:
MySQLDBUtil.update(query, (status, category), self._db_conn)
except Exception:
self.logger.exception('Update record status')
def get_row_count(self):
"""获取excel的行数"""
Logger.debug(
"TestDataReaderCommonUtil.get_row_count 获取excel的行数 file name is : "
+ self.file_name, self.__class__.__name__)
return self.read_excel_sheet().max_row
class Replier(object):
"""消息回复"""
level_map = {
1: "隔壁村的幼稚鬼",
2: "村子里的弟中弟",
3: "村民中的初学者",
4: "村子里的高中生",
5: "合格的成年村民",
}
q_a_list = {
"生日": ["杨超越的生日是什么时候 a)7月31 b)7月29 c)7月28", "a"],
"演艺": ["杨超越获得了哪年的演艺人物奖？a)2017 b)2018 c)2019", "b"],
"主食": ["杨超越最新欢的主食是？a)土豆 b)米饭 c)燕麦片", "b"]
}
jokes = [
'''美国外交代表团到苏联访问，苏修接待官员陪他们参观“建设的伟大成就”，并且得意的说：“到了下一个五年计划，每个苏联家庭都可以拥有一架私人飞机！”\n美国人惊讶的问：“ 他们要飞机干什么呢？”\n苏修官员说：“当然有用啊……譬如你在莫斯科听说列宁格勒开始供应面包了，你可以马上开飞机赶去排队啊。''',
'''斯大林、赫鲁晓夫和勃列日涅夫乘坐火车出门。开着开着，火车突然停了。\n斯大林把头伸出车窗外，怒吼道：“枪毙火车司机！”可是车还是没有动。\n接着赫鲁晓夫说：“给火车司机恢复名誉！”车仍然没有动。\n勃列日涅夫说：“同志们，不如拉上窗帘，坐在座位上自己摇动身体，做出列车还在前进的样子……”''',
'''美术馆里有一幅描写亚当和夏娃的画。\n一个英国人看了，说：“他们一定是英国人，男士有好吃的东西就和女士分享。\n一个法国人看了，说：“他们一定是法国人，情侣裸体散步。\n一个苏联人看了，说：“他们一定是苏联人，他们没有衣服，吃得很少，却还以为自己在天堂！''',
'''杨村制度的优越性在哪里？\n成功克服了其他制度里不存在的问题。''',
'''黑子：哈哈哈，杨超越这么重要的场合居然口误？\n路人：什么场合？\n黑子：不知道？\n路人：那你看了这么久的视频在看什么？\n黑子：在看杨超越！'''
]
def __init__(self, bot):
# self.group = group
self.api_key = api_key
self.log = Logger()
self.ycy = YcyReplier()
self.tuling = Tuling(api_key=TULING_KEY)
self.user = User()
self.group = Group()
self.rsp_game_player_map = {}
self.bot = bot
self.draw_lots_game = DrawLots()
self.real_estate = RealEstate()
self.user_lots_map = {}
self.user_lots_read_map = {}
self.answer = ""
self.red_bag_num = 0
'''
开启每日定时器，每日零时清空抽签内容
'''
self.scheduler = BackgroundScheduler()
self.scheduler.add_job(self.init_lots_map, 'cron', hour='0')
self.scheduler.start()
def init_lots_map(self):
"""
清空抽签内容，可以重新开始抽签
"""
self.user_lots_map = {}
self.user_lots_read_map = {}
self.log.info("=== Init Lots ===")
def random_img(self, msg) -> tuple:
"""
随机获取图片
:return:
"""
print("===天降超越===")
print(msg.text)
if "天降超越" in msg.text or "天将超越" in msg.text: # todo 待增加
list_dir = os.listdir(os.path.join('resources', 'pics'))
path = choice(list_dir)
self.log.info('choose:-->{}'.format(path))
self.log.debug(os.path.join('resources', 'pics', path))
return 'img', os.path.join('resources', 'pics', path), ''
return empty_result
def robot_init(self, msg) -> tuple:
"""
机器人初始化
:param msg:
:return:
"""
real_msg = msg.text.split()
if msg.member.puid == self.group.admin_puid and len(
real_msg) != 1: # 如果是管理员
if real_msg[len(real_msg) - 1] == "初始化":
self.log.info(msg.sender)
# self.group.update_group(msg.sender, self.api_key)
self.user.update_users(msg.sender, self.api_key)
self.log.info("初始化完成！")
return 'text', "初始化完成！", ''
elif real_msg[1] == "口令红包":
self.log.info("设置口令红包！")
print("===口令红包信息===")
print(real_msg[2])
print(real_msg[3])
try:
self.red_bag_num = int(real_msg[2])
except:
self.red_bag_num = 0
if real_msg[3] in self.q_a_list:
item = self.q_a_list[real_msg[3]]
self.answer = item[1]
return 'text', item[0], ''
else:
self.answer = real_msg[3]
return 'text', "口令红包设置完成！", ''
else:
return empty_result
return empty_result
def update_user_info(self, msg):
self.log.info("更新用户信息中……")
self.user.update_users(msg.sender, self.api_key)
self.log.info("用户信息更新完毕……")
def chaoyue_ana(self, msg) -> tuple:
"""
超越语录
:return:
"""
real_msg = msg.text.split()
respond_msg = self.ycy.reply_text(real_msg[len(real_msg) -
1]) # 超越语录无需要@
if respond_msg:
return 'text', '@' + msg.member.name + ' ' + respond_msg, ''
return empty_result
def set_group(self, puid):
self.group.set_group(puid)
def handle_leave_message(self, msg) -> tuple:
"""
处理留言
:param msg:
:return:
"""
if "村头留言板" in msg.text:
return "text", "@我并回复:「留言:你想说的话」，则可在村头留言板上留下你的留言内容哦", ""
is_leave_message = re.search(r'(留言:|留言：)(.*)', msg.text)
if is_leave_message:
content = is_leave_message.group(2).strip() # 获取第二组内容并去除前后空格
self.log.info('留言内容:{}'.format(content))
status = create_messages(
name=msg.member.name,
content=content,
fans_id=msg.member.puid,
)
if status == "ok":
return 'text', '@' + msg.member.name + ' ' + "留言成功！点击 {} 可查看你的留言".format(
'http://ahasmarter.com/', ), ''
else:
return 'text', '@' + msg.member.name + ' ' + "留言失败！稍后再尝试吧", ''
return empty_result
def get_group_introduction(self, msg) -> tuple:
"""
获取群介绍
:param msg:
:return:
"""
real_msg = msg.text.split()
if real_msg[len(real_msg) - 1] == "群信息" or real_msg[len(real_msg) -
1] == "群简介":
return 'text', self.group.intro, ''
return empty_result
def finger_guessing_game(self, msg) -> tuple:
"""
猜拳游戏
:param msg:
:return:
"""
group_id = msg.member.group.puid # 群组唯一id
name = msg.member.name # 玩家名
user_id = msg.member.puid # 玩家id
real_msg = msg.text.split()
if real_msg[len(real_msg) - 1] == "石头剪刀布" or real_msg[len(real_msg) - 1] == "剪刀石头布" \
or real_msg[len(real_msg) - 1] == "猜拳":
self.log.debug('---init猜拳----')
# { 群组id : {玩家id: [游戏对象 , 开始游戏的时间, 玩家名]}}
self.rsp_game_player_map.update(
{
group_id: [user_id,
RspGame(1),
now_to_datetime4(), name],
}, )
self.rsp_game_player_map[group_id][1].start(name) # 开始游戏
return 'text', '@' + msg.member.name + \
" 石头剪刀布开始，你先出吧，赢了我有奖励哦(1局定胜)", ''
return empty_result
def play_game(self, msg) -> tuple:
"""
游戏
:param msg:
:return:
"""
if "超越猜拳" in msg.text:
return "text", "@我并回复你的出招(比如「剪刀」)就能跟我玩猜拳游戏，赢了我会奖励3积分，输了扣除3积分，如果积分不够则不会进行奖惩", ""
real_msg = msg.text.split()
if "石头" in real_msg[len(real_msg) - 1] or "剪刀" in real_msg[len(real_msg) - 1] \
or "布" in real_msg[len(real_msg) - 1]:
game = RspGame(1)
game.start(msg.member.name)
cancel, result, pic = game.play(msg)
return 'both', pic, result
else:
return empty_result
"""
group_id = msg.member.group.puid
user_id = msg.member.puid
player_map = self.rsp_game_player_map
self.log.info(player_map)
# 如果字典中包含群组id并且 玩家id在字典中
if player_map.get(group_id):
is_overtime = now_to_datetime4() > two_minutes_later(player_map[group_id][2])
self.log.info('游戏是否超时:%s' % is_overtime)
if is_overtime:
msg = '@' + str(player_map[group_id][3]) + ' 游戏已经超时自动终止了呀!'
msg.chat.send_msg(msg)
player_map.pop(group_id) # 超时删除群组id对应的字典
if player_map.get(group_id): # 超时可能会pop掉该key,需要重新判断
if user_id not in player_map.get(group_id, []): # 不是玩家的消息，不进行回应
return 'text', '@' + msg.member.name + " 先等等哦，我正在跟@" + \
player_map[group_id][3] + " 玩石头剪刀布", ''
else:
cancel, result, pic = player_map[group_id][1].play(msg) # 玩游戏
self.log.debug('game result:{} pic:{}'.format(result, pic))
if cancel == 1:
player_map.pop(group_id) # 如果游戏结束, 删除群组id对应的字典
return 'both', pic, result
typ, content1, content2 = self.finger_guessing_game(msg) # 猜拳游戏
if typ == 'text':
return typ, content1, content2
return empty_result"""
def red_bag(self, msg) -> tuple:
if "口令红包" in msg.text:
return "text", "管理员会在某些时间在群里发出超越百科抢答红包，回答正确会得到超越积分，多多留意~", ""
real_msg = msg.text.split()
if self.red_bag_num == 0: #如果红包剩余数量为 0
self.answer = "" # answer清零
else:
print(self.answer)
if self.answer == real_msg[1] and msg.is_at:
user_puid = msg.member.puid
bot_id = self.bot.self.puid
result = self.user.transfer(bot_id, user_puid, self.group.puid,
3, self.api_key)
self.red_bag_num -= 1
if result["status"] == "success":
return 'text', " 口令正确！奖励给" + msg.member.name + " 3 个超越积分！", ''
else:
return 'text', '红包领完啦！', ''
return empty_result
def draw_lots(self, msg) -> tuple:
if "超越抽签" in msg.text:
return "text", "每日0点过后，@我并回复「抽签」，可以抽出你当日的运势签，@我并回复「解签」会解释抽签内容~", ""
real_msg = msg.text.split()
user_id = msg.member.puid
if real_msg[len(real_msg) - 1] == "抽签":
if user_id in self.user_lots_map:
return 'text', '@' + msg.member.name + ' 今日你的运势签: ' + self.user_lots_map[
user_id], ''
else:
msg1, msg2 = self.draw_lots_game.play()
self.user_lots_map[user_id] = msg1
self.user_lots_read_map[user_id] = msg2
return 'text', '@' + msg.member.name + ' 今日你的运势签: ' + msg1, ''
elif real_msg[len(real_msg) - 1] == "解签":
if user_id in self.user_lots_read_map:
return 'text', '@' + msg.member.name + ' 解签: ' + self.user_lots_read_map[
user_id], ''
else:
return 'text', '@' + msg.member.name + ' 今日还未进行抽签哦，请@我回复抽签', ''
else:
return empty_result
def reward(self, msg) -> tuple:
"""
打赏
:param msg:
:return:
"""
if str.find(msg.text, "打赏") != -1:
str_after_dashang = msg.text[str.find(msg.text, "打赏") + 3:].split()
to = self.user.find_user_by_name(msg.sender, str_after_dashang[0])
from_puid = msg.member.puid
self.log.info(from_puid)
self.log.info(to.puid)
result = self.user.transfer(
from_puid,
to.puid,
self.group.puid,
int(str_after_dashang[1], ),
self.api_key,
)
if result["status"] == "success":
payload = '打赏成功！' + msg.member.name + " 打赏给 " + \
to.name + " " + str_after_dashang[1] + "个超越积分！"
return 'text', payload, ''
else:
return 'text', '打赏失败！', ''
return empty_result
def integral(self, msg) -> tuple:
"""
积分相关
:return:
"""
real_msg = msg.text.split()
if real_msg[len(real_msg) - 1] == "超越积分":
msg = "超越积分可以用来干很多好玩的事情，后续推出虚拟房产和虚拟商店，可作为购买力进行交易哦，还支持个人打赏，@我并回复「余额」来查看你的积分总额。"
return 'text', msg, ''
if real_msg[len(real_msg) - 1] in ["余额", "积分"]:
user_puid = msg.member.puid
print("想拿余额的puid:")
print(user_puid)
balance = self.user.get_balance_by_puid(user_puid, self.group.puid)
msg = "你有" + str(balance) + "超越积分"
return 'text', msg, ''
if real_msg[len(real_msg) - 1] == "等级":
user_puid = msg.member.puid
level = self.user.get_level_by_puid(user_puid, self.group.puid)
msg = "你现在是" + str(level) + "级: " + self.level_map[int(level)]
return 'text', msg, ''
return empty_result
def houses(self, msg) -> tuple:
"""
房产相关
:return:
"""
real_msg = msg.text.split()
try:
if "超越买房" in msg.text:
return "text", "超越买房是实验性功能，@我并回复「看房」查看目前「超越大陆」上的房产所有者\n\n"+\
"@我并回复「买房 房产名 价格」可以进行房产购买，例如「@全村的希望 买房 火炉堡 30」\n"+\
"注意！！！你出的价格至少要比当前的价格大 1，才能买房成功 \n" +\
"如果你是房产所有者，@我并回复「房产名 签名：「你要签名的内容」」可进行签名，例如「@全村的希望 火炉堡 签名：靓仔」", ""
if real_msg[len(real_msg) - 1] == "看房":
print("=== 看房ing ===")
msg = self.real_estate.look()
return 'text', msg, ''
elif re.search(r'(签名:|签名：)(.*)', msg.text):
print("=== 签名ing ===")
house_name = real_msg[1]
print(msg.text)
signature = msg.text[msg.text.find("签名") + 3:]
print(house_name)
print(self.api_key)
print(msg.member.puid)
print(self.group.puid)
print(signature)
res = self.real_estate.leave_sig(msg.member.puid,
self.group.puid, signature,
house_name, self.api_key)
print(res)
if res["result"] == "success":
payload = "你在" + house_name + "上留下了你的签名：" + signature
return 'text', payload, ''
else:
payload = "签名失败！"
return 'text', payload, ''
elif real_msg[1] == "买房":
print("=== 买房ing ===")
house_name = real_msg[2]
amount = int(real_msg[3])
print(house_name)
print(amount)
res = self.real_estate.buy_house(msg.member.puid,
self.group.puid, house_name,
amount, self.api_key)
if res["result"] == "success":
payload = "买房成功！\n你现在是 " + house_name + " 的领主！"
return 'text', payload, ''
else:
payload = "买房失败！"
return 'text', payload, ''
return empty_result
except:
return empty_result
def extra(self, msg) -> tuple:
"""
额外添加
:param msg:
:return:
"""
real_msg = msg.text.split()
if real_msg[len(real_msg) - 1] in ["致谢", "鸣谢"]:
return 'text', "感谢「心理医生聪」与「禹sen」，提供超越语录的支持！", ''
if real_msg[len(real_msg) - 1] in ["帮助", "?", "？"]:
payload = "本 AI 目前支持以下功能: \n" + \
"- 超越积分\n" + \
"- 天降超越\n" + \
"- 超越猜拳\n" + \
"- 村头留言板\n" + \
"- 超越抽签\n" + \
"- 超越接龙\n" + \
"- 口令红包（管理员功能）\n" + \
"- 超越买房"
return 'text', payload, ''
if real_msg[len(real_msg) - 1] == "投票":
payload = "https://ke.qq.com/cates/ccyy/index.html?act_id=1&work_id=29&mmticket= ⬅ 欢迎猛戳链接投票"
return 'text', payload, ''
if real_msg[len(real_msg) - 1].find("笑话") != -1:
payload = choice(self.jokes)
return 'text', payload, ''
return empty_result
def handle_solo_msg(self, msg):
"""
处理私聊回复
:param msg:
:return:
"""
friend = msg.sender
# 向新的好友发送消息
friend.send('你好呀,我是全村的希望!')
friend.send('参与内测看我朋友圈的图片,扫二维码加群')
# friend.send_image('group.jpeg')
def handle_group_msg(self, msg) -> tuple:
"""
处理群组回复消息
:param msg:
:return:
"""
self.log.info('receive: %s' % msg.text)
typ, content1, content2 = self.reward(msg) # 打赏可能被@ 也可能不被@
if typ:
self.log.info(content1)
return typ, content1, content2
typ, content1, content2 = self.robot_init(msg) # 紧急情况下的初始化 以及口令红包的初始化
if typ:
self.log.info(content1)
return typ, content1, content2
typ, content1, content2 = self.red_bag(msg) # 口令红包
if typ:
self.log.info(content1)
return typ, content1, content2
typ, content1, content2 = self.random_img(msg) # 天降超越
if typ:
self.log.info(content1)
return typ, content1, content2
typ, content1, content2 = self.chaoyue_ana(msg) # 超越语录
if typ:
self.log.info(content1)
return typ, content1, content2
if msg.is_at: # 如果@到机器人，才进行的回应
typ, content1, content2 = self.play_game(
msg) # 玩游戏,高优先级,内部存在拦截其他回复
if typ:
self.log.info(content1)
user_puid = msg.member.puid
bot_id = self.bot.self.puid
user_balance = self.user.get_balance_by_puid(
user_puid, msg.member.group.puid)
bot_balance = self.user.get_balance_by_puid(
bot_id, msg.member.group.puid)
if user_balance < 3:
payload = " 由于你余额不足 3 积分，所以本次游戏没有奖惩哦~"
elif bot_balance < 3:
payload = " 超越宝宝的钱包瘪了，所以本次游戏没有奖惩哦~"
else:
if "游戏结束，恭喜你赢了" in content2:
from_puid = bot_id
print(from_puid)
to_puid = user_puid
result = self.user.transfer(from_puid, to_puid,
self.group.puid, 3,
self.api_key)
if result["status"] == "success":
payload = " 奖励给 " + msg.member.name + " 3 个超越积分！"
else:
payload = " 但是我没钱啦~"
elif "你输了" in content2:
from_puid = user_puid
to_puid = bot_id
result = self.user.transfer(from_puid, to_puid,
self.group.puid, 3,
self.api_key)
if result["status"] == "success":
payload = " 扣除 " + msg.member.name + " 3 个超越积分！"
else:
payload = " 你钱不够，接下来的游戏会没有奖励哦~"
else:
payload = ""
return typ, content1, content2 + payload
typ, content1, content2 = self.draw_lots(msg) # 抽签
if typ:
self.log.info(content1)
return typ, content1, content2
typ, content1, content2 = self.handle_leave_message(msg) # 处理留言请求
if typ:
self.log.info(content1)
return typ, content1, content2
typ, content1, content2 = self.get_group_introduction(msg) # 群简介
if typ:
self.log.info(content1)
return typ, content1, content2
typ, content1, content2 = self.integral(msg) # 积分相关
if typ:
self.log.info(content1)
return typ, content1, content2
typ, content1, content2 = self.houses(msg) # 积分相关
if typ:
self.log.info(content1)
return typ, content1, content2
typ, content1, content2 = self.extra(msg) # 额外信息
if typ:
self.log.info(content1)
return typ, content1, content2
tuling_reply = self.tuling.reply_text(msg).replace(
"图灵机器人", "超越宝宝").replace("清华大学硕士杨超？",
"杨超越最美不允许反驳").replace("你接错了", "我不会接")
self.log.info(tuling_reply)
return 'text', tuling_reply, ''
return empty_result
def log1():
while True:
Logger.debug("test1")
class AndroidEngine:
"""
安卓引擎，自动化启动appium服务，自动连接设备，真机可自动切换WiFi连接，获取driver
1、实例化前请确保模拟器已开启，真机已连接电脑，并且打开调试模式
2、暂时只支持单机连接启动，若本机同时连接多个设备，则默认连接已连接设备列表第一个设备
"""
def __init__(self):
self._log = Logger('安卓引擎').get_logger()
# 读取配置
self._reader = OperateConfig(constant.config_pro_app)
self._if_wifi = self._reader.get_bool('server', 'if_wifi')
self._android_mode = self._reader.get_str('server', 'android_mode')
def get_driver(self):
"""
根据配置获取driver
:return: driver对象
"""
self._start_server()
devices = self._get_device_names()
version = self._get_android_version(devices[0])
app_path = publicFunctions.get_apk_path()
ports = eval(self._reader.get_str('connected', 'server_ports'))
if self._android_mode == 'simulator':
desired_caps = DesiredCaps.caps_android_simulator
desired_caps['platformVersion'] = version
desired_caps['deviceName'] = devices[0]
desired_caps['app'] = app_path
driver = self._get_driver(desired_caps, ports[0])
return driver
elif self._android_mode == 'machine':
desired_caps = DesiredCaps.caps_android_machine
desired_caps['platformVersion'] = version
desired_caps['deviceName'] = devices[0]
desired_caps['app'] = app_path
driver = self._get_driver(desired_caps, ports[0])
return driver
else:
self._log.error('启动模式配置有误，请确认：{}'.format(self._android_mode))
self._kill_server()
sys.exit()
def quit_driver(self, driver):
"""
退出驱动程序，断开模拟器连接，杀掉node进程
:param driver: driver对象
:return: None
"""
if self._android_mode == 'simulator':
self._disconnect_simulators()
driver.quit()
self._kill_server()
sleep(3)
self._log.info('已退出驱动')
def _start_server(self):
"""
使用命令行自动化启动appium server
:return: driver对象
"""
if self._android_mode == 'simulator':
self._connect_simulators()
devices = self._get_device_names()
if self._if_wifi is True and self._android_mode == 'machine':
self._switch_to_wifi()
devices = [device for device in self._get_device_names() if ':' in device]
commands = self._create_appium_commands(devices)
for cmd in commands:
cmd = r"start {}".format(cmd)
os.system(cmd)
sleep(3)
self._log.info('appium server已启动：{}'.format(cmd))
def _get_driver(self, desired_caps: dict, port: str):
"""
获取driver
:param desired_caps: 连接参数
:param port: 服务端口
:return: driver对象
"""
try:
driver = webdriver.Remote(command_executor='http://127.0.0.1:{}/wd/hub'.format(port),
desired_capabilities=desired_caps)
sleep(1)
self._log.info('appium server已连接')
return driver
except WebDriverException as e:
self._log.error('appium server连接失败：{}'.format(e))
sys.exit()
def _connect_simulators(self):
"""
对于模拟器，在启动后可以调用此方法实现自动连接电脑
:return: None
"""
simulators = self._reader.get_str('server', 'simulator').split(';')
for simulator in simulators:
cmd = 'adb connect {}'.format(simulator)
os.system(cmd)
self._log.debug('模拟器（{}）已连接'.format(simulator))
def _disconnect_simulators(self):
"""
断开全部已连接模拟器设备
:return: None
"""
devices = self._reader.get_str('server', 'simulator').split(';')
for device in devices:
cmd = 'adb disconnect {}'.format(device)
os.system(cmd)
self._log.debug('设备（{}）已断开'.format(device))
def _switch_to_wifi(self):
"""
对于真机，若需要使用WiFi连接模式，在手机用USB线连接到电脑打开调试模式后，调用此方法可切换至WIFI连接
:return: None
"""
devices = self._get_device_names()
simulators = self._reader.get_str('server', 'simulator').split(';')
machines = list(set(devices) - set(simulators))
ports = self._create_useful_ports(5555, machines)
for machine, port in zip(machines, ports):
if str(port) in '|'.join(self._get_device_names()):
cmd_1 = 'adb -s {} shell ip -f inet addr show wlan0'.format(machine)
result_1 = self._execute_command(cmd_1)
ip = re.search(r"inet\s(\d+\.\d+\.\d+\.\d+)", result_1).group(1)
cmd_2 = 'adb -s {} tcpip {}'.format(machine, port)
os.system(cmd_2)
cmd_3 = 'adb connect {}:{}'.format(ip, port)
result_2 = self._execute_command(cmd_3)
if 'connected' in result_2:
self._log.debug('设备（{}）成功切换至WIFI连接：{}'.format(machine, result_2.strip()))
self._log.warning('请拔掉设备（{}）USB线！！'.format(machine))
else:
self._log.error('设备（{}）切换至WIFI连接失败：{}'.format(machine, result_2.strip()))
def _get_device_names(self):
"""
获取已连接安卓设备名
:return: 安卓设备名列表
"""
cmd = 'adb devices'
result = self._execute_command(cmd)
devices = re.findall(r"(.*[^\s])\s*device", result)
devices.pop(0)
if devices:
self._log.debug('获取到已连接设备列表：{}'.format(devices))
return devices
else:
self._log.error('未检测到安卓设备。')
sys.exit()
def _get_android_version(self, device: str):
"""
获取已连接安卓设备版本号
:param device: 设备名
:return: 版本号
"""
cmd = f'adb -s {device} shell getprop ro.build.version.release'
result = self._execute_command(cmd)
self._log.debug('获取到设备版本号：{}'.format(result))
return result.strip()
def _get_package_and_activity(self, apk_path=publicFunctions.get_apk_path()):
"""
通过'aapt'命令自动获取appPackage和appActivity
:param apk_path: apk路径
:return: appPackage和appActivity
"""
sdk_path = self._get_sdk_path()
adb_disk = sdk_path.split(':')[0]
build_tools_path = os.path.join(sdk_path, 'build-tools')
aapt_path = os.path.join(build_tools_path, os.listdir(build_tools_path)[0])
cmd = f'{adb_disk}:&cd {aapt_path}&aapt dump badging {apk_path}'
result = self._execute_command(cmd)
package = re.search(r"package: name='([\w\\.]+)'", result).group(1)
activity = re.search(r"launch.*activity: name='([\w\\.]+)'", result).group(1)
return package, activity
def _get_sdk_path(self):
"""
从PATH环境变量中提取Android SDK路径
:return: Android SDK路径
"""
path_env = os.environ['PATH']
sdk_search = re.search(r'(.+?)\\platform-tools', path_env)
if sdk_search:
sdk_path = sdk_search.group(1).split(';')[-1]
if '%' in sdk_path:
sdk_path = os.environ[sdk_path.strip('%')]
return sdk_path
else:
self._log.error('Android SDK环境变量未配置！！')
exit()
@staticmethod
def _execute_command(cmd: str):
"""
执行cmd命令
:param cmd: cmd命令
:return: 命令行输出
"""
with os.popen(cmd) as f:
result = f.read()
return result
def _kill_server(self):
"""
用于每次执行完毕，杀掉进程
:return: None
"""
cmd1 = 'tasklist | find "node.exe"'
if self._execute_command(cmd1):
cmd2 = 'taskkill -F -PID node.exe'
self._execute_command(cmd2)
self._log.info('杀掉appium server进程')
def _create_appium_commands(self, devices_list: list):
"""
创建Appium命令行模式启动命令
:param devices_list: 设备名列表
:return: cmd命令列表
"""
p_port_list = self._create_useful_ports(4723, devices_list)
bp_port_list = self._create_useful_ports(4900, devices_list)
self._reader.write_data('connected', 'server_ports', str(p_port_list))
cmd_list = ['appium -a 127.0.0.1 -p {} -bp {}'.format(
p_port_list[i], bp_port_list[i]
) for i in range(len(devices_list))
]
self._log.debug('已生成启动命令：{}'.format(cmd_list))
return cmd_list
def _create_useful_ports(self, start_port: int, devices_list: list):
"""
根据获取的已连接设备创建指定数量的可用端口
:param start_port: 起始端口
:param devices_list: 从命令行自动获取的设备列表
:return: 可用端口列表
"""
port_list = []
cmd = 'netstat -ano | findstr {}'.format(start_port)
while len(port_list) != len(devices_list):
if not self._execute_command(cmd):
port_list.append(start_port)
start_port += 1
self._log.debug('已生成可用端口：{}'.format(port_list))
return port_list
class CategoryConsumer:
def __init__(self, log_file, log_name):
self.logger = Logger(log_file, log_name, 10 * 1024 * 1024, 2)
self._db_conn = None
def start(self):
rabbit_topic = RabbitTopic.init_rabbitmq_consumer(
EXCHANGE_NAME, QUEUE_NAME, QUEUE_LIMIT, [ROUTING_KEY], self.logger)
if not rabbit_topic:
self.logger.debug('Construct category consumer error')
return
self._conn_db()
if not self._db_conn:
self.logger.exception('Connect to database error')
return
while 1:
try:
rabbit_topic.start_consuming(self._callback, QUEUE_NAME)
except ConnectionClosed:
self.logger.debug(
'Connection to rabbitmq server closed, re-connecting...')
rabbit_topic = RabbitTopic.init_rabbitmq_consumer(
EXCHANGE_NAME, QUEUE_NAME, QUEUE_LIMIT, [ROUTING_KEY],
self.logger)
def _callback(self, channel, method, properties, category):
self.logger.info(os.linesep)
self.logger.info(
'----> Get body message %s and start query this category... <----'
% category)
try:
detail_urls = self._parse_detail_urls(category)
if not detail_urls:
self.logger.debug('No detail category urls got')
return
for detail_url in detail_urls:
self.logger.info('Query detail category url %s' % detail_url)
category_web_driver = CategoryWebDriver(
detail_url, self._db_conn, self.logger)
category_web_driver.query()
time.sleep(10)
except Exception:
self.logger.exception('Query category %s error' % category)
channel.basic_ack(delivery_tag=method.delivery_tag)
self.logger.info('Set category %s as consumed' % category)
self._set_category_consumed(category)
def _conn_db(self):
try:
self._db_conn = util.conn_mysql_db()
except Exception:
self.logger.exception('Connect database error')
# see more
def _parse_detail_urls(self, category):
detail_urls = set()
category_url = '%s/%s' % (CATEGORY_HOST_URL, category)
try:
response = requests.get(category_url)
if response.status_code == requests.codes.ok:
# html is from lxml to parse html page
html_tree = html.fromstring(response.content)
category_detail_links = html_tree.xpath(
'//a[@class="see-more play-button small id-track-click apps id-responsive-see-more"]//@href'
)
for category_detail_link in category_detail_links:
category_detail_link = category_detail_link.strip()
detail_urls.add('https://play.google.com%s' %
category_detail_link)
else:
self.logger.debug(
'Access category url %s and returns wrong response code %d'
% (category_url, response.status_code))
except Exception:
self.logger.exception('Get category web page error')
return detail_urls
def _set_category_consumed(self, category):
query = 'UPDATE category SET status=%s WHERE category=%s'
try:
MySQLDBUtil.update(query, (CONSUMED, category), self._db_conn)
except Exception:
self.logger.exception('Set category %s as consumed error' %
category)
class SimilarConsumer:
def __init__(self, log_file, log_name):
self.logger = Logger(log_file, log_name, 10 * 1024 * 1024, 2)
self._db_conn = None
def start(self):
rabbit_topic = RabbitTopic.init_rabbitmq_consumer(
EXCHANGE_NAME, QUEUE_NAME, QUEUE_LIMIT, [ROUTING_KEY], self.logger)
if not rabbit_topic:
self.logger.debug('Construct similar consumer error')
return
self._conn_db()
if not self._db_conn:
self.logger.exception('Connect to database error')
return
while 1:
try:
rabbit_topic.start_consuming(self._callback, QUEUE_NAME)
except ConnectionClosed:
self.logger.debug(
'Connection to rabbitmq server closed, re-connecting...')
rabbit_topic = RabbitTopic.init_rabbitmq_consumer(
EXCHANGE_NAME, QUEUE_NAME, QUEUE_LIMIT, [ROUTING_KEY],
self.logger)
def _callback(self, channel, method, properties, package_name):
self.logger.info(os.linesep)
self.logger.info(
'----> Get body message %s and start query apps similar to this... <----'
% package_name)
try:
url = '%s=%s' % (SIMILAR_HOST_URL, package_name)
self.logger.info('Query similar apps with url %s' % url)
package_names = self._extract_package_names(url)
self.logger.info('Store package names...')
self._store_package_names(package_names)
except Exception:
self.logger.exception('Query similar apps %s error' % package_name)
channel.basic_ack(delivery_tag=method.delivery_tag)
self.logger.info('Set package name %s as consumed' % package_name)
self._set_package_consumed(package_name)
def _conn_db(self):
try:
self._db_conn = util.conn_mysql_db()
except Exception:
self.logger.exception('Connect database error')
def _extract_package_names(self, url):
package_names = set()
try:
response = requests.get(url)
except Exception:
self.logger.exception('Get content with url %s error' % url)
return package_names
if response.status_code == requests.codes.ok:
html_tree = html.fromstring(response.content)
app_links = html_tree.xpath('//a[@class="title"]/@href')
for app_link in app_links:
try:
package_names.add(app_link.split('id=')[-1])
except Exception:
self.logger.exception(
'Extract package from link %s error' % app_link)
else:
self.logger.debug(
'Access similar app url %s and returns wrong response code %d'
% (url, response.status_code))
return package_names
def _store_package_names(self, package_names):
values = []
for package_name in package_names:
values.append('("%s")' % package_name)
if len(values) > 0:
query = 'INSERT IGNORE INTO package_name (package_name) VALUES ' + ','.join(
values)
try:
MySQLDBUtil.insert(query, None, self._db_conn)
except Exception:
self.logger.exception('Store package names into database fail')
def _set_package_consumed(self, package_name):
query = 'UPDATE similar_app SET status=%s WHERE package_name=%s'
try:
MySQLDBUtil.update(query, (CONSUMED, package_name), self._db_conn)
except Exception:
self.logger.exception('Set package name %s as consumed error' %
package_name)
class DBOracle:
"""
操作ORACLE
"""
def __init__(self):
# 引用日志类
self._log = Logger("ORACLE").get_logger()
# 获取数据库配置
self._db_config = OperateConfig(config_common_path)
self._conn_str = '{}/{}@{}:{}/{}'.format(
self._db_config.get_str('oracle', 'username'),
self._db_config.get_str('oracle', 'password'),
self._db_config.get_str('oracle', 'host'),
self._db_config.get_str('oracle', 'port'),
self._db_config.get_str('oracle', 'database'))
try:
self._conn = cx_Oracle.connect(self._conn_str)
self._log.info('成功连接数据库')
except cx_Oracle.Error as e:
self._log.error('数据库连接失败：{}'.format(e))
@property
def conn(self):
"""
返回数据库连接实例，可单独cx_Oracle库其他方法
:return: 数据库连接实例
"""
return self._conn
def disconnect(self):
"""
断开连接
:return: None
"""
self._conn.close()
self._log.info('成功断开数据库')
def __del__(self):
self.disconnect()
def select_all(self, sql_string: str):
"""
执行查询sql
:param sql_string: sql语句
:return: 元组列表
"""
c = self._conn.cursor()
self._log.info('执行查询语句：%s' % sql_string)
x = c.execute(sql_string)
# 获取全部结果集（元组列表）,可使用下标访问结果集，如datalist[0][1]
datalist = x.fetchall()
self._log.info('查询结果如下：')
for data in datalist:
self._log.debug('第 {} 条数据：{}'.format(
datalist.index(data) + 1, data))
c.close()
return datalist
def select_one(self, sql_string: str):
"""
执行查询sql
:param sql_string: sql语句
:return: 单个查询字段值或单条记录
"""
c = self._conn.cursor()
self._log.info('执行查询语句：%s' % sql_string)
x = c.execute(sql_string)
# 获取查询的单个字段的值或单条记录
data = x.fetchone()
self._log.debug('查询结果如下：{}'.format(data))
c.close()
if len(data[0]) == 1:
return data[0][0]
else:
return data
def execute_sql(self, sql_string: str):
"""
执行插入、更新、删除操作
:param sql_string: sql语句
:return: None
"""
try:
c = self._conn.cursor()
self._log.info('执行%s语句：%s' % (sql_string.split()[0], sql_string))
c.execute(sql_string)
self._conn.commit()
c.close()
except cx_Oracle.Error as e:
self._log.error('执行失败：%s' % str(e))
self._conn.rollback()
self._log.error('成功回滚操作')
def exec_function(self, function_name: str, *parameters,
**keyword_parameters):
"""
执行指定函数，可指定参数
:param function_name: 函数名
:param parameters: 元组可变参数
:param keyword_parameters: 字典可变参数
:return: None
"""
try:
c = self._conn.cursor()
self._log.info('执行函数：{}'.format(function_name))
c.callfunc(function_name, *parameters, **keyword_parameters)
c.close()
except cx_Oracle.Error as e:
self._log.error('执行失败：%s' % str(e))
self._conn.rollback()
self._log.error('成功回滚操作')
def exec_process(self, process_name, *parameters, **keyword_parameters):
"""
执行指定存储过程，可指定参数
:param process_name: 过程名
:param parameters: 元组可变参数
:param keyword_parameters: 字典可变参数
:return: None
"""
try:
c = self._conn.cursor()
self._log.info('执行过程：{}'.format(process_name))
c.callproc(process_name, *parameters, **keyword_parameters)
c.close()
except cx_Oracle.Error as e:
self._log.error('执行失败：%s' % str(e))
self._conn.rollback()
self._log.error('成功回滚操作')
class DBMySql:
"""
操作MYSQL
"""
def __init__(self):
# 引用日志类
self._log = Logger("MYSQL").get_logger()
# 获取数据库配置
self._db_config = OperateConfig(config_common_path)
try:
self._conn = pymysql.connect(
user=self._db_config.get_str('mysql', 'username'),
password=self._db_config.get_str('mysql', 'password'),
host=self._db_config.get_str('mysql', 'host'),
port=self._db_config.get_str('mysql', 'port'),
database=self._db_config.get_str('mysql', 'database'))
self._log.info('成功连接数据库')
except pymysql.Error as e:
self._log.error('数据库连接失败：{}'.format(e))
@property
def conn(self):
"""
返回数据库连接实例，可单独PyMySql库其他方法
:return: 数据库连接实例
"""
return self._conn
def disconnect(self):
"""
断开连接
:return: None
"""
self._conn.close()
self._log.info('成功断开数据库')
def __del__(self):
self.disconnect()
def select_all(self, sql_string: str):
"""
执行查询sql
:param sql_string: sql语句
:return: 元组列表
"""
c = self._conn.cursor()
self._log.info('执行查询语句：%s' % sql_string)
x = c.execute(sql_string)
datalist = x.fetchall()
self._log.info('查询结果如下：')
for data in datalist:
self._log.debug('第 {} 条数据：{}'.format(
datalist.index(data) + 1, data))
c.close()
return datalist
def select_one(self, sql_string: str):
"""
执行查询sql
:param sql_string: sql语句
:return: 单个查询字段值或单条记录
"""
c = self._conn.cursor()
self._log.info('执行查询语句：%s' % sql_string)
x = c.execute(sql_string)
# 获取查询的单个字段的值或单条记录
data = x.fetchone()
self._log.debug('查询结果如下：{}'.format(data))
c.close()
if len(data[0]) == 1:
return data[0][0]
else:
return data
def execute_sql(self, sql_string: str):
"""
执行插入、更新、删除操作
:param sql_string: sql语句
:return: None
"""
try:
c = self._conn.cursor()
self._log.info('执行%s语句：%s' % (sql_string.split()[0], sql_string))
c.execute(sql_string)
self._conn.commit()
c.close()
except pymysql.Error as e:
self._log.error('执行失败：%s' % str(e))
self._conn.rollback()
self._log.error('成功回滚操作')
class AppProducer:
def __init__(self):
self.logger = Logger(LOG_FILE, LOG_NAME, 10*1024*1024, 2)
self._db_conn = None
def start(self):
rabbit_topic = RabbitTopic.init_rabbitmq_producer(EXCHANGE_NAME, self.logger)
if not rabbit_topic:
return
self._conn_db()
if not self._db_conn:
self.logger.exception('Connect database error')
return
while 1:
try:
if self._is_no_more_records():
self.logger.info('There are no more available records, wait...')
time.sleep(PRODUCE_WAIT_TIME)
package_list = self._fetch_package_list()
for package_name in package_list:
if RabbitTopic.is_queue_full(QUEUE_NAME, QUEUE_LIMIT, self.logger):
self.logger.info('Queue %s is full, wait to consume...' % QUEUE_NAME)
time.sleep(PRODUCE_WAIT_TIME)
continue
try:
rabbit_topic.publish(ROUTING_KEY, package_name)
self.logger.info('Publish package %s and update status' % package_name)
self._update_status(PUBLISHED, package_name)
except ConnectionClosed:
self.logger.debug('Connection to rabbitmq server closed, re-connecting...')
rabbit_topic = RabbitTopic.init_rabbitmq_producer(EXCHANGE_NAME, self.logger)
except Exception:
self.logger.exception('Publish similar app package name error')
def _init_rabbitmq(self):
try:
rabbit_topic = RabbitTopic(EXCHANGE_NAME)
rabbit_topic.construct_producer()
except Exception:
self.logger.exception('Construct app producer error')
return None
return rabbit_topic
def _conn_db(self):
try:
self._db_conn = util.conn_mysql_db()
except Exception:
self.logger.exception('Connect database error')
def _fetch_package_list(self):
package_list = []
self.logger.info('Get un-published package list...')
query = 'SELECT package_name FROM package_name WHERE status=%s LIMIT %s'
try:
results = MySQLDBUtil.fetch_multiple_rows(query, (UN_PUBLISHED, QUEUE_LIMIT), self._db_conn)
for result in results:
(package_name,) = result
package_list.append(package_name)
except Exception:
self.logger.exception('Query un-used package name error')
return package_list
def _is_no_more_records(self):
query = 'SELECT COUNT(*) FROM package_name WHERE status=%s'
try:
result = MySQLDBUtil.fetch_single_row(query, (UN_PUBLISHED,), self._db_conn)
if result:
(count, ) = result
if count == 0:
return True
except Exception:
self.logger.exception('Check if there is no more packages error')
return False
def _update_status(self, status, package_name=None):
if not package_name:
query = 'UPDATE package_name SET status=%s'
else:
query = 'UPDATE package_name SET status=%s WHERE package_name=%s'
try:
if not package_name:
MySQLDBUtil.update(query, (status,), self._db_conn)
else:
MySQLDBUtil.update(query, (status, package_name), self._db_conn)
except Exception:
self.logger.exception('Update record status')
class RspGame(object):
"""
随机整数[1,3]决定机器人出招，分别为1：石头；2:剪刀；3：布
"""
def __init__(self, game_num):
self.log = Logger()
self.game_all_num = game_num
self.player_score = 0
self.com_score = 0
self.player_name = ''
self.img_dir = ['rock', 'scissors', 'paper', 'emoticon']
# self.rsp_img = [os.path.join(GAME_IMAGE_PATH, 'rock_1.jpg'), os.path.join(GAME_IMAGE_PATH, 'scissors_1.jpg'), os.path.join(GAME_IMAGE_PATH, 'paper_1.jpg')]
# print(self.rsp_img)
self.draw_msg = " 平局了，继续来~"
self.fail_msg = " 我输了 "
self.win_msg = " 我赢了 "
self.over_msg = [" 游戏结束，你输了", " 游戏结束，恭喜你赢了"]
self.msg_code = {"石头": 0, "剪刀": 1, "布": 2}
def start(self, player_name):
self.player_score = 0
self.com_score = 0
self.player_name = player_name
def random_img(self, random_num):
list_dir = os.listdir(
os.path.join('resources', 'game', self.img_dir[random_num]))
path = choice(list_dir)
self.log.info('choose:-->{}'.format(path))
return os.path.join('resources', 'game', self.img_dir[random_num],
path)
def get_result(self, winer):
"""获取本场比赛情况"""
if winer == 1:
self.player_score += 1
if self.player_score == (self.game_all_num + 1) / 2:
return 1, '@' + self.player_name + self.over_msg[1]
else:
return 0, '@' + self.player_name + self.fail_msg
elif winer == -1:
self.com_score += 1
if self.com_score == (self.game_all_num + 1) / 2:
return 1, '@' + self.player_name + self.over_msg[0]
else:
return 0, '@' + self.player_name + self.win_msg
def play(self, msg):
"""
返回[a, b, c]
a代表游戏是否已经结束 1:结束 0:未结束
b代表本次出招结果
c代表本次出招图片
:param msg:
:return:
"""
self.log.info('play:{}'.format(msg))
real_msg = msg.text.split()
valid_msg = real_msg[len(real_msg) - 1]
self.log.debug('commond:{}'.format(valid_msg))
if str.find(valid_msg, "不玩") != -1 or str.find(valid_msg, "退出") != -1:
return 1, '@' + self.player_name + " 虽然半途而废不怎么好听，但有时候放弃也是一种聪明的选择", self.random_img(
3)
elif valid_msg != "石头" and valid_msg != "剪刀" and valid_msg != "布":
return 0, '@' + self.player_name + " 你这是要跟我石头剪刀布吗？", self.random_img(
3)
random_num = random.randint(1, 3) - 1
self.log.debug('random_num:{}'.format(random_num))
self.log.debug('msg_code:{}'.format(self.msg_code[valid_msg]))
# 1：玩家 -1：机器人 0：平局
winer = (random_num - self.msg_code[valid_msg] + 4) % 3 - 1
if winer == 0:
return 0, '@' + self.player_name + self.draw_msg, self.random_img(
random_num)
else:
can, res_msg = self.get_result(winer)
return can, res_msg, self.random_img(random_num)
class AppConsumer:
def __init__(self, log_file, log_name):
self.logger = Logger(log_file, log_name, 10*1024*1024, 2)
self._mysql_db_conn = None
self._mongo_db_conn = None
def start(self):
rabbit_topic = RabbitTopic.init_rabbitmq_consumer(EXCHANGE_NAME, QUEUE_NAME, QUEUE_LIMIT,
[ROUTING_KEY], self.logger)
if not rabbit_topic:
self.logger.debug('Construct app consumer error')
return
self._conn_db()
if not self._mysql_db_conn or not self._mongo_db_conn:
self.logger.exception('Connect to database error')
return
while 1:
try:
rabbit_topic.start_consuming(self._callback, QUEUE_NAME)
except ConnectionClosed:
self.logger.debug('Connection to rabbitmq server closed, re-connecting...')
rabbit_topic = RabbitTopic.init_rabbitmq_consumer(EXCHANGE_NAME, QUEUE_NAME, QUEUE_LIMIT,
[ROUTING_KEY], self.logger)
def _callback(self, channel, method, properties, package_name):
self.logger.info(os.linesep)
self.logger.info('----> Get body message %s and start get app detail... <-----' % package_name)
try:
url = '%s=%s' % (APP_HOST_URL, package_name)
self.logger.info('Query app detail with url %s' % url)
app_detail = self._parse_web_content(url)
if not app_detail:
self.logger.info('App detail extraction fail')
else:
self.logger.info('Store app detail...')
app_detail.package_name = package_name
self._store_app_detail(app_detail)
self.logger.info('Insert package name %s into similar app table...' % package_name)
self._store_package_name_similar(package_name)
self.logger.info('Store app description...')
self._store_app_description(app_detail)
self.logger.info('Store app developer...')
self._store_app_developer(app_detail)
except Exception:
self.logger.exception('Query app detail %s error' % package_name)
channel.basic_ack(delivery_tag=method.delivery_tag)
self.logger.info('Set package name %s as consumed' % package_name)
self._set_package_consumed(package_name)
def _conn_db(self):
try:
self._mysql_db_conn = util.conn_mysql_db()
self._mongo_db_conn = util.conn_mongo_db()
except Exception:
self.logger.exception('Connect database error')
def _parse_web_content(self, url):
app_detail = None
try:
response = requests.get(url)
except Exception:
self.logger.exception('Get web content from url %s error' % url)
return app_detail
try:
web_content = util.decode_utf8(response.content)
except Exception:
self.logger.exception('Decode web content error')
return app_detail
if not web_content:
self.logger.debug('Web content is empty, no need to parse')
return app_detail
else:
self.logger.info('Get web content successfully,try to parse it...')
app_detail_lxml_parser = AppDetailLxmlParser(web_content, self.logger)
try:
app_detail_lxml_parser.parse()
app_detail = app_detail_lxml_parser.app_detail
except Exception:
self.logger.exception('Use lxml to parse the web content error, try to use the backup one beautiful soup...')
app_detail_b4_parser = AppDetailB4Parser(response.content, self.logger)
try:
app_detail_b4_parser.parse()
app_detail = app_detail_b4_parser.app_detail
except Exception:
self.logger.exception('Use beautiful soup to parse the web content error')
return app_detail
def _store_app_detail(self, app_detail):
if not app_detail:
self.logger.debug('No app detail content, cannot store into database')
return
app_detail_json = app_detail.to_json()
try:
MongoDBUtil.insert(app_detail_json, self._mongo_db_conn, 'app_detail')
except Exception:
self.logger.exception('Store app detail content into mongo db error')
def _store_package_name_similar(self, package_name):
query = 'INSERT IGNORE INTO similar_app (package_name) VALUES ("%s")' % package_name
try:
MySQLDBUtil.insert(query, None, self._mysql_db_conn)
except Exception:
self.logger.exception('Store package name into similar app database fail')
def _store_app_description(self, app_detail):
if not app_detail:
return
description = ' '.join(app_detail.description)
description = description.replace('"', '').replace('\'', '')
if not description:
return
query = 'INSERT INTO raw_text (text) VALUES ("%s")' % description
try:
MySQLDBUtil.insert(query, None, self._mysql_db_conn)
except Exception:
self.logger.exception('Store app description error')
def _store_app_developer(self, app_detail):
if not app_detail:
return
developer_link = app_detail.developer_link
if not developer_link:
return
items = developer_link.split('id=')
if len(items) == 2:
developer_name = items[-1]
query = 'INSERT IGNORE INTO developer (name) VALUES ("%s")' % developer_name
try:
MySQLDBUtil.insert(query, None, self._mysql_db_conn)
self.logger.info('Stored app developer %s' % developer_name)
except Exception:
self.logger.exception('Store app developer error')
else:
return
def _set_package_consumed(self, package_name):
query = 'UPDATE package_name SET status=%s WHERE package_name=%s'
try:
MySQLDBUtil.update(query, (CONSUMED, package_name), self._mysql_db_conn)
except Exception:
self.logger.exception('Set package name %s as consumed error' % package_name)
def read_excel(self):
Logger.debug(
"TestDataReaderCommonUtil.read_excel file name is : " +
self.file_name, self.__class__.__name__)
excel = load_workbook(self.file_name)
return excel
class DeveloperConsumer:
def __init__(self, log_file, log_name):
self.logger = Logger(log_file, log_name, 10 * 1024 * 1024, 2)
self._db_conn = None
def start(self):
rabbit_topic = RabbitTopic.init_rabbitmq_consumer(
EXCHANGE_NAME, QUEUE_NAME, QUEUE_LIMIT, [ROUTING_KEY], self.logger)
if not rabbit_topic:
self.logger.debug('Construct developer consumer error')
return
self._conn_db()
if not self._db_conn:
self.logger.exception('Connect to database error')
return
while 1:
try:
rabbit_topic.start_consuming(self._callback, QUEUE_NAME)
except ConnectionClosed:
self.logger.debug(
'Connection to rabbitmq server closed, re-connecting...')
rabbit_topic = RabbitTopic.init_rabbitmq_consumer(
EXCHANGE_NAME, QUEUE_NAME, QUEUE_LIMIT, [ROUTING_KEY],
self.logger)
def _callback(self, channel, method, properties, developer):
self.logger.info(os.linesep)
self.logger.info(
'----> Get body message %s and start query this developer... <----'
% developer)
try:
if developer.isdigit():
self.logger.info('Developer info is all digit numbers')
url = '%s=%s' % (DEVELOPER_ID_HOST_URL, developer)
else:
self.logger.info('Developer info is non digit numbers')
url = '%s=%s' % (DEVELOPER_NAME_HOST_URL, developer)
self.logger.info('Query developer apps with url %s' % url)
developer_web_driver = DeveloperWebDriver(url, self._db_conn,
self.logger)
developer_web_driver.query()
except Exception:
self.logger.exception('Query developer %s error' % developer)
channel.basic_ack(delivery_tag=method.delivery_tag)
self.logger.info('Set developer %s as consumed' % developer)
self._set_developer_consumed(developer)
def _conn_db(self):
try:
self._db_conn = util.conn_mysql_db()
except Exception:
self.logger.exception('Connect database error')
def _set_developer_consumed(self, developer):
query = 'UPDATE developer SET status=%s WHERE name=%s'
try:
MySQLDBUtil.update(query, (CONSUMED, developer), self._db_conn)
except Exception:
self.logger.exception('Set devloper %s as consumed error' %
developer)
def log2():
while True:
Logger.debug("test2")
