def run(self):
'''
Run our entire collection of suites.
'''
for logger in self.result_loggers:
logger.begin_testing()
log.info(separator())
log.info("Building all non 'lazy_init' fixtures")
failed_builds = self.setup_unbuilt(self.suites.iter_fixtures(),
setup_lazy_init=False)
if failed_builds:
error_str = ''
for fixture, error in failed_builds:
error_str += 'Failed to build %s\n' % fixture
error_str += '%s' % error
log.warn('Error(s) while building non lazy_init fixtures.')
log.warn(error_str)
outcomes = set()
for suite in self.suites:
outcome = self.run_suite(suite)
outcomes.add(outcome)
if outcome in Outcome.failfast and config.fail_fast:
break
for logger in self.result_loggers:
logger.end_testing()
return self._suite_outcome(outcomes)
def evaluate_nulls_coverage(cnfs_file, gold_cnfs_file):
system_confs = analyze_cnfs_file(cnfs_file)
system_values = set(system_confs.keys())
gold_confs = analyze_cnfs_file(gold_cnfs_file)
gold_values = set(gold_confs.keys())
tp, fp, tn, fn = 0, 0, 0, 0
for val in gold_values.union(system_values):
if val in gold_values:
if val in system_values:
if gold_confs[val] == '<null>' and system_confs[val] == '<null>':
tp += 1
elif gold_confs[val] != '<null>' and system_confs[val] != '<null>':
tn += 1
else:
log.warn("not handled case!")
else:
fn += 1
else:
if system_confs[val] == '<null>':
fp += 1
else:
log.warn("not handled case!")
return (tp, tn, fp, fn)
def downloader_async(self, remote_dir):
threads = []
for i in range(30):
log.debug('-------connecting %s' % i)
self._queue.put(SftpHelper(self.host, self.username, self.passwd))
log.debug('stop connect')
for i in self.get_file_list(remote_dir, is_root=True):
log.debug('%%%%')
try:
sftp_helper = self._queue.get(timeout=1)
log.debug('------get %s' % i)
except Queue.Empty:
log.warn('No sftp is available')
sftp_helper = SftpHelper(self.host, self.username, self.passwd)
log.info('add new sftp')
thr = threading.Thread(target=self.get_one, args=(sftp_helper, i))
threads.append(thr)
thr.start()
if len(threads) > 200:
for t in threads:
t.join(timeout=10)
threads = []
for t in threads:
t.join()
while not self._queue.empty():
i = self._queue.get()
i.sftp.close()
i.t.close()
self.sftp.close()
self.t.close()
def copy_remote_file_to_nodes(self, remote_file, nodes, dest=None):
"""
Copies a remote file from this Node instance to another Node instance
without passwordless ssh between the two.
dest - path to store the data in on the node (defaults to remote_file)
"""
if not dest:
dest = remote_file
rf = self.ssh.remote_file(remote_file, 'r')
contents = rf.read()
sts = rf.stat()
mode = stat.S_IMODE(sts.st_mode)
uid = sts.st_uid
gid = sts.st_gid
rf.close()
for node in nodes:
if self.id == node.id and remote_file == dest:
log.warn("src and destination are the same: %s, skipping" %
remote_file)
continue
nrf = node.ssh.remote_file(dest, 'w')
nrf.write(contents)
nrf.chown(uid, gid)
nrf.chmod(mode)
nrf.close()
def get_traffic_counters(self, chain, wrap=True, zero=False):
"""Return the sum of the traffic counters of all rules of a chain."""
cmd_tables = self._get_traffic_counters_cmd_tables(chain, wrap)
if not cmd_tables:
LOG.warn(('Attempted to get traffic counters of chain %s which '
'does not exist'), chain)
return
name = get_chain_name(chain, wrap)
acc = {'pkts': 0, 'bytes': 0}
for cmd, table in cmd_tables:
args = [cmd, '-t', table, '-L', name, '-n', '-v', '-x']
if zero:
args.append('-Z')
if self.namespace:
args = ['ip', 'netns', 'exec', self.namespace] + args
current_table = self.execute(args, run_as_root=True)
current_lines = current_table.split('\n')
for line in current_lines[2:]:
if not line:
break
data = line.split()
if (len(data) < 2 or not data[0].isdigit()
or not data[1].isdigit()):
break
acc['pkts'] += int(data[0])
acc['bytes'] += int(data[1])
return acc
def get_one(self, sftp_helper, remote_file):
pat = re.compile(self.remote_dir)
local_file = pat.sub(self.local_dir, remote_file)
local_dir = os.path.dirname(local_file)
if not os.path.exists(local_dir):
try:
os.makedirs(local_dir)
except OSError:
pass
log.debug('Start download %s -------' % remote_file)
try:
sftp_helper.sftp.get(remote_file, local_file)
log.info('Download %s successfully' % local_file)
sftp_helper.sftp.remove(remote_file)
log.info('remove %s successfully' % remote_file)
# remote_dir = os.path.dirname(remote_file)
# try:
# FileHandler.delete_all(sftp_helper, remote_dir, self.remote_dir)
# except Exception as e:
# pass
except Exception as e:
log.warn(e)
try:
self._queue.put(sftp_helper, block=False)
except Exception:
pass
def download_logs(client: InventoryClient, cluster: dict, dest: str,
must_gather: bool, retry_interval: int = RETRY_INTERVAL):
output_folder = get_logs_output_folder(dest, cluster)
if os.path.isdir(output_folder):
log.info(f"Skipping. The logs directory {output_folder} already exists.")
return
recreate_folder(output_folder)
recreate_folder(os.path.join(output_folder, "cluster_files"))
try:
write_metadata_file(client, cluster, os.path.join(output_folder, 'metdata.json'))
with suppress(assisted_service_client.rest.ApiException):
client.download_ignition_files(cluster['id'], os.path.join(output_folder, "cluster_files"))
for host_id in map(lambda host: host['id'], cluster['hosts']):
with suppress(assisted_service_client.rest.ApiException):
client.download_host_ignition(cluster['id'], host_id, os.path.join(output_folder, "cluster_files"))
with suppress(assisted_service_client.rest.ApiException):
client.download_cluster_events(cluster['id'], os.path.join(output_folder, f"cluster_{cluster['id']}_events.json"))
shutil.copy2(os.path.join(os.path.dirname(os.path.realpath(__file__)), "events.html"), output_folder)
with suppress(assisted_service_client.rest.ApiException):
for i in range(MAX_RETRIES):
cluster_logs_tar = os.path.join(output_folder, f"cluster_{cluster['id']}_logs.tar")
with suppress(FileNotFoundError):
os.remove(cluster_logs_tar)
client.download_cluster_logs(cluster['id'], cluster_logs_tar)
min_number_of_logs = len(cluster['hosts']) + 1 if cluster['status'] == ClusterStatus.INSTALLED else len(cluster['hosts'])
try:
verify_logs_uploaded(cluster_logs_tar, min_number_of_logs, cluster['status'] == ClusterStatus.INSTALLED)
break
except AssertionError as ex:
log.warn(f"Cluster logs verification failed: {ex}")
# Skip sleeping on last retry
if i < MAX_RETRIES - 1:
log.info(f"Going to retry in {retry_interval} seconds")
time.sleep(retry_interval)
kubeconfig_path = os.path.join(output_folder, "kubeconfig-noingress")
with suppress(assisted_service_client.rest.ApiException):
client.download_kubeconfig_no_ingress(cluster['id'], kubeconfig_path)
if must_gather:
recreate_folder(os.path.join(output_folder, "must-gather"))
config_etc_hosts(cluster['name'], cluster['base_dns_domain'],
helper_cluster.get_api_vip_from_cluster(client, cluster))
download_must_gather(kubeconfig_path, os.path.join(output_folder, "must-gather"))
finally:
run_command(f"chmod -R ugo+rx '{output_folder}'")
def run(self):
random.seed()
self.on_init()
from gtcpclient import GTcpClient
log.info('tcp_worker_start|id=%d', self._id)
self._client = GTcpClient(self._config.WORK_ENDPOINT['address'], self._config.WORK_ENDPOINT['port'], 0)
while True:
try:
request = self._client.receive()
if request is None:
log.warn('tcp_worker_lost_connection|client_id=%s,client=%s', self._client.id.encode('hex'), self._client.remote_address)
self._client.close()
elif len(request) < GTCP_HEADER_SIZE:
log.error('tcp_worker_request_packet_error|client_id=%s,client=%s,request=%s', self._client.id.encode('hex'), self._client.remote_address, request.encode('hex'))
self._client.close()
else:
request_cmd = request[:GTCP_CMD_SIZE]
request_client = TcpEndpoint(request[GTCP_CMD_SIZE:GTCP_HEADER_SIZE])
reply_body = None
if request_cmd == GTCP_CMD_RELAY:
request_body = request[GTCP_HEADER_SIZE:]
reply_body = self.on_packet(request_client, request_body)
elif request_cmd == GTCP_CMD_CONNECT:
reply_body = self.on_client_connect(request_client)
elif request_cmd == GTCP_CMD_DISCONNECT:
self.on_client_disconnect(request_client)
if reply_body is None:
self._client.send(GTCP_CMD_NONE + request_client.client_id)
else:
self._client.send(GTCP_CMD_RELAY + request_client.client_id + reply_body)
except Exception as ex:
log.exception('tcp_worker_exception|id=%u,exception=%s', self._id, ex, exc_info=True)
self._client.close()
def report_action(self, action, params=None, description=None):
"""
Callback function for each evaluated statement to report macro actions
"""
# store the action for later use:
try:
if (isinstance(action, str)):
action = unidecode.unidecode(action.decode('unicode-escape'))
except UnicodeDecodeError:
action = ''.join(filter(lambda x:x in string.printable, action))
if (isinstance(params, str)):
try:
decoded = params.replace("\\", "#ESCAPED_SLASH#").decode('unicode-escape').replace("#ESCAPED_SLASH#", "\\")
params = unidecode.unidecode(decoded)
except Exception as e:
log.warn("Unicode decode of action params failed. " + str(e))
params = ''.join(filter(lambda x:x in string.printable, params))
try:
if (isinstance(description, str)):
description = unidecode.unidecode(description.decode('unicode-escape'))
except UnicodeDecodeError as e:
log.warn("Unicode decode of action description failed. " + str(e))
description = ''.join(filter(lambda x:x in string.printable, description))
self.actions.append((action, params, description))
log.info("ACTION: %s - params %r - %s" % (action, params, description))
def parsexml(self,xml):
log.debug('in parsexml')
# 如果传进来的是文件，则直接读取内容
# 如果是字符串，则先尝试解释，如果不是xml则再尝试作为文件路径，再不行，则抛异常了。
xml_content = ''
if type(xml) == file:
xml_content = xml.read()
xml.close()
elif type(xml) == str or type(xml) == unicode:
log.debug('try to load file')
if os.path.exists(xml):
xml_file = open(xml,'r')
xml_content = xml_file.read()
xml_file.close()
else:
xml_content = xml
else:
log.error('could not init testcase from xml')
raise TypeError,'fromxml need a file instance or a string instance for argument'
log.debug('starting parse xml')
log.debug('xml content: %s'%xml_content)
doc = minidom.parseString(xml_content)
ret = get_child_tags(doc,'test')#this statement return a list
log.debug('child tag len : %d'%len(ret))
if ret:
test_node = ret[0]
return test_node
else:
log.warn('no test node in the xml!')
def parse_radius_log(radius_log_file, out_dir):
log_file = open(radius_log_file, 'r')
outfile_name = "RADIUS_MAC2NSP_%s.txt" % (DATE.strftime('%Y%m%d'))
output_file = open(os.path.join(out_dir, outfile_name), 'w')
existing = set()
try:
for record in rad_recv_record_extractor(log_file):
for converter_func, check_func in CONVERTER_CHAIN:
if check_func(record):
try:
mac, nssp = converter_func(record)
except Exception, e:
#logging.error("Error passing radius record: " + repr(e))
log.exception('Error parsing radius record')
continue
mac2nsp = "%s %s" % (mac, nssp)
if mac2nsp in existing:
log.warn("Duplicate record found: %s, ignoring..." %
mac2nsp)
else:
log.info("Matched: %s %s" % (mac, nssp))
output_file.write("%s %s\n" % (mac, nssp))
existing.add(mac2nsp)
break
else:
log.info("Radius record not matching rule: " + repr(record))
finally:
log_file.close()
output_file.close()
async def onData(self, ws, gamePacket):
""" 收到ws数据包时回调 """
log.info('ws received:' + gamePacket.dump() + ' ws:' + str(ws))
protocol = gamePacket.protocol
roomId = ws_connects_mgr.getRoomId(ws)
uid = ws_connects_mgr.getUid(ws)
room = room_manager.get_room(roomId)
log.info("handle uid:{0} protocol:{1} of room:{2}".format(
uid, protocol, room.roomId if room else ''))
if not protocol or not room:
return
# 根据客户端的包，来执行相应的操作
if protocol == PROTOCOL.C2SHeartbeat: # 游戏心跳包
# 原样返回
await ws_channel_util.write(
ws, GamePacket(PROTOCOL.S2CHeartbeat, gamePacket.payload))
elif protocol == PROTOCOL.C2SSignup: # 加入游戏
room.signup(uid, json.loads(gamePacket.payload))
elif protocol == PROTOCOL.C2SStart: # 开始游戏
room.start(uid)
elif protocol == PROTOCOL.C2SGameover: # 主播主动结束游戏
room.gameOver(uid)
else:
log.warn("protocol invalid:{0},{1}".format(protocol),
gamePacket.dump())
pass
pass
def parse_radius_log(radius_log_file, out_dir):
log_file = open(radius_log_file, 'r')
outfile_name = "RADIUS_MAC2NSP_%s.txt" % (DATE.strftime('%Y%m%d'))
output_file = open(os.path.join(out_dir, outfile_name), 'w')
existing = set()
try:
for record in rad_recv_record_extractor(log_file):
for converter_func, check_func in CONVERTER_CHAIN:
if check_func(record):
try:
mac, nssp = converter_func(record)
except Exception, e:
#logging.error("Error passing radius record: " + repr(e))
log.exception('Error parsing radius record')
continue
mac2nsp = "%s %s" % (mac, nssp)
if mac2nsp in existing:
log.warn("Duplicate record found: %s, ignoring..." % mac2nsp)
else:
log.info("Matched: %s %s" % (mac, nssp))
output_file.write("%s %s\n" % (mac, nssp))
existing.add(mac2nsp)
break
else:
log.info("Radius record not matching rule: " + repr(record))
finally:
log_file.close()
output_file.close()
def request(self, request):
if self._socket is None:
if not self._connect():
return None
packet = struct.pack('<I%ds' % len(request), len(request), request)
try:
self._socket.sendall(packet)
length_data = self._recv(4)
except Exception as ex:
self.close()
if not self._retry:
log.exception('tcp_request_fail|error=recv_length_fail,address=%s,port=%u,request=%s', self._address, self._port, request.encode('hex'))
return None
if isinstance(ex, socket.timeout):
log.exception('tcp_request_fail|error=recv_length_timeout,address=%s,port=%u,retry=0,request=%s', self._address, self._port, request.encode('hex'))
return None
log.warn('tcp_request_fail|error=recv_length_fail_will_retry,address=%s,port=%u,retry=0,request=%s,ex=%s', self._address, self._port, request.encode('hex'), ex)
if not self._connect():
log.exception('tcp_request_fail|error=retry_reconnect,address=%s,port=%u,retry=0,request=%s', self._address, self._port, request.encode('hex'))
return None
try:
self._socket.sendall(packet)
length_data = self._recv(4)
except Exception as ex:
log.exception('tcp_request_fail|error=retry_recv_length_fail,address=%s,port=%u,retry=1,request=%s', self._address, self._port, request.encode('hex'))
self.close()
return None
try:
length = struct.unpack('<I', length_data)[0]
return self._recv(length)
except Exception as ex:
log.exception('tcp_request_fail|error=recv_data_fail,address=%s,port=%u,request=%s', self._address, self._port, request.encode('hex'))
self.close()
return None
def add_compiled_module(self, m, stream):
"""Add an already parsed and processed module.
@param m (Module object) The parsed object.
@param stream (str) The OLE stream name containing the module.
"""
if (m is None):
return
self.modules.append(m)
for name, _sub in m.subs.items():
# Append the stream name for duplicate subs
if (name in self.globals):
new_name = safe_str_convert(stream) + "::" + safe_str_convert(
name)
log.warn("Renaming duplicate function " + name + " to " +
new_name)
name = new_name
# Save the sub.
if (log.getEffectiveLevel() == logging.DEBUG):
log.debug('(1) storing sub "%s" in globals' % name)
self.globals[name.lower()] = _sub
self.globals[name] = _sub
# Functions.
for name, _function in m.functions.items():
if (log.getEffectiveLevel() == logging.DEBUG):
log.debug('(1) storing function "%s" in globals' % name)
self.globals[name.lower()] = _function
self.globals[name] = _function
# Properties.
for name, _prop in m.functions.items():
if (log.getEffectiveLevel() == logging.DEBUG):
log.debug('(1) storing property let "%s" in globals' % name)
self.globals[name.lower()] = _prop
self.globals[name] = _prop
# External DLL functions.
for name, _function in m.external_functions.items():
if (log.getEffectiveLevel() == logging.DEBUG):
log.debug('(1) storing external function "%s" in globals' %
name)
self.globals[name.lower()] = _function
self.externals[name.lower()] = _function
# Global variables.
for name, _var in m.global_vars.items():
if (log.getEffectiveLevel() == logging.DEBUG):
log.debug('(1) storing global var "%s" = %s in globals (1)' %
(name, safe_str_convert(_var)))
if (isinstance(name, str)):
self.globals[name.lower()] = _var
if (isinstance(name, list)):
self.globals[name[0].lower()] = _var
self.types[name[0].lower()] = name[1]
def _get_ipcluster_plugin(self, node):
ipyversion = self._get_ipy_version(node)
if ipyversion < '0.11':
if not ipyversion.startswith('0.10'):
log.warn("Trying unsupported IPython version %s" % ipyversion)
return IPCluster10()
else:
return IPCluster11(self.enable_notebook, self.notebook_passwd)
def read(self, strData):
self.autoDeserialize(strData)
if self.error is not None:
log.error(self.error)
if self.info is not None:
log.info(self.info)
if self.warning is not None:
log.warn(self.warning)
def __live_disconnect(self):
""" 订阅服务连接断开回调 """
log.warn("监听直播间订阅消息连接断开:{}".format(self.roomId))
if self.state != ERoomState.GameOver:
reTrytimer = threading.Timer(
2+self.reconnTimes*2, self.__retry_connect_live_subscriber)
reTrytimer.start()
def read(self, strData):
self.autoDeserialize(strData)
if self.error is not None:
log.error(self.error)
if self.info is not None:
log.info(self.info)
if self.warning is not None:
log.warn(self.warning)
def _check_required_parameters(self):
parameters_found = True
for param_path in self._parameters["required"]:
value = get_dict_value(self._configuration, param_path.split('.'))
if value in [None, ""]:
parameters_found = False
log.warn("Required parameter is empty: {}".format(param_path))
return parameters_found
def close(self):
try:
self.cursor
self.cursor.close()
self.connection.close()
log.info('Oracle Connection Closed')
except AttributeError:
log.warn('No Oracle Cursor Open')
def _parse_attr(self,xml):
log.debug('parse attribute')
attrs = xml.attributes
if attrs.has_key('name') and attrs.has_key('value'):
name = attrs['name']
value = attrs['value']
parent._context[name] = value
else:
log.warn('invalid attribute tag')
def _parse_attr(self, xml):
log.debug('parse attribute')
attrs = xml.attributes
if attrs.has_key('name') and attrs.has_key('value'):
name = attrs['name']
value = attrs['value']
parent._context[name] = value
else:
log.warn('invalid attribute tag')
def _sbdix(self, data, mode):
d = re.search(".+SBDIX\: (.+)", data)
if d:
log.debug("SBDIX values - %s" % d.group(1))
if len(d.group(1).split(",")) > 1:
status = {}
status["mostatus"] = int(d.group(1).split(",")[0])
status["momsn"] = int(d.group(1).split(",")[1])
status["mtstatus"] = int(d.group(1).split(",")[2])
status["mtmsn"] = int(d.group(1).split(",")[3])
status["mtlength"] = int(d.group(1).split(",")[4])
status["mtqueued"] = int(d.group(1).split(",")[5])
# for key in status.keys():
# log.debug("%s - %s" % (key, status[key]))
if mode[0] == "listen":
log.debug("***** Listen mode.")
if status["mtstatus"] == 1:
self.mtqueue = status["mtqueued"]
return "AT+SBDRT"
elif status["mtstatus"] == 2:
log.info(
"Message failure, retrying in %s seconds...\n\
sbdix: %s" % (self.retry, status)
)
self._try_again()
return "AT+SBDIX"
else:
log.warn(
"No message available on server"
)
return "AT\nOK"
elif mode[0] == "send":
log.debug("***** Send mode.")
if status["mostatus"] < 4:
log.debug("MO message sent.")
self.mtqueue = status["mtqueued"]
if status["mtstatus"] == 1:
self.holding_mt = True
else:
self.holding_mt = False
return "CLEAR"
else:
log.info(
"Message failed to send retrying in %s seconds..."
% self.retry
)
self._try_again()
return "AT+SBDIX"
else:
log.debug("wtf mode")
else:
log.warn("Failed to split incoming message.")
def finish(self):
log.warn('Connection from {} lost'.format(self.client_address),
game=self.game)
if self.game is not None and self.player is not None and self.player.in_game:
self.game.remove_player(self.player)
if not self.observer:
game_db.add_action(self.game_idx,
Action.LOGOUT,
player_idx=self.player.idx)
self.HANDLERS.pop(id(self))
def handleConvert():
try:
kind = request.json['type']
font = request.json['font']
log.info('conerting font..')
converted = convert.convertFont(font, kind)
return make_response(jsonify(converted), 200)
except:
log.warn("converting font went wrong")
return make_response(jsonify(error="subsetting went wrong"), 500)
def put_records(kclient, records):
resp = kclient.put_records(
Records=records, StreamName=config["KINESIS_STREAM_NAME"]
)
if resp["FailedRecordCount"]:
log.warn(
"PutRecords call with %d records had a failure count of %d",
len(records),
resp["FailedRecordCount"],
)
def handleSubset():
try:
text = request.json['text']
font = request.json['font']
log.info("subsetting font..")
subset = subset.subsetFont(font, text)
return make_response(jsonify(subset), 200)
except:
log.warn("subsetting font went wrong", request)
return make_response(jsonify(error="subsetting went wrong"), 500)
def _threadFuncReadIncoming(self):
while True:
try:
if self._quit:
break
if not self.isConnected and self._tryToConnect:
try:
self.socket.connect(self.addr)
self.isConnected = True
log.info("Connected to server!")
except Exception as ex:
log.warn("Failed to connect to server: %s",
ex.__class__.__name__)
log.debug("... reason for failure: %s", ex)
self.isConnected = False
self._tryToConnect = False
if not self.isConnected:
time.sleep(0.5)
if self.isConnected:
# Get header
data, addr = self.socket.recvfrom(NetMessage.HEADER_LENGTH)
msgid = int(data[:4])
msgsize = int(data[5:])
type = NetMessageType.CLASS_LOOKUP[msgid]
# Get body based on header
totalSizeReceived = 0
data = b''
while totalSizeReceived < msgsize:
data_part, addr = self.socket.recvfrom(
msgsize - totalSizeReceived)
data += data_part
totalSizeReceived = len(data)
# Completed receiving full message
log.debug("Received message '%s' (%sbytes)", type.__name__,
msgsize)
inMsg = IncomingMessage(msgid, type, data, msgsize, addr)
self.incomingMessages.append(inMsg)
except (ConnectionResetError, ConnectionAbortedError):
log.debug("Connection lost")
self.isConnected = False
self.socket = None
except Exception as ex:
log.error("Exception in thread: %s", ex.__class__.__name__)
log.debug("... exception details: %s", ex)
break
self.isConnected = False
log.debug("THREAD ENDED!")
def train(self, model, data):
if os.path.exists(model):
log.warn("model already exists: {}".format(model))
if not os.path.exists(data):
log.error("data file does not exist: {}".format(data))
options = (config['vw-options'] or "") + VWTrainer.DEFAULT_OPTIONS
log.info("train VW: {}".format(model))
cmd.run("{vw}/vowpalwabbit/vw -f {model} -d {data} -c {options}" \
.format(vw=self.vw, model=model, data=data, options=options))
def request_json(self, request):
request_data = json.dumps(request)
reply = self.request(request_data)
if reply is None:
return None
try:
return json.loads(reply)
except Exception as ex:
log.warn('tcp_request_parse_json_fail|address=%s,port=%u,reply=%s,ex=%s', self._address, self._port, reply, ex)
self.close()
return None
def receive(self):
if self._socket is None:
if not self._connect():
return None
try:
length_data = self._recv(4)
length = struct.unpack('<I', length_data)[0]
return self._recv(length)
except Exception as ex:
log.warn('tcp_recv_fail|address=%s,port=%u,ex=%s', self._address, self._port, ex, exc_info=True)
self.close()
return None
async def write(ws, data):
"""
往websocket连接写数据
data支持json字符串，或者GamePacket对象
"""
if ws:
if isinstance(data, GamePacket):
await ws.send(data.dump())
else:
await ws.send(data)
else:
log.warn('ws is closed.')
def tonedcharactersfromreading(characters, words):
try:
visitor = TonedCharactersFromReadingVisitor(characters)
words = [word.map(visitor) for word in words]
if len(visitor.characters) > 0:
raise TonedCharactersFromReadingException("We had some residual characters: " + visitor.characters)
else:
return words
except TonedCharactersFromReadingException, e:
# Fall back on the untoned characters
log.warn("Couldn't produce toned characters for %s because: %s", characters, e)
return [Word(Text(characters))]
def _threadFuncReadIncoming(self):
while True:
try:
if self._quit:
break
if not self.isConnected and self._tryToConnect:
try:
self.socket.connect(self.addr)
self.isConnected = True
log.info("Connected to server!")
except Exception as ex:
log.warn("Failed to connect to server: %s", ex.__class__.__name__)
log.debug("... reason for failure: %s", ex)
self.isConnected = False
self._tryToConnect = False
if not self.isConnected:
time.sleep(0.5)
if self.isConnected:
# Get header
data, addr = self.socket.recvfrom(NetMessage.HEADER_LENGTH)
msgid = int(data[:4])
msgsize = int(data[5:])
type = NetMessageType.CLASS_LOOKUP[msgid]
# Get body based on header
totalSizeReceived = 0
data = b''
while totalSizeReceived < msgsize:
data_part, addr = self.socket.recvfrom(msgsize - totalSizeReceived)
data += data_part
totalSizeReceived = len(data)
# Completed receiving full message
log.debug("Received message '%s' (%sbytes)", type.__name__, msgsize)
inMsg = IncomingMessage(msgid, type, data, msgsize, addr)
self.incomingMessages.append(inMsg)
except (ConnectionResetError, ConnectionAbortedError):
log.debug("Connection lost")
self.isConnected = False
self.socket = None
except Exception as ex:
log.error("Exception in thread: %s", ex.__class__.__name__)
log.debug("... exception details: %s", ex)
break
self.isConnected = False
log.debug("THREAD ENDED!")
def compose_feature_set(name, base_feature, set_name, extra=[]):
global FEATURE_SETS
if set_name not in FEATURE_SETS:
log.error("feature set '{}' does not exist!".format(set_name))
return None
if name in FEATURE_SETS and FEATURE_SETS[name] is not None:
log.warn("feature set '{}' exist and is not empty!".format(name))
return None
FEATURE_SETS[name] = list(set([base_feature + '&' + feat
for feat in FEATURE_SETS[set_name]] + extra))
return len(FEATURE_SETS[name])
def update_edits(self, edits, err_toks, fact_toks):
factor_id = FACTORS.TAGS[self.ngrams.factor]
err_tags = fact_toks[factor_id]
if err_tags is None:
log.warn("Missing required '{}' tags!".format(self.ngrams.factor))
return edits
new_edits = self.find_nulls(err_toks, err_tags, edits)
n = len(new_edits) - len(edits)
if n > 0:
log.info("Found {} more edits".format(n))
return new_edits
def parse_config(self, config_path, secrets_path):
config_file = open(config_path)
log.info("Using configuration from file: {}".format(config_path))
json_config = config_file.read()
self.content.update(json.loads(json_config))
if secrets_path and os.path.isfile(secrets_path):
secrets_file = open(secrets_path)
log.info("Using secrets from file: {}".format(secrets_path))
json_secrets = secrets_file.read()
self.content.update(json.loads(json_secrets))
else:
log.warn("No secrets file found, trying to import secrets from configuration")
def get_class_data(url):
"""
Get class config data.
:param url: <str> class config url
:return: <dict> config data
"""
try:
with open(url, "r") as conf:
data = json.load(conf)
conf.close()
except Exception:
data = None
logs.warn("Upload class path '%s' do not exit." % url)
return data
def __calculate_num_of_AA_cnfs_to_remove(error_rate, matrix):
log.debug("base error rate: {}, desired error rate: {}" \
.format(matrix.error_rate(), error_rate))
if error_rate < matrix.error_rate():
log.warn("decreasing of error rate is not supported")
return 0
num_AB_edits = matrix.num_of_AB_edits()
new_AA_edits = int(math.floor(num_AB_edits * (1 / error_rate) - num_AB_edits))
num_rm = max(0, matrix.num_of_AA_edits() - new_AA_edits)
log.debug("number of AA confusions to remove: {}".format(num_rm))
return num_rm
def report_action(self, action, params=None, description=None):
"""Save information about an interesting action.
@param action (str) The action to save in the context.
@param params (list or str) Any parameter values for the
action.
@param description (str) A human readable description of the
action.
"""
# Make sure all the action info is a proper string.
try:
if (isinstance(action, str)):
action = unidecode.unidecode(action.decode('unicode-escape'))
except UnicodeDecodeError:
action = ''.join(filter(lambda x: x in string.printable, action))
if (isinstance(params, str)):
try:
decoded = params.replace(
"\\", "#ESCAPED_SLASH#").decode('unicode-escape').replace(
"#ESCAPED_SLASH#", "\\")
params = unidecode.unidecode(decoded)
except Exception as e:
log.warn("Unicode decode of action params failed. " + str(e))
params = ''.join(
filter(lambda x: x in string.printable, params))
try:
if (isinstance(description, str)):
description = unidecode.unidecode(
description.decode('unicode-escape'))
except UnicodeDecodeError as e:
log.warn("Unicode decode of action description failed. " + str(e))
description = ''.join(
filter(lambda x: x in string.printable, description))
# Throttle actions that happen a lot.
action_tuple = (action, params, description)
action_str = safe_str_convert(action_tuple)
if (action_str not in self.action_count):
self.action_count[action_str] = 0
self.action_count[action_str] += 1
if (self.action_count[action_str] < self.action_limit):
self.actions.append(action_tuple)
log.info("ACTION: %s - params %r - %s" %
(action, params, description))
def delete_group(self, group_email):
log.warn("Deleting Group: %s" % group_email)
group_id = self._groupid_from_groupemail(group_email)
try:
# Delete Group from Google
self.group_delete_group(group_email)
except Exception, e:
if e.error_code == 1301:
log.exception(e)
# Entity Exists
else:
log.exception(e)
raise e
def send_message(self, topic, payload):
if not self.is_connected():
log.warn("tried to publish a message while not connected ...")
return
if 'unitID' not in payload:
payload['unitID'] = self._unitID
if( payload['unitID'] is None ):
log.warn("tried to publish a message while not having a unitID ... aborting")
return
res, mid = self._connection.publish(topic, json.dumps(payload))
if res != mqtt_client.MQTT_ERR_SUCCESS:
log.error("on message published to topic " + topic)
def run_service(self):
while True:
clusters = self.get_clusters()
random.shuffle(clusters)
if not clusters:
log.warn(f'No clusters were found, waiting {RETRY_INTERVAL/60} min')
time.sleep(RETRY_INTERVAL)
break
cluster_count = len(clusters)
for i, cluster in enumerate(clusters):
cluster_id = cluster["id"]
log.info(f"{i}/{cluster_count}: Starting process of cluster {cluster_id}")
self.process_cluster(cluster)
def main():
args = handle_arguments()
while True:
try:
scrape_events = ScrapeEvents(inventory_url=args.inventory_url,
offline_token=args.offline_token,
index=args.index,
es_server=args.es_server,
es_user = args.es_user,
es_pass = args.es_pass,
backup_destination=args.backup_destination)
scrape_events.run_service()
except Exception as EX:
log.warn(f"Elastefying logs failed with error {EX}, sleeping for {RETRY_INTERVAL} and retrying")
time.sleep(RETRY_INTERVAL)
def suspended_users(bixbycursor, usertype=None, appclient=None):
"""This function handles the suspended users"""
if not usertype:
usertype = 'NULL'
appclient = google.appsclient.AppsClient()
bixbycursor.execute(queries.get_suspended_users, (usertype,))
suspended_user_list = bixbycursor.fetchall()
for user in suspended_user_list:
uid, username, domain, userstatus, old_ustat, suspendedaccount, old_suspend, utype = user
# Suspend Staff
suspend = True
try:
if (userstatus == 2 or suspendedaccount == 1) and utype == 'Staff':
appclient.suspend_user(username, utype)
log.info('Suspending %s UID: %s %s' %(utype, uid, username))
# Suspend Students
elif (userstatus == 0 and suspendedaccount == 1) and utype == 'Student':
appclient.suspend_user(username, utype)
log.info('Suspending %s UID: %s %s' %(utype, uid, username))
# Need to add a suspend account for students who are exited. May look at date too?
# Restore Any Un-Suspended Account
elif userstatus == 0 and suspendedaccount == 0:
appclient.restore_user(username, utype)
log.info('Restoring %s UID: %s %s' %(utype, uid, username))
else:
suspend = False
log.debug('Suspended %s Uesr %s %s not Suspended' %(utype, uid, username) )
# Add check user leave date. Mark account for deletion.
# THIS UPDATES THE RECORD if they should be suspended
if suspend:
bixbycursor.execute(queries.update_suspended_user, (userstatus, suspendedaccount, uid))
else:
log.debug('Suspended User Not Suspended')
except gdata.apps.service.AppsForYourDomainException, e:
if e.error_code == 1301:
log.warn('%s User UID: %s %s Account Does Not Exist! MARKED DELETED' %(utype, uid, username))
# This updates the record
bixbycursor.execute(queries.update_suspended_user, (userstatus, suspendedaccount, uid))
# This sets the account as deleted ACCOUNT_STATUS = 5 Account does not exist/deleted
bixbycursor.execute(queries.update_account_status, (5, uid))
def _add_to_known_hosts(self, node):
log.info("Configuring local known_hosts file")
user_home = os.path.expanduser('~')
khosts = os.path.join(user_home, '.ssh', 'known_hosts')
if not os.path.isfile(khosts):
log.warn("Unable to configure known_hosts: file does not exist")
return
contents = open(khosts).read()
if node.dns_name not in contents:
server_pkey = node.ssh.get_server_public_key()
khostsf = open(khosts, 'a')
if contents[-1] != '\n':
khostsf.write('\n')
name_entry = '%s,%s' % (node.dns_name, node.ip_address)
khostsf.write(' '.join([name_entry, server_pkey.get_name(),
base64.b64encode(str(server_pkey)), '\n']))
khostsf.close()
def sum_feature_sets(name, set_names):
global FEATURE_SETS
if name in FEATURE_SETS and FEATURE_SETS[name] is not None:
log.warn("feature set '{}' exist and is not empty!".format(name))
return None
if type(set_names) == type(''):
set_names = set_names.split()
result = []
for set_name in set_names:
if set_name not in FEATURE_SETS:
log.error("feature set '{}' does not exist!".format(set_name))
else:
result += FEATURE_SETS[set_name]
FEATURE_SETS[name] = list(set(result))
return len(FEATURE_SETS[name])
def tonedcharactersfromreading(characters, reading_tokens):
# If we can't associate characters with tokens on a one-to-one basis we had better give up
if len(characters) != len(reading_tokens):
log.warn("Couldn't produce toned characters for %s because there are a different number of reading tokens than characters", characters)
return [Text(characters)]
# Add characters to the tokens /without/ spaces between them, but with tone info
tokens = []
for character, reading_token in zip(characters, reading_tokens):
# Avoid making the numbers from the supplementary dictionary into toned
# things, because it confuses users :-)
if hasattr(reading_token, "toneinfo") and not(character.isdecimal()):
tokens.append(TonedCharacter(character, reading_token.toneinfo))
else:
# Sometimes the tokens do not have tones (e.g. in the translation for T-shirt)
tokens.append(Text(character))
return tokens
def _start_notebook(self, master, user, profile_dir):
log.info("Setting up IPython web notebook for user: %s" % user)
user_cert = posixpath.join(profile_dir, '%s.pem' % user)
ssl_cert = posixpath.join(profile_dir, '%s.pem' % user)
if not master.ssh.isfile(user_cert):
log.info("Creating SSL certificate for user %s" % user)
ssl_subj = "/C=US/ST=SC/L=STAR/O=Dis/CN=%s" % master.dns_name
master.ssh.execute(
"openssl req -new -newkey rsa:4096 -days 365 "
'-nodes -x509 -subj %s -keyout %s -out %s' %
(ssl_subj, ssl_cert, ssl_cert))
else:
log.info("Using existing SSL certificate...")
f = master.ssh.remote_file('%s/ipython_notebook_config.py' %
profile_dir)
notebook_port = 8888
sha1py = 'from IPython.lib import passwd; print passwd("%s")'
sha1cmd = "python -c '%s'" % sha1py
sha1pass = master.ssh.execute(sha1cmd % self.notebook_passwd)[0]
f.write('\n'.join([
"c = get_config()",
"c.IPKernelApp.pylab = 'inline'",
"c.NotebookApp.certfile = u'%s'" % ssl_cert,
"c.NotebookApp.ip = '*'",
"c.NotebookApp.open_browser = False",
"c.NotebookApp.password = u'%s'" % sha1pass,
"c.NotebookApp.port = %d" % notebook_port,
]))
f.close()
master.ssh.execute_async("ipython notebook")
group = master.cluster_groups[0]
world_cidr = '0.0.0.0/0'
port_open = master.ec2.has_permission(group, 'tcp', notebook_port,
notebook_port, world_cidr)
if not port_open:
log.info("Authorizing tcp port %s on %s" %
(notebook_port, world_cidr))
group.authorize('tcp', notebook_port, notebook_port, world_cidr)
log.info("IPython notebook URL: https://%s:%s" %
(master.dns_name, notebook_port))
log.info("The notebook password is: %s" % self.notebook_passwd)
log.warn("Please check your local firewall settings if you're having "
"issues connecting to the IPython notebook",
extra=dict(__textwrap__=True))
def shell(self, user=None, forward_x11=False, command=None):
"""
Attempts to launch an interactive shell by first trying the system's
ssh client. If the system does not have the ssh command it falls back
to a pure-python ssh shell.
"""
if self.update() != 'running':
try:
alias = self.alias
except exception.BaseException:
alias = None
label = 'instance'
if alias == "master":
label = "master"
elif alias:
label = "node"
instance_id = alias or self.id
raise exception.InstanceNotRunning(instance_id, self.state,
label=label)
user = user or self.user
if utils.has_required(['ssh']):
log.debug("Using native OpenSSH client")
sshopts = '-i %s' % self.key_location
if forward_x11:
sshopts += ' -Y'
ssh_cmd = static.SSH_TEMPLATE % dict(opts=sshopts, user=user,
host=self.dns_name)
if command:
command = "'source /etc/profile && %s'" % command
ssh_cmd = ' '.join([ssh_cmd, command])
log.debug("ssh_cmd: %s" % ssh_cmd)
return subprocess.call(ssh_cmd, shell=True)
else:
log.debug("Using Pure-Python SSH client")
if forward_x11:
log.warn("X11 Forwarding not available in Python SSH client")
if command:
orig_user = self.ssh.get_current_user()
self.ssh.switch_user(user)
self.ssh.execute(command, silent=False, source_profile=True)
self.ssh.switch_user(orig_user)
return self.ssh.get_last_status()
self.ssh.interactive_shell(user=user)
def update_proper_names(bixbycursor, usertype=None, appclient=None):
log.info("""######### Updating Propper Names ########""")
if not usertype:
usertype = 'NULL' # There has to be a better way BEH
appclient = google.appsclient.AppsClient()
bixbycursor.execute(queries.get_changed_propper_name, (usertype,))
changed_proper_names = bixbycursor.fetchall()
for user in changed_proper_names:
try:
uid, username, old_lastname, new_lastname, old_firstname, new_firstname = user
log.debug('Changing Propper Name for User: %s From: %s %s To: %s %s' %(uid, old_firstname, old_lastname, new_firstname, new_lastname))
appclient.modify_propper_name(username, new_firstname, new_lastname, usertype)
bixbycursor.execute(queries.update_propper_name, (new_firstname, new_lastname, uid))
except gdata.apps.service.AppsForYourDomainException, e:
if e.error_code == 1301:
log.warn('User: %s Does Not Exist!' %username)
# Mark the account deleted ACCOUNT_STATUS = 5 Account does not exist/deleted
bixbycursor.execute(queries.update_account_status, (5, uid))
def load_setting_file(args):
setting_file = args.model + '.settings'
log.info("loading setting file: {}".format(setting_file))
if not os.path.exists(setting_file):
log.warn("setting file {} does not exist".format(setting_file))
return args
with open(setting_file) as file:
for line in file:
opt, val = line.strip().split('=')
if opt == 'use' and not args.algorithm:
log.debug("loading algorithm: {}".format(val))
args.algorithm = val
elif opt == 'set' and not args.confusion_set:
log.debug("loading confusion set: {}".format(val))
args.confusion_set = val
elif opt == 'cnf' and not args.cnf_opts:
log.debug("loading confusion extraction options: {}".format(val))
args.cnf_opts = val
elif opt == 'mng' and not args.mng_opts:
log.debug("loading confusion managing options: {}".format(val))
args.mng_opts = val
elif opt == 'ext' and not args.ext_opts:
log.debug("loading feature extraction options: {}".format(val))
args.ext_opts = val
elif opt == 'vec' and not args.vec_opts:
log.debug("loading vectorization options: {}".format(val))
args.vec_opts = val
elif opt == 'cls' and not args.cls_opts:
log.debug("loading classification options: {}".format(val))
args.cls_opts = val
elif opt == 'evl' and not args.evl_opts:
log.debug("loading evaluation options: {}".format(val))
args.evl_opts = val
else:
log.warn("unrecognized setting: {}={}".format(opt, val))
return args
def start_worker_thread(self, app, dev, cer_file, key_file, conf):
kwargs = {
'develop': dev,
'app_key': app,
'cer_file': cer_file,
'key_file': key_file,
'server_info': self.server_info
}
# 检查配置里关于推送线程数的配置，启动相应数量的推送线程
thread_cnt = 1
log.debug('startting worker with config : %s' % conf)
if conf:
conf_prefix = 'config_dev:' if dev else 'config:'
self.rds.hmset(conf_prefix + app, conf)
try:
thread_cnt = int(conf.get('worker', '1'))
except:
log.warn('invalid config of workers, just start one.')
workers = []
for cnt in range(thread_cnt):
if cnt == 0:
params = kwargs
worker = threading.Thread(target=self.push, kwargs=params)
else:
params = kwargs.copy()
params.update({'channel': cnt})
worker = threading.Thread(target=self.push, kwargs=params)
workers.append(worker)
feedback_job = threading.Thread(target=self.feedback, kwargs=kwargs)
enhance_job = threading.Thread(target=self.enhance, kwargs=kwargs)
feedback_job.setDaemon(True)
enhance_job.setDaemon(True)
for w in workers:
w.setDaemon(True)
w.start()
feedback_job.start()
enhance_job.start()
def get_file_list(self, remote_dir, is_root=False):
log.debug('now start get list')
# if is_root:
# files = self.get_roor_dir()
# else:
# files = self.sftp.listdir_attr(remote_dir)
try:
files = self.sftp.listdir_attr(remote_dir)
except Exception:
log.warn('now re connect')
self.t = paramiko.Transport((self.host, self.port))
self.connect()
self.sftp = paramiko.SFTPClient.from_transport(self.t)
files = self.sftp.listdir_attr(remote_dir)
for f in files:
filename = remote_dir + '/' + f.filename
log.debug('now search %s' % filename)
if stat.S_ISDIR(f.st_mode):
for i in self.get_file_list(filename):
yield i
else:
yield filename
def create_new_group(
self,
group_email,
group_name,
group_description,
group_type,
group_status=1,
email_permission="Owner",
domain_id=1,
department_id=None,
unique_attribute=None,
):
"""department_id, group_email, group_name, group_description, group_status
, domain_id, group_type, unique_attribute
PERMISSION_OWNER = 'Owner'
PERMISSION_MEMBER = 'Member'
PERMISSION_DOMAIN = 'Domain'
PERMISSION_ANYONE = 'Anyone'
"""
groups_fields = (
department_id,
group_email,
group_name,
group_description,
group_status,
domain_id,
group_type,
unique_attribute,
)
try:
self.group_create_group(group_email, group_name, group_description, email_permission=email_permission)
except gdata.apps.service.AppsForYourDomainException, e:
if e.error_code == 1300:
log.exception(e)
log.warn("""The Group: %s Exists. It's OK that happens.""" % group_email)
else:
raise e
