def run(self): ''' Run our entire collection of suites. ''' for logger in self.result_loggers: logger.begin_testing() log.info(separator()) log.info("Building all non 'lazy_init' fixtures") failed_builds = self.setup_unbuilt(self.suites.iter_fixtures(), setup_lazy_init=False) if failed_builds: error_str = '' for fixture, error in failed_builds: error_str += 'Failed to build %s\n' % fixture error_str += '%s' % error log.warn('Error(s) while building non lazy_init fixtures.') log.warn(error_str) outcomes = set() for suite in self.suites: outcome = self.run_suite(suite) outcomes.add(outcome) if outcome in Outcome.failfast and config.fail_fast: break for logger in self.result_loggers: logger.end_testing() return self._suite_outcome(outcomes)
def evaluate_nulls_coverage(cnfs_file, gold_cnfs_file): system_confs = analyze_cnfs_file(cnfs_file) system_values = set(system_confs.keys()) gold_confs = analyze_cnfs_file(gold_cnfs_file) gold_values = set(gold_confs.keys()) tp, fp, tn, fn = 0, 0, 0, 0 for val in gold_values.union(system_values): if val in gold_values: if val in system_values: if gold_confs[val] == '<null>' and system_confs[val] == '<null>': tp += 1 elif gold_confs[val] != '<null>' and system_confs[val] != '<null>': tn += 1 else: log.warn("not handled case!") else: fn += 1 else: if system_confs[val] == '<null>': fp += 1 else: log.warn("not handled case!") return (tp, tn, fp, fn)
def downloader_async(self, remote_dir): threads = [] for i in range(30): log.debug('-------connecting %s' % i) self._queue.put(SftpHelper(self.host, self.username, self.passwd)) log.debug('stop connect') for i in self.get_file_list(remote_dir, is_root=True): log.debug('%%%%') try: sftp_helper = self._queue.get(timeout=1) log.debug('------get %s' % i) except Queue.Empty: log.warn('No sftp is available') sftp_helper = SftpHelper(self.host, self.username, self.passwd) log.info('add new sftp') thr = threading.Thread(target=self.get_one, args=(sftp_helper, i)) threads.append(thr) thr.start() if len(threads) > 200: for t in threads: t.join(timeout=10) threads = [] for t in threads: t.join() while not self._queue.empty(): i = self._queue.get() i.sftp.close() i.t.close() self.sftp.close() self.t.close()
def copy_remote_file_to_nodes(self, remote_file, nodes, dest=None): """ Copies a remote file from this Node instance to another Node instance without passwordless ssh between the two. dest - path to store the data in on the node (defaults to remote_file) """ if not dest: dest = remote_file rf = self.ssh.remote_file(remote_file, 'r') contents = rf.read() sts = rf.stat() mode = stat.S_IMODE(sts.st_mode) uid = sts.st_uid gid = sts.st_gid rf.close() for node in nodes: if self.id == node.id and remote_file == dest: log.warn("src and destination are the same: %s, skipping" % remote_file) continue nrf = node.ssh.remote_file(dest, 'w') nrf.write(contents) nrf.chown(uid, gid) nrf.chmod(mode) nrf.close()
def get_traffic_counters(self, chain, wrap=True, zero=False): """Return the sum of the traffic counters of all rules of a chain.""" cmd_tables = self._get_traffic_counters_cmd_tables(chain, wrap) if not cmd_tables: LOG.warn(('Attempted to get traffic counters of chain %s which ' 'does not exist'), chain) return name = get_chain_name(chain, wrap) acc = {'pkts': 0, 'bytes': 0} for cmd, table in cmd_tables: args = [cmd, '-t', table, '-L', name, '-n', '-v', '-x'] if zero: args.append('-Z') if self.namespace: args = ['ip', 'netns', 'exec', self.namespace] + args current_table = self.execute(args, run_as_root=True) current_lines = current_table.split('\n') for line in current_lines[2:]: if not line: break data = line.split() if (len(data) < 2 or not data[0].isdigit() or not data[1].isdigit()): break acc['pkts'] += int(data[0]) acc['bytes'] += int(data[1]) return acc
def get_one(self, sftp_helper, remote_file): pat = re.compile(self.remote_dir) local_file = pat.sub(self.local_dir, remote_file) local_dir = os.path.dirname(local_file) if not os.path.exists(local_dir): try: os.makedirs(local_dir) except OSError: pass log.debug('Start download %s -------' % remote_file) try: sftp_helper.sftp.get(remote_file, local_file) log.info('Download %s successfully' % local_file) sftp_helper.sftp.remove(remote_file) log.info('remove %s successfully' % remote_file) # remote_dir = os.path.dirname(remote_file) # try: # FileHandler.delete_all(sftp_helper, remote_dir, self.remote_dir) # except Exception as e: # pass except Exception as e: log.warn(e) try: self._queue.put(sftp_helper, block=False) except Exception: pass
def download_logs(client: InventoryClient, cluster: dict, dest: str, must_gather: bool, retry_interval: int = RETRY_INTERVAL): output_folder = get_logs_output_folder(dest, cluster) if os.path.isdir(output_folder): log.info(f"Skipping. The logs directory {output_folder} already exists.") return recreate_folder(output_folder) recreate_folder(os.path.join(output_folder, "cluster_files")) try: write_metadata_file(client, cluster, os.path.join(output_folder, 'metdata.json')) with suppress(assisted_service_client.rest.ApiException): client.download_ignition_files(cluster['id'], os.path.join(output_folder, "cluster_files")) for host_id in map(lambda host: host['id'], cluster['hosts']): with suppress(assisted_service_client.rest.ApiException): client.download_host_ignition(cluster['id'], host_id, os.path.join(output_folder, "cluster_files")) with suppress(assisted_service_client.rest.ApiException): client.download_cluster_events(cluster['id'], os.path.join(output_folder, f"cluster_{cluster['id']}_events.json")) shutil.copy2(os.path.join(os.path.dirname(os.path.realpath(__file__)), "events.html"), output_folder) with suppress(assisted_service_client.rest.ApiException): for i in range(MAX_RETRIES): cluster_logs_tar = os.path.join(output_folder, f"cluster_{cluster['id']}_logs.tar") with suppress(FileNotFoundError): os.remove(cluster_logs_tar) client.download_cluster_logs(cluster['id'], cluster_logs_tar) min_number_of_logs = len(cluster['hosts']) + 1 if cluster['status'] == ClusterStatus.INSTALLED else len(cluster['hosts']) try: verify_logs_uploaded(cluster_logs_tar, min_number_of_logs, cluster['status'] == ClusterStatus.INSTALLED) break except AssertionError as ex: log.warn(f"Cluster logs verification failed: {ex}") # Skip sleeping on last retry if i < MAX_RETRIES - 1: log.info(f"Going to retry in {retry_interval} seconds") time.sleep(retry_interval) kubeconfig_path = os.path.join(output_folder, "kubeconfig-noingress") with suppress(assisted_service_client.rest.ApiException): client.download_kubeconfig_no_ingress(cluster['id'], kubeconfig_path) if must_gather: recreate_folder(os.path.join(output_folder, "must-gather")) config_etc_hosts(cluster['name'], cluster['base_dns_domain'], helper_cluster.get_api_vip_from_cluster(client, cluster)) download_must_gather(kubeconfig_path, os.path.join(output_folder, "must-gather")) finally: run_command(f"chmod -R ugo+rx '{output_folder}'")
def run(self): random.seed() self.on_init() from gtcpclient import GTcpClient log.info('tcp_worker_start|id=%d', self._id) self._client = GTcpClient(self._config.WORK_ENDPOINT['address'], self._config.WORK_ENDPOINT['port'], 0) while True: try: request = self._client.receive() if request is None: log.warn('tcp_worker_lost_connection|client_id=%s,client=%s', self._client.id.encode('hex'), self._client.remote_address) self._client.close() elif len(request) < GTCP_HEADER_SIZE: log.error('tcp_worker_request_packet_error|client_id=%s,client=%s,request=%s', self._client.id.encode('hex'), self._client.remote_address, request.encode('hex')) self._client.close() else: request_cmd = request[:GTCP_CMD_SIZE] request_client = TcpEndpoint(request[GTCP_CMD_SIZE:GTCP_HEADER_SIZE]) reply_body = None if request_cmd == GTCP_CMD_RELAY: request_body = request[GTCP_HEADER_SIZE:] reply_body = self.on_packet(request_client, request_body) elif request_cmd == GTCP_CMD_CONNECT: reply_body = self.on_client_connect(request_client) elif request_cmd == GTCP_CMD_DISCONNECT: self.on_client_disconnect(request_client) if reply_body is None: self._client.send(GTCP_CMD_NONE + request_client.client_id) else: self._client.send(GTCP_CMD_RELAY + request_client.client_id + reply_body) except Exception as ex: log.exception('tcp_worker_exception|id=%u,exception=%s', self._id, ex, exc_info=True) self._client.close()
def report_action(self, action, params=None, description=None): """ Callback function for each evaluated statement to report macro actions """ # store the action for later use: try: if (isinstance(action, str)): action = unidecode.unidecode(action.decode('unicode-escape')) except UnicodeDecodeError: action = ''.join(filter(lambda x:x in string.printable, action)) if (isinstance(params, str)): try: decoded = params.replace("\\", "#ESCAPED_SLASH#").decode('unicode-escape').replace("#ESCAPED_SLASH#", "\\") params = unidecode.unidecode(decoded) except Exception as e: log.warn("Unicode decode of action params failed. " + str(e)) params = ''.join(filter(lambda x:x in string.printable, params)) try: if (isinstance(description, str)): description = unidecode.unidecode(description.decode('unicode-escape')) except UnicodeDecodeError as e: log.warn("Unicode decode of action description failed. " + str(e)) description = ''.join(filter(lambda x:x in string.printable, description)) self.actions.append((action, params, description)) log.info("ACTION: %s - params %r - %s" % (action, params, description))
def parsexml(self,xml): log.debug('in parsexml') # 如果传进来的是文件,则直接读取内容 # 如果是字符串,则先尝试解释,如果不是xml则再尝试作为文件路径,再不行,则抛异常了。 xml_content = '' if type(xml) == file: xml_content = xml.read() xml.close() elif type(xml) == str or type(xml) == unicode: log.debug('try to load file') if os.path.exists(xml): xml_file = open(xml,'r') xml_content = xml_file.read() xml_file.close() else: xml_content = xml else: log.error('could not init testcase from xml') raise TypeError,'fromxml need a file instance or a string instance for argument' log.debug('starting parse xml') log.debug('xml content: %s'%xml_content) doc = minidom.parseString(xml_content) ret = get_child_tags(doc,'test')#this statement return a list log.debug('child tag len : %d'%len(ret)) if ret: test_node = ret[0] return test_node else: log.warn('no test node in the xml!')
def parse_radius_log(radius_log_file, out_dir): log_file = open(radius_log_file, 'r') outfile_name = "RADIUS_MAC2NSP_%s.txt" % (DATE.strftime('%Y%m%d')) output_file = open(os.path.join(out_dir, outfile_name), 'w') existing = set() try: for record in rad_recv_record_extractor(log_file): for converter_func, check_func in CONVERTER_CHAIN: if check_func(record): try: mac, nssp = converter_func(record) except Exception, e: #logging.error("Error passing radius record: " + repr(e)) log.exception('Error parsing radius record') continue mac2nsp = "%s %s" % (mac, nssp) if mac2nsp in existing: log.warn("Duplicate record found: %s, ignoring..." % mac2nsp) else: log.info("Matched: %s %s" % (mac, nssp)) output_file.write("%s %s\n" % (mac, nssp)) existing.add(mac2nsp) break else: log.info("Radius record not matching rule: " + repr(record)) finally: log_file.close() output_file.close()
async def onData(self, ws, gamePacket): """ 收到ws数据包时回调 """ log.info('ws received:' + gamePacket.dump() + ' ws:' + str(ws)) protocol = gamePacket.protocol roomId = ws_connects_mgr.getRoomId(ws) uid = ws_connects_mgr.getUid(ws) room = room_manager.get_room(roomId) log.info("handle uid:{0} protocol:{1} of room:{2}".format( uid, protocol, room.roomId if room else '')) if not protocol or not room: return # 根据客户端的包,来执行相应的操作 if protocol == PROTOCOL.C2SHeartbeat: # 游戏心跳包 # 原样返回 await ws_channel_util.write( ws, GamePacket(PROTOCOL.S2CHeartbeat, gamePacket.payload)) elif protocol == PROTOCOL.C2SSignup: # 加入游戏 room.signup(uid, json.loads(gamePacket.payload)) elif protocol == PROTOCOL.C2SStart: # 开始游戏 room.start(uid) elif protocol == PROTOCOL.C2SGameover: # 主播主动结束游戏 room.gameOver(uid) else: log.warn("protocol invalid:{0},{1}".format(protocol), gamePacket.dump()) pass pass
def request(self, request): if self._socket is None: if not self._connect(): return None packet = struct.pack('<I%ds' % len(request), len(request), request) try: self._socket.sendall(packet) length_data = self._recv(4) except Exception as ex: self.close() if not self._retry: log.exception('tcp_request_fail|error=recv_length_fail,address=%s,port=%u,request=%s', self._address, self._port, request.encode('hex')) return None if isinstance(ex, socket.timeout): log.exception('tcp_request_fail|error=recv_length_timeout,address=%s,port=%u,retry=0,request=%s', self._address, self._port, request.encode('hex')) return None log.warn('tcp_request_fail|error=recv_length_fail_will_retry,address=%s,port=%u,retry=0,request=%s,ex=%s', self._address, self._port, request.encode('hex'), ex) if not self._connect(): log.exception('tcp_request_fail|error=retry_reconnect,address=%s,port=%u,retry=0,request=%s', self._address, self._port, request.encode('hex')) return None try: self._socket.sendall(packet) length_data = self._recv(4) except Exception as ex: log.exception('tcp_request_fail|error=retry_recv_length_fail,address=%s,port=%u,retry=1,request=%s', self._address, self._port, request.encode('hex')) self.close() return None try: length = struct.unpack('<I', length_data)[0] return self._recv(length) except Exception as ex: log.exception('tcp_request_fail|error=recv_data_fail,address=%s,port=%u,request=%s', self._address, self._port, request.encode('hex')) self.close() return None
def add_compiled_module(self, m, stream): """Add an already parsed and processed module. @param m (Module object) The parsed object. @param stream (str) The OLE stream name containing the module. """ if (m is None): return self.modules.append(m) for name, _sub in m.subs.items(): # Append the stream name for duplicate subs if (name in self.globals): new_name = safe_str_convert(stream) + "::" + safe_str_convert( name) log.warn("Renaming duplicate function " + name + " to " + new_name) name = new_name # Save the sub. if (log.getEffectiveLevel() == logging.DEBUG): log.debug('(1) storing sub "%s" in globals' % name) self.globals[name.lower()] = _sub self.globals[name] = _sub # Functions. for name, _function in m.functions.items(): if (log.getEffectiveLevel() == logging.DEBUG): log.debug('(1) storing function "%s" in globals' % name) self.globals[name.lower()] = _function self.globals[name] = _function # Properties. for name, _prop in m.functions.items(): if (log.getEffectiveLevel() == logging.DEBUG): log.debug('(1) storing property let "%s" in globals' % name) self.globals[name.lower()] = _prop self.globals[name] = _prop # External DLL functions. for name, _function in m.external_functions.items(): if (log.getEffectiveLevel() == logging.DEBUG): log.debug('(1) storing external function "%s" in globals' % name) self.globals[name.lower()] = _function self.externals[name.lower()] = _function # Global variables. for name, _var in m.global_vars.items(): if (log.getEffectiveLevel() == logging.DEBUG): log.debug('(1) storing global var "%s" = %s in globals (1)' % (name, safe_str_convert(_var))) if (isinstance(name, str)): self.globals[name.lower()] = _var if (isinstance(name, list)): self.globals[name[0].lower()] = _var self.types[name[0].lower()] = name[1]
def _get_ipcluster_plugin(self, node): ipyversion = self._get_ipy_version(node) if ipyversion < '0.11': if not ipyversion.startswith('0.10'): log.warn("Trying unsupported IPython version %s" % ipyversion) return IPCluster10() else: return IPCluster11(self.enable_notebook, self.notebook_passwd)
def read(self, strData): self.autoDeserialize(strData) if self.error is not None: log.error(self.error) if self.info is not None: log.info(self.info) if self.warning is not None: log.warn(self.warning)
def __live_disconnect(self): """ 订阅服务连接断开回调 """ log.warn("监听直播间订阅消息连接断开:{}".format(self.roomId)) if self.state != ERoomState.GameOver: reTrytimer = threading.Timer( 2+self.reconnTimes*2, self.__retry_connect_live_subscriber) reTrytimer.start()
def _check_required_parameters(self): parameters_found = True for param_path in self._parameters["required"]: value = get_dict_value(self._configuration, param_path.split('.')) if value in [None, ""]: parameters_found = False log.warn("Required parameter is empty: {}".format(param_path)) return parameters_found
def close(self): try: self.cursor self.cursor.close() self.connection.close() log.info('Oracle Connection Closed') except AttributeError: log.warn('No Oracle Cursor Open')
def _parse_attr(self,xml): log.debug('parse attribute') attrs = xml.attributes if attrs.has_key('name') and attrs.has_key('value'): name = attrs['name'] value = attrs['value'] parent._context[name] = value else: log.warn('invalid attribute tag')
def _parse_attr(self, xml): log.debug('parse attribute') attrs = xml.attributes if attrs.has_key('name') and attrs.has_key('value'): name = attrs['name'] value = attrs['value'] parent._context[name] = value else: log.warn('invalid attribute tag')
def _sbdix(self, data, mode): d = re.search(".+SBDIX\: (.+)", data) if d: log.debug("SBDIX values - %s" % d.group(1)) if len(d.group(1).split(",")) > 1: status = {} status["mostatus"] = int(d.group(1).split(",")[0]) status["momsn"] = int(d.group(1).split(",")[1]) status["mtstatus"] = int(d.group(1).split(",")[2]) status["mtmsn"] = int(d.group(1).split(",")[3]) status["mtlength"] = int(d.group(1).split(",")[4]) status["mtqueued"] = int(d.group(1).split(",")[5]) # for key in status.keys(): # log.debug("%s - %s" % (key, status[key])) if mode[0] == "listen": log.debug("***** Listen mode.") if status["mtstatus"] == 1: self.mtqueue = status["mtqueued"] return "AT+SBDRT" elif status["mtstatus"] == 2: log.info( "Message failure, retrying in %s seconds...\n\ sbdix: %s" % (self.retry, status) ) self._try_again() return "AT+SBDIX" else: log.warn( "No message available on server" ) return "AT\nOK" elif mode[0] == "send": log.debug("***** Send mode.") if status["mostatus"] < 4: log.debug("MO message sent.") self.mtqueue = status["mtqueued"] if status["mtstatus"] == 1: self.holding_mt = True else: self.holding_mt = False return "CLEAR" else: log.info( "Message failed to send retrying in %s seconds..." % self.retry ) self._try_again() return "AT+SBDIX" else: log.debug("wtf mode") else: log.warn("Failed to split incoming message.")
def finish(self): log.warn('Connection from {} lost'.format(self.client_address), game=self.game) if self.game is not None and self.player is not None and self.player.in_game: self.game.remove_player(self.player) if not self.observer: game_db.add_action(self.game_idx, Action.LOGOUT, player_idx=self.player.idx) self.HANDLERS.pop(id(self))
def handleConvert(): try: kind = request.json['type'] font = request.json['font'] log.info('conerting font..') converted = convert.convertFont(font, kind) return make_response(jsonify(converted), 200) except: log.warn("converting font went wrong") return make_response(jsonify(error="subsetting went wrong"), 500)
def put_records(kclient, records): resp = kclient.put_records( Records=records, StreamName=config["KINESIS_STREAM_NAME"] ) if resp["FailedRecordCount"]: log.warn( "PutRecords call with %d records had a failure count of %d", len(records), resp["FailedRecordCount"], )
def handleSubset(): try: text = request.json['text'] font = request.json['font'] log.info("subsetting font..") subset = subset.subsetFont(font, text) return make_response(jsonify(subset), 200) except: log.warn("subsetting font went wrong", request) return make_response(jsonify(error="subsetting went wrong"), 500)
def _threadFuncReadIncoming(self): while True: try: if self._quit: break if not self.isConnected and self._tryToConnect: try: self.socket.connect(self.addr) self.isConnected = True log.info("Connected to server!") except Exception as ex: log.warn("Failed to connect to server: %s", ex.__class__.__name__) log.debug("... reason for failure: %s", ex) self.isConnected = False self._tryToConnect = False if not self.isConnected: time.sleep(0.5) if self.isConnected: # Get header data, addr = self.socket.recvfrom(NetMessage.HEADER_LENGTH) msgid = int(data[:4]) msgsize = int(data[5:]) type = NetMessageType.CLASS_LOOKUP[msgid] # Get body based on header totalSizeReceived = 0 data = b'' while totalSizeReceived < msgsize: data_part, addr = self.socket.recvfrom( msgsize - totalSizeReceived) data += data_part totalSizeReceived = len(data) # Completed receiving full message log.debug("Received message '%s' (%sbytes)", type.__name__, msgsize) inMsg = IncomingMessage(msgid, type, data, msgsize, addr) self.incomingMessages.append(inMsg) except (ConnectionResetError, ConnectionAbortedError): log.debug("Connection lost") self.isConnected = False self.socket = None except Exception as ex: log.error("Exception in thread: %s", ex.__class__.__name__) log.debug("... exception details: %s", ex) break self.isConnected = False log.debug("THREAD ENDED!")
def train(self, model, data): if os.path.exists(model): log.warn("model already exists: {}".format(model)) if not os.path.exists(data): log.error("data file does not exist: {}".format(data)) options = (config['vw-options'] or "") + VWTrainer.DEFAULT_OPTIONS log.info("train VW: {}".format(model)) cmd.run("{vw}/vowpalwabbit/vw -f {model} -d {data} -c {options}" \ .format(vw=self.vw, model=model, data=data, options=options))
def request_json(self, request): request_data = json.dumps(request) reply = self.request(request_data) if reply is None: return None try: return json.loads(reply) except Exception as ex: log.warn('tcp_request_parse_json_fail|address=%s,port=%u,reply=%s,ex=%s', self._address, self._port, reply, ex) self.close() return None
def receive(self): if self._socket is None: if not self._connect(): return None try: length_data = self._recv(4) length = struct.unpack('<I', length_data)[0] return self._recv(length) except Exception as ex: log.warn('tcp_recv_fail|address=%s,port=%u,ex=%s', self._address, self._port, ex, exc_info=True) self.close() return None
async def write(ws, data): """ 往websocket连接写数据 data支持json字符串,或者GamePacket对象 """ if ws: if isinstance(data, GamePacket): await ws.send(data.dump()) else: await ws.send(data) else: log.warn('ws is closed.')
def tonedcharactersfromreading(characters, words): try: visitor = TonedCharactersFromReadingVisitor(characters) words = [word.map(visitor) for word in words] if len(visitor.characters) > 0: raise TonedCharactersFromReadingException("We had some residual characters: " + visitor.characters) else: return words except TonedCharactersFromReadingException, e: # Fall back on the untoned characters log.warn("Couldn't produce toned characters for %s because: %s", characters, e) return [Word(Text(characters))]
def _threadFuncReadIncoming(self): while True: try: if self._quit: break if not self.isConnected and self._tryToConnect: try: self.socket.connect(self.addr) self.isConnected = True log.info("Connected to server!") except Exception as ex: log.warn("Failed to connect to server: %s", ex.__class__.__name__) log.debug("... reason for failure: %s", ex) self.isConnected = False self._tryToConnect = False if not self.isConnected: time.sleep(0.5) if self.isConnected: # Get header data, addr = self.socket.recvfrom(NetMessage.HEADER_LENGTH) msgid = int(data[:4]) msgsize = int(data[5:]) type = NetMessageType.CLASS_LOOKUP[msgid] # Get body based on header totalSizeReceived = 0 data = b'' while totalSizeReceived < msgsize: data_part, addr = self.socket.recvfrom(msgsize - totalSizeReceived) data += data_part totalSizeReceived = len(data) # Completed receiving full message log.debug("Received message '%s' (%sbytes)", type.__name__, msgsize) inMsg = IncomingMessage(msgid, type, data, msgsize, addr) self.incomingMessages.append(inMsg) except (ConnectionResetError, ConnectionAbortedError): log.debug("Connection lost") self.isConnected = False self.socket = None except Exception as ex: log.error("Exception in thread: %s", ex.__class__.__name__) log.debug("... exception details: %s", ex) break self.isConnected = False log.debug("THREAD ENDED!")
def compose_feature_set(name, base_feature, set_name, extra=[]): global FEATURE_SETS if set_name not in FEATURE_SETS: log.error("feature set '{}' does not exist!".format(set_name)) return None if name in FEATURE_SETS and FEATURE_SETS[name] is not None: log.warn("feature set '{}' exist and is not empty!".format(name)) return None FEATURE_SETS[name] = list(set([base_feature + '&' + feat for feat in FEATURE_SETS[set_name]] + extra)) return len(FEATURE_SETS[name])
def update_edits(self, edits, err_toks, fact_toks): factor_id = FACTORS.TAGS[self.ngrams.factor] err_tags = fact_toks[factor_id] if err_tags is None: log.warn("Missing required '{}' tags!".format(self.ngrams.factor)) return edits new_edits = self.find_nulls(err_toks, err_tags, edits) n = len(new_edits) - len(edits) if n > 0: log.info("Found {} more edits".format(n)) return new_edits
def parse_config(self, config_path, secrets_path): config_file = open(config_path) log.info("Using configuration from file: {}".format(config_path)) json_config = config_file.read() self.content.update(json.loads(json_config)) if secrets_path and os.path.isfile(secrets_path): secrets_file = open(secrets_path) log.info("Using secrets from file: {}".format(secrets_path)) json_secrets = secrets_file.read() self.content.update(json.loads(json_secrets)) else: log.warn("No secrets file found, trying to import secrets from configuration")
def get_class_data(url): """ Get class config data. :param url: <str> class config url :return: <dict> config data """ try: with open(url, "r") as conf: data = json.load(conf) conf.close() except Exception: data = None logs.warn("Upload class path '%s' do not exit." % url) return data
def __calculate_num_of_AA_cnfs_to_remove(error_rate, matrix): log.debug("base error rate: {}, desired error rate: {}" \ .format(matrix.error_rate(), error_rate)) if error_rate < matrix.error_rate(): log.warn("decreasing of error rate is not supported") return 0 num_AB_edits = matrix.num_of_AB_edits() new_AA_edits = int(math.floor(num_AB_edits * (1 / error_rate) - num_AB_edits)) num_rm = max(0, matrix.num_of_AA_edits() - new_AA_edits) log.debug("number of AA confusions to remove: {}".format(num_rm)) return num_rm
def report_action(self, action, params=None, description=None): """Save information about an interesting action. @param action (str) The action to save in the context. @param params (list or str) Any parameter values for the action. @param description (str) A human readable description of the action. """ # Make sure all the action info is a proper string. try: if (isinstance(action, str)): action = unidecode.unidecode(action.decode('unicode-escape')) except UnicodeDecodeError: action = ''.join(filter(lambda x: x in string.printable, action)) if (isinstance(params, str)): try: decoded = params.replace( "\\", "#ESCAPED_SLASH#").decode('unicode-escape').replace( "#ESCAPED_SLASH#", "\\") params = unidecode.unidecode(decoded) except Exception as e: log.warn("Unicode decode of action params failed. " + str(e)) params = ''.join( filter(lambda x: x in string.printable, params)) try: if (isinstance(description, str)): description = unidecode.unidecode( description.decode('unicode-escape')) except UnicodeDecodeError as e: log.warn("Unicode decode of action description failed. " + str(e)) description = ''.join( filter(lambda x: x in string.printable, description)) # Throttle actions that happen a lot. action_tuple = (action, params, description) action_str = safe_str_convert(action_tuple) if (action_str not in self.action_count): self.action_count[action_str] = 0 self.action_count[action_str] += 1 if (self.action_count[action_str] < self.action_limit): self.actions.append(action_tuple) log.info("ACTION: %s - params %r - %s" % (action, params, description))
def delete_group(self, group_email): log.warn("Deleting Group: %s" % group_email) group_id = self._groupid_from_groupemail(group_email) try: # Delete Group from Google self.group_delete_group(group_email) except Exception, e: if e.error_code == 1301: log.exception(e) # Entity Exists else: log.exception(e) raise e
def send_message(self, topic, payload): if not self.is_connected(): log.warn("tried to publish a message while not connected ...") return if 'unitID' not in payload: payload['unitID'] = self._unitID if( payload['unitID'] is None ): log.warn("tried to publish a message while not having a unitID ... aborting") return res, mid = self._connection.publish(topic, json.dumps(payload)) if res != mqtt_client.MQTT_ERR_SUCCESS: log.error("on message published to topic " + topic)
def run_service(self): while True: clusters = self.get_clusters() random.shuffle(clusters) if not clusters: log.warn(f'No clusters were found, waiting {RETRY_INTERVAL/60} min') time.sleep(RETRY_INTERVAL) break cluster_count = len(clusters) for i, cluster in enumerate(clusters): cluster_id = cluster["id"] log.info(f"{i}/{cluster_count}: Starting process of cluster {cluster_id}") self.process_cluster(cluster)
def main(): args = handle_arguments() while True: try: scrape_events = ScrapeEvents(inventory_url=args.inventory_url, offline_token=args.offline_token, index=args.index, es_server=args.es_server, es_user = args.es_user, es_pass = args.es_pass, backup_destination=args.backup_destination) scrape_events.run_service() except Exception as EX: log.warn(f"Elastefying logs failed with error {EX}, sleeping for {RETRY_INTERVAL} and retrying") time.sleep(RETRY_INTERVAL)
def suspended_users(bixbycursor, usertype=None, appclient=None): """This function handles the suspended users""" if not usertype: usertype = 'NULL' appclient = google.appsclient.AppsClient() bixbycursor.execute(queries.get_suspended_users, (usertype,)) suspended_user_list = bixbycursor.fetchall() for user in suspended_user_list: uid, username, domain, userstatus, old_ustat, suspendedaccount, old_suspend, utype = user # Suspend Staff suspend = True try: if (userstatus == 2 or suspendedaccount == 1) and utype == 'Staff': appclient.suspend_user(username, utype) log.info('Suspending %s UID: %s %s' %(utype, uid, username)) # Suspend Students elif (userstatus == 0 and suspendedaccount == 1) and utype == 'Student': appclient.suspend_user(username, utype) log.info('Suspending %s UID: %s %s' %(utype, uid, username)) # Need to add a suspend account for students who are exited. May look at date too? # Restore Any Un-Suspended Account elif userstatus == 0 and suspendedaccount == 0: appclient.restore_user(username, utype) log.info('Restoring %s UID: %s %s' %(utype, uid, username)) else: suspend = False log.debug('Suspended %s Uesr %s %s not Suspended' %(utype, uid, username) ) # Add check user leave date. Mark account for deletion. # THIS UPDATES THE RECORD if they should be suspended if suspend: bixbycursor.execute(queries.update_suspended_user, (userstatus, suspendedaccount, uid)) else: log.debug('Suspended User Not Suspended') except gdata.apps.service.AppsForYourDomainException, e: if e.error_code == 1301: log.warn('%s User UID: %s %s Account Does Not Exist! MARKED DELETED' %(utype, uid, username)) # This updates the record bixbycursor.execute(queries.update_suspended_user, (userstatus, suspendedaccount, uid)) # This sets the account as deleted ACCOUNT_STATUS = 5 Account does not exist/deleted bixbycursor.execute(queries.update_account_status, (5, uid))
def _add_to_known_hosts(self, node): log.info("Configuring local known_hosts file") user_home = os.path.expanduser('~') khosts = os.path.join(user_home, '.ssh', 'known_hosts') if not os.path.isfile(khosts): log.warn("Unable to configure known_hosts: file does not exist") return contents = open(khosts).read() if node.dns_name not in contents: server_pkey = node.ssh.get_server_public_key() khostsf = open(khosts, 'a') if contents[-1] != '\n': khostsf.write('\n') name_entry = '%s,%s' % (node.dns_name, node.ip_address) khostsf.write(' '.join([name_entry, server_pkey.get_name(), base64.b64encode(str(server_pkey)), '\n'])) khostsf.close()
def sum_feature_sets(name, set_names): global FEATURE_SETS if name in FEATURE_SETS and FEATURE_SETS[name] is not None: log.warn("feature set '{}' exist and is not empty!".format(name)) return None if type(set_names) == type(''): set_names = set_names.split() result = [] for set_name in set_names: if set_name not in FEATURE_SETS: log.error("feature set '{}' does not exist!".format(set_name)) else: result += FEATURE_SETS[set_name] FEATURE_SETS[name] = list(set(result)) return len(FEATURE_SETS[name])
def tonedcharactersfromreading(characters, reading_tokens): # If we can't associate characters with tokens on a one-to-one basis we had better give up if len(characters) != len(reading_tokens): log.warn("Couldn't produce toned characters for %s because there are a different number of reading tokens than characters", characters) return [Text(characters)] # Add characters to the tokens /without/ spaces between them, but with tone info tokens = [] for character, reading_token in zip(characters, reading_tokens): # Avoid making the numbers from the supplementary dictionary into toned # things, because it confuses users :-) if hasattr(reading_token, "toneinfo") and not(character.isdecimal()): tokens.append(TonedCharacter(character, reading_token.toneinfo)) else: # Sometimes the tokens do not have tones (e.g. in the translation for T-shirt) tokens.append(Text(character)) return tokens
def _start_notebook(self, master, user, profile_dir): log.info("Setting up IPython web notebook for user: %s" % user) user_cert = posixpath.join(profile_dir, '%s.pem' % user) ssl_cert = posixpath.join(profile_dir, '%s.pem' % user) if not master.ssh.isfile(user_cert): log.info("Creating SSL certificate for user %s" % user) ssl_subj = "/C=US/ST=SC/L=STAR/O=Dis/CN=%s" % master.dns_name master.ssh.execute( "openssl req -new -newkey rsa:4096 -days 365 " '-nodes -x509 -subj %s -keyout %s -out %s' % (ssl_subj, ssl_cert, ssl_cert)) else: log.info("Using existing SSL certificate...") f = master.ssh.remote_file('%s/ipython_notebook_config.py' % profile_dir) notebook_port = 8888 sha1py = 'from IPython.lib import passwd; print passwd("%s")' sha1cmd = "python -c '%s'" % sha1py sha1pass = master.ssh.execute(sha1cmd % self.notebook_passwd)[0] f.write('\n'.join([ "c = get_config()", "c.IPKernelApp.pylab = 'inline'", "c.NotebookApp.certfile = u'%s'" % ssl_cert, "c.NotebookApp.ip = '*'", "c.NotebookApp.open_browser = False", "c.NotebookApp.password = u'%s'" % sha1pass, "c.NotebookApp.port = %d" % notebook_port, ])) f.close() master.ssh.execute_async("ipython notebook") group = master.cluster_groups[0] world_cidr = '0.0.0.0/0' port_open = master.ec2.has_permission(group, 'tcp', notebook_port, notebook_port, world_cidr) if not port_open: log.info("Authorizing tcp port %s on %s" % (notebook_port, world_cidr)) group.authorize('tcp', notebook_port, notebook_port, world_cidr) log.info("IPython notebook URL: https://%s:%s" % (master.dns_name, notebook_port)) log.info("The notebook password is: %s" % self.notebook_passwd) log.warn("Please check your local firewall settings if you're having " "issues connecting to the IPython notebook", extra=dict(__textwrap__=True))
def shell(self, user=None, forward_x11=False, command=None): """ Attempts to launch an interactive shell by first trying the system's ssh client. If the system does not have the ssh command it falls back to a pure-python ssh shell. """ if self.update() != 'running': try: alias = self.alias except exception.BaseException: alias = None label = 'instance' if alias == "master": label = "master" elif alias: label = "node" instance_id = alias or self.id raise exception.InstanceNotRunning(instance_id, self.state, label=label) user = user or self.user if utils.has_required(['ssh']): log.debug("Using native OpenSSH client") sshopts = '-i %s' % self.key_location if forward_x11: sshopts += ' -Y' ssh_cmd = static.SSH_TEMPLATE % dict(opts=sshopts, user=user, host=self.dns_name) if command: command = "'source /etc/profile && %s'" % command ssh_cmd = ' '.join([ssh_cmd, command]) log.debug("ssh_cmd: %s" % ssh_cmd) return subprocess.call(ssh_cmd, shell=True) else: log.debug("Using Pure-Python SSH client") if forward_x11: log.warn("X11 Forwarding not available in Python SSH client") if command: orig_user = self.ssh.get_current_user() self.ssh.switch_user(user) self.ssh.execute(command, silent=False, source_profile=True) self.ssh.switch_user(orig_user) return self.ssh.get_last_status() self.ssh.interactive_shell(user=user)
def update_proper_names(bixbycursor, usertype=None, appclient=None): log.info("""######### Updating Propper Names ########""") if not usertype: usertype = 'NULL' # There has to be a better way BEH appclient = google.appsclient.AppsClient() bixbycursor.execute(queries.get_changed_propper_name, (usertype,)) changed_proper_names = bixbycursor.fetchall() for user in changed_proper_names: try: uid, username, old_lastname, new_lastname, old_firstname, new_firstname = user log.debug('Changing Propper Name for User: %s From: %s %s To: %s %s' %(uid, old_firstname, old_lastname, new_firstname, new_lastname)) appclient.modify_propper_name(username, new_firstname, new_lastname, usertype) bixbycursor.execute(queries.update_propper_name, (new_firstname, new_lastname, uid)) except gdata.apps.service.AppsForYourDomainException, e: if e.error_code == 1301: log.warn('User: %s Does Not Exist!' %username) # Mark the account deleted ACCOUNT_STATUS = 5 Account does not exist/deleted bixbycursor.execute(queries.update_account_status, (5, uid))
def load_setting_file(args): setting_file = args.model + '.settings' log.info("loading setting file: {}".format(setting_file)) if not os.path.exists(setting_file): log.warn("setting file {} does not exist".format(setting_file)) return args with open(setting_file) as file: for line in file: opt, val = line.strip().split('=') if opt == 'use' and not args.algorithm: log.debug("loading algorithm: {}".format(val)) args.algorithm = val elif opt == 'set' and not args.confusion_set: log.debug("loading confusion set: {}".format(val)) args.confusion_set = val elif opt == 'cnf' and not args.cnf_opts: log.debug("loading confusion extraction options: {}".format(val)) args.cnf_opts = val elif opt == 'mng' and not args.mng_opts: log.debug("loading confusion managing options: {}".format(val)) args.mng_opts = val elif opt == 'ext' and not args.ext_opts: log.debug("loading feature extraction options: {}".format(val)) args.ext_opts = val elif opt == 'vec' and not args.vec_opts: log.debug("loading vectorization options: {}".format(val)) args.vec_opts = val elif opt == 'cls' and not args.cls_opts: log.debug("loading classification options: {}".format(val)) args.cls_opts = val elif opt == 'evl' and not args.evl_opts: log.debug("loading evaluation options: {}".format(val)) args.evl_opts = val else: log.warn("unrecognized setting: {}={}".format(opt, val)) return args
def start_worker_thread(self, app, dev, cer_file, key_file, conf): kwargs = { 'develop': dev, 'app_key': app, 'cer_file': cer_file, 'key_file': key_file, 'server_info': self.server_info } # 检查配置里关于推送线程数的配置,启动相应数量的推送线程 thread_cnt = 1 log.debug('startting worker with config : %s' % conf) if conf: conf_prefix = 'config_dev:' if dev else 'config:' self.rds.hmset(conf_prefix + app, conf) try: thread_cnt = int(conf.get('worker', '1')) except: log.warn('invalid config of workers, just start one.') workers = [] for cnt in range(thread_cnt): if cnt == 0: params = kwargs worker = threading.Thread(target=self.push, kwargs=params) else: params = kwargs.copy() params.update({'channel': cnt}) worker = threading.Thread(target=self.push, kwargs=params) workers.append(worker) feedback_job = threading.Thread(target=self.feedback, kwargs=kwargs) enhance_job = threading.Thread(target=self.enhance, kwargs=kwargs) feedback_job.setDaemon(True) enhance_job.setDaemon(True) for w in workers: w.setDaemon(True) w.start() feedback_job.start() enhance_job.start()
def get_file_list(self, remote_dir, is_root=False): log.debug('now start get list') # if is_root: # files = self.get_roor_dir() # else: # files = self.sftp.listdir_attr(remote_dir) try: files = self.sftp.listdir_attr(remote_dir) except Exception: log.warn('now re connect') self.t = paramiko.Transport((self.host, self.port)) self.connect() self.sftp = paramiko.SFTPClient.from_transport(self.t) files = self.sftp.listdir_attr(remote_dir) for f in files: filename = remote_dir + '/' + f.filename log.debug('now search %s' % filename) if stat.S_ISDIR(f.st_mode): for i in self.get_file_list(filename): yield i else: yield filename
def create_new_group( self, group_email, group_name, group_description, group_type, group_status=1, email_permission="Owner", domain_id=1, department_id=None, unique_attribute=None, ): """department_id, group_email, group_name, group_description, group_status , domain_id, group_type, unique_attribute PERMISSION_OWNER = 'Owner' PERMISSION_MEMBER = 'Member' PERMISSION_DOMAIN = 'Domain' PERMISSION_ANYONE = 'Anyone' """ groups_fields = ( department_id, group_email, group_name, group_description, group_status, domain_id, group_type, unique_attribute, ) try: self.group_create_group(group_email, group_name, group_description, email_permission=email_permission) except gdata.apps.service.AppsForYourDomainException, e: if e.error_code == 1300: log.exception(e) log.warn("""The Group: %s Exists. It's OK that happens.""" % group_email) else: raise e