def exit(self): m = Buffer() m.put_int(MUX_C_TERMINATE) rid = self.rid self.rid += 1 m.put_int(rid) if self._write_packet(m) is not None: return (False, 'Can\'t send TERMINATE request') # ignore reply # m = self._read_packet() # # rep_msg = m.get_int() # # rep_rid = m.get_int() # if rep_rid != rid: # return (False, 'Got unexpected request id %u, expected %u' % (rep_rid, rid)) # # if rep_msg != MUX_S_OK: # rep_reason = m.get_str() # if rep_msg == MUX_S_FAILURE: # return (False, 'Failure in EXIT message: %s' % (rep_reason,)) # elif rep_msg == MUX_S_PERMISSION_DENIED: # return (False, 'Permission denied for EXIT message: %s' % (rep_reason,)) # return (False, 'Unexpected server reply, got %u' % (rep_msg,)) return (True, None)
def __init__(self,path): self.filePath = path print "Flight initialisation" self.reader = csv.reader(open(self.filePath, 'rb')) self.writer = csv.writer(open('SimulationOutput.csv', 'wb')) flight_initialized = True self.buffer = Buffer()
def analyzation(self): buff = Buffer() stat = State() dict_state = dict_of_states() self.curr_p = -1 self.curr_symbs[1] = self.text[0] while stat.get_state()!=3: self.get_next() print(self.curr_p,' ', self.curr_symbs[0], self.define_symb(self.curr_symbs[0]), stat.get_state() ) if stat.get_state() == 1: next_state = dict_state.find_state( stat.get_state(), self.define_symb(self.curr_symbs[0]), self.define_symb(self.curr_symbs[0])) self.state_act(stat.get_state(),buff, next_state ) stat.set_state( next_state ) print('->' ,stat.get_state()) next_state = dict_state.find_state( stat.get_state(), self.define_symb(self.curr_symbs[0]), self.define_symb(self.curr_symbs[1])) print("BUFFER",buff.get_buff()) self.state_act(stat.get_state(),buff, next_state) stat.set_state( next_state ) print('->' ,stat.get_state()) self.state_act(stat.get_state(),buff, 3) for i in range(len(self.Lexems_arr)): print(self.Lexems_arr[i].get())
def __init__(self, symbols,period_start,period_end,mode,CASH_BIAS = 0,WINDOW_SIZE = 48,LR = 2e-5,MODEL_NAME = 'CNN',reward = 'avg_log_cum_return'): tf.compat.v1.disable_eager_execution() # permet l'utilisation complète de Tensorflow 1 self.symbols = symbols self.period_start = period_start self.period_end = period_end self.symbols_num = len(symbols) self.mode = mode self.nb_ep = 0 self.MODEL_NAME = MODEL_NAME #HYPER PARAMETERS self.BUFFER_SIZE = 200 self.BATCH_SIZE = 10 self.SHOW_EVERY = WINDOW_SIZE*7*4 #Affichage des résultats tous les MOIS (30jours) self.WINDOW_SIZE = WINDOW_SIZE # Une journée self.CASH_BIAS = CASH_BIAS self.NB_FEATURES = 9 self.SAMPLE_BIAS = 1.05 self.state_dim = (self.symbols_num,self.WINDOW_SIZE,self.NB_FEATURES) self.action_size = self.symbols_num +1 self.LR_list = {'train':2e-5,'test':9e-5,'valid':9e-5} self.ROLLING_STEPS_dic = {'train':1,'test':0,'valid':0} self.ROLLING_STEPS = self.ROLLING_STEPS_dic[mode] if LR in [2e-5,9e-5]: self.LR = self.LR_list[mode] else: self.LR = LR #Initialisation self.episode_reward = [] self.total_step = 0 self.session = self.tf_session() np.random.seed(4) self.agent = Agent(self.session,self.state_dim,self.action_size,self.BATCH_SIZE,self.LR,reward,MODEL_NAME) self.buffer = Buffer(self.BUFFER_SIZE, self.SAMPLE_BIAS)
def prepare_loaded_data(self, file_name): """ Przygotowuje i wczytuje dane z pliku :param file_name: plik z danymi :return: """ #utworzenie bufora buff = Buffer() data = CsvReader.read_from_file(file_name, 1) #zmiana czasu data = DataOperation.change_time_relative(data) #zmiana wysokosci na metry data = DataOperation.change_altitude_cm_m(data) #stworzenie zapisywacza saver = FileSaver("saved_data/dane.txt") #kazda linijke z pliku csv buforujemy osobno for d in data: buff.set_data(d) buffered_data = buff.get_data() #sprawdzamy czy kazda linijka jest poprawnie zapisana if InputValidator.input_val(buffered_data): #zapisujemy kazda linijke do nowego pliku saver.save_data(buffered_data) #odczyt danych z pliku csv i wizualizacja r_data = CsvReader.read_from_file(saver.get_file_name()) #tworzymy wizualizator, drugi parametr do interwal czasowy self.visualizer = PlotCreator(r_data, 1) self.data_loaded = True print "Dane zaladowane"
def runSimulation(self): """ :return: """ sensors = self.sensors sensorsList = SensorFactory.createSensorList(sensors, self.plane) header = encoder(sensorsList) buffer = Buffer(10, sensorsList) self.plane.takeoff() # start the simulation while self.plane.isFlying(): self.plane.update() buffer.readData() data = buffer.returnDataTr() n = 0 k = 0 for s in self.sensors.keys(): self.getChart(n).clear() self.getChart(n).set_title(s) for i in range(self.sensors[s]): self.getChart(n).plot(data[i+k]) k += self.sensors[s] n += 1 self.canvas.show() return self.plane.getName(), header, buffer.returnDataCopy()
def Test1(): env=simpy.Environment() B = Buffer() B.capacity=5 Source_Geo(env,"Source",0.5,B) Server_Geo(env,"Server",0.6,B) env.run(until=20)
def getANSIC(): import_file_path = filedialog.askopenfilename( title="Seleccionar Archivo", filetypes=((".c Files", "*.c"), )) Buffer.cargar_buffer(import_file_path) # Listas para cada lista devuelta lista de la función tokenizar token = [] lexema = [] fila = [] columna = [] total = [] # tokenizar y recargando el buffer for i in Buffer.cargar_buffer(import_file_path): t, lex, lin, col, tot = Analizador.tokenizar(i) token += t lexema += lex fila += lin columna += col total += [str(tot)] S = tk.Scrollbar(root) T = tk.Text(root, height=20, width=92) S.pack(side=tk.RIGHT, fill=tk.Y) T.pack(side=tk.LEFT, fill=tk.Y) S.config(command=T.yview) T.config(yscrollcommand=S.set) res = '\nTokens reconocidos:\n\n' + str( token) + '\n\n\nLISTA DE TOKENS DETALLADA: \n\n' + '\n'.join( [str(elem) for elem in total]) T.insert(tk.END, res)
class WeightBasedExpReplay(object): def __init__(self, maxSize, alpha=0.6, epsilon=0.000001): self.maxSize = maxSize self.buffer = Buffer(self.maxSize) self.sumTree = SumTree(self.maxSize) self.weights = {} self.alpha = 0.6 self.curSize = 0 self.epsilon = epsilon self.heap = Heap() def addExperience(self, experience): weight = self.heap.getMaxPriority() index = self.buffer.getPointer() self.buffer.insert(experience) prevWeight = 0 if index in self.weights: prevWeight = self.weights[index] diffWeight = weight - prevWeight self.weights[index] = weight self.sumTree.insert(diffWeight, index) self.heap.add(index, weight) self.curSize = min(self.curSize + 1, self.maxSize) def modifyExperience(self, weight, index): weight = weight + self.epsilon weight = weight**self.alpha prevWeight = 0 if index in self.weights: prevWeight = self.weights[index] diffWeight = weight - prevWeight self.weights[index] = weight self.sumTree.insert(diffWeight, index) self.heap.add(index, weight) def sample(self, samplesAmount): startPoints = np.linspace(0, self.sumTree.getAllSum(), samplesAmount + 1).tolist() expList = [] weightList = [] indexList = [] for a in range(len(startPoints) - 1): start = startPoints[a] end = startPoints[a + 1] sampledNum = np.random.uniform(start, end) retrIndex = self.sumTree.search(sampledNum) expList.append(self.buffer.getItem(retrIndex)) weightList.append(self.weights[retrIndex] / self.sumTree.getAllSum()) indexList.append(retrIndex) return np.asarray(expList), np.asarray(weightList), np.asarray( indexList) def getMaxPriority(self): if self.heap.size == 0: return sys.float_info.max return self.heap.p2w[1]
def __init__(self, maxSize, alpha=0.6, epsilon=0.000001): self.maxSize = maxSize self.buffer = Buffer(self.maxSize) self.sumTree = SumTree(self.maxSize) self.weights = {} self.alpha = 0.6 self.curSize = 0 self.epsilon = epsilon self.heap = Heap()
def __init__(self, papaiNoel, listaElfo, listaRena, tamanhoGrupoElfos): self.listaRena = listaRena self.listaElfo = listaElfo self.papaiNoel = papaiNoel self.bufferElfo = Buffer(tamanhoGrupoElfos) self.bufferRena = Buffer(len(self.listaRena)) self.contadorElfo = 0 self.contadorRena = 0 self.porta = Lock() self.anel = Lock()
def __init__(self, h, w): self.win = Window(top=0, x=w, y=h) self.buffer = Buffer() self.filename = "" self.cursor = Cursor() self.mode = "normal" self.exit = False self.handlers = Handler() self.command = "" self.message = ""
def __init__(self, env_dict, params): """ option_num, state_dim, action_dim, action_bound, gamma, learning_rate, replacement, buffer_capacity, epsilon gamma: (u_gamma, l_gamma) learning_rate: (lr_u_policy, lr_u_critic, lr_option, lr_termin, lr_l_critic) """ # session self.sess = tf.Session() # environment parameters self.sd = env_dict['state_dim'] self.ad = env_dict['action_dim'] a_bound = env_dict['action_scale'] assert a_bound.shape == (self.ad,), 'Action bound does not match action dimension!' # hyper parameters self.on = params['option_num'] epsilon = params['epsilon'] u_gamma = params['upper_gamma'] l_gamma = params['lower_gamma'] u_capac = params['upper_capacity'] l_capac = params['lower_capacity'] u_lrcri = params['upper_learning_rate_critic'] l_lrcri = params['lower_learning_rate_critic'] l_lrpol = params['lower_learning_rate_policy'] l_lrter = params['lower_learning_rate_termin'] # the frequency of training termination function if params['delay'] == 'inf': self.delay = -1 else: self.delay = params['delay'] # Upper critic and buffer self.u_critic = UCritic(session=self.sess, state_dim=self.sd, option_num=self.on, gamma=u_gamma, epsilon=epsilon, learning_rate=u_lrcri) self.u_buffer = Buffer(state_dim=self.sd, action_dim=1, capacity=u_capac) # Lower critic, options and buffer HER self.l_critic = LCritic(session=self.sess, state_dim=self.sd, action_dim=self.ad, gamma=l_gamma, learning_rate=l_lrcri) self.l_options = [Option(session=self.sess, state_dim=self.sd, action_dim=self.ad, ordinal=i, learning_rate=[l_lrpol, l_lrter]) for i in range(self.on)] self.l_buffers = [Buffer(state_dim=self.sd, action_dim=self.ad, capacity=l_capac) for i in range(self.on)] # Initialize all coefficients and saver self.sess.run(tf.global_variables_initializer()) self.saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=100) self.tc = 0 # counter for training termination self.mc = 0 # counter for model
def connect(self, blocking=1): self.fd.connect(self.path) self.fd.setblocking(blocking) m = Buffer() m.put_int(MUX_MSG_HELLO) m.put_int(SSHMUX_VER) if self._write_packet(m) is not None: return (False, 'Can\'t send HELLO message') m = self._read_packet() mux_msg = m.get_int() if mux_msg != MUX_MSG_HELLO: return (False, 'Expected HELLO reply, got %u' % (mux_msg,)) mux_ver = m.get_int() if mux_ver != SSHMUX_VER: return (False, 'Unsupported multiplexing protocol version %d (expected %d)' % (mux_ver, SSHMUX_VER)) extensions = {} while len(m): name = m.get_str() value = m.get_str() extensions[name] = value return (True, extensions)
def connect_online_disconnect_offline(my_config): server_thread = threading.Thread(target=run_simulation_server, args=[15]) server_thread.start() my_buffer = Buffer(my_config) ua_listener = OPCUAListener(my_config, my_buffer) time.sleep(3) ua_listener.connect() time.sleep(30) ua_listener.exit() time.sleep(15) assert my_buffer.len() >= 0
def __init__(self, Env_dim, Nb_action): self.memory = Buffer(Memory_size) self.eval_nn = Network(Env_dim, Nb_action) self.target_nn = Network(Env_dim, Nb_action) self.optimizer = torch.optim.Adam(self.eval_nn.parameters(), lr=Learning_rate) self.criterion = nn.MSELoss(reduction='sum') self.counter = 0 self.target_nn.fc1 = self.eval_nn.fc1 self.target_nn.fc2 = self.eval_nn.fc2 self.target_nn.out = self.eval_nn.out
def __init__(self, connection: socket): self.time_start = time.time() self.connection = connection self.buffer = Buffer(connection) self.step = 0 self.username = None self.stop_event = threading.Event() self.username_wrong = None self.start_time = time.time() t = threading.Timer(30, self.after_done) t.start() super().__init__()
def Buffers(self, j): o = flatbuffers.number_types.UOffsetTFlags.py_type( self._tab.Offset(12)) if o != 0: x = self._tab.Vector(o) x += flatbuffers.number_types.UOffsetTFlags.py_type(j) * 4 x = self._tab.Indirect(x) from Buffer import Buffer obj = Buffer() obj.Init(self._tab.Bytes, x) return obj return None
def __init__(self, config): self.myBuffer = Buffer(config) self.out_csv = '' self.in_io_count = 0 self.out_io_count = 0 self.csv_row = ['-', '-', 0, 0, 0, 0, 0, 'hadoop', 'bin/spark'] self.in_ran_io_count = 0 self.in_seq_size_count = 0 self.out_ran_io_count = 0 self.out_seq_size = 0 self.write_seq_threshold = config['write_seq_threshold'] self.print_input_config(config)
def __init__(self, number: int, gel: "Global Event List"): self.name = f"Host {number}" self.status = "idle" self.buffer = Buffer() self.channel = gel.channel self.GEL = gel self.arrivalRate = 0.8 # lambda self.senseTime = 0.01 # 0.01 ms self.DIFS = 0.1 # 0.10 ms self.SIFS = 0.05 # 0.05 ms self.notACKedDict = {} self.ackId = 0
def test_polling_interval(my_config): server_thread = threading.Thread(target=run_simulation_server, args=[20]) server_thread.start() my_buffer = Buffer(my_config) ua_listener = OPCUAListener(my_config, my_buffer) time.sleep(3) ua_listener.connect() time.sleep(5) ua_listener.exit() time.sleep(5) server_thread.join() print('Buffer length=', my_buffer.len()) assert 9 <= my_buffer.len() <= 11
def stop(self): m = Buffer() m.put_int(MUX_C_STOP_LISTENING) rid = self.rid self.rid += 1 m.put_int(rid) if self._write_packet(m) is not None: return (False, 'Can\'t send STOP-LISTENING request') m = self._read_packet() rep_msg = m.get_int() rep_rid = m.get_int() if rep_rid != rid: return (False, 'Got unexpected request id %u, expected %u' % (rep_rid, rid)) if rep_msg != MUX_S_OK: rep_reason = m.get_str() if rep_msg == MUX_S_FAILURE: return (False, 'Failure in STOP message: %s' % (rep_reason,)) elif rep_msg == MUX_S_PERMISSION_DENIED: return (False, 'Permission denied for STOP message: %s' % (rep_reason,)) return (False, 'Unexpected server reply, got %u' % (rep_msg,)) return (True, None)
def convertToCgrom2(self, str): buf = Buffer() buf.buffer = str for i in range(len(buf)): ch = ord(buf[i]) if ch == 0x5d: buf[i] = chr(252) elif ch == 0x5b: buf[i] = chr(250) elif ch == 0x24: buf[i] = chr(162) elif ch == 0x40: buf[i] = chr(160) elif ch == 0x5c: buf[i] = chr(251) elif ch == 0x7b: buf[i] = chr(253) elif ch == 0x7d: buf[i] = chr(255) elif ch == 0x7c: buf[i] = chr(254) elif ch == 0x27 or ch == 0x60 or ch == 0xB4: buf[i] = chr(39) elif ch == 0xe8: buf[i] = chr(164) elif ch == 0xe9: buf[i] = chr(165) elif ch == 0xc8: buf[i] = chr(197) elif ch == 0xc9: buf[i] = chr(207) elif ch == 0xe4: buf[i] = chr(123) elif ch == 0xc4: buf[i] = chr(91) elif ch == 0xf6: buf[i] = chr(124) elif ch == 0xd6: buf[i] = chr(92) elif ch == 0xfc: buf[i] = chr(126) elif ch == 0xdc: buf[i] = chr(94) elif ch == 0x5e: buf[i] = chr(253) elif ch == 0x5f: buf[i] = chr(254) return buf.buffer
class StreamSocketDispatcher (Dispatcher): def __init__(self, sock_family, sock_type, remote_address, fd=None): Dispatcher.__init__(self, fd) self.sock_family = sock_family self.sock_type = sock_type self.remote_address = remote_address self.read_buffer = Buffer(4096) self.write_buffer = Buffer(4096) self.uid = None self.gids = set() # For AF_UNIX sockets find out the uid/gids of the caller. if self.fd is not None and self.sock_family == socket.AF_UNIX: self.uid = utils.get_peer_uid(fd) self.gids = utils.uid_gids.get(self.uid, []) # Connect if this is client side. if self.client: self.handle_connect() print("uid:", self.uid, self.gids) def readable(self): return not self.read_buffer.full() def writeable(self): return not self.write_buffer.empty() def handle_connect(self): Dispatcher.handle_connect(self) try: fd = socket.socket(self.sock_family, self.sock_type) fd.connect(self.remote_address) self.fd = fd except socket.error as e: print("ERROR could not open socket %s, %s" % (e, self), file=sys.stderr) def handle_read(self): received = self.read_buffer.recv_into(self.fd) if received == 0: print("read: close") self.handle_close() else: print("read:", self.read_buffer) def handle_write(self): sent = self.write_buffer.send_outoff(self.fd) if sent == 0: self.handle_close()
def __init__(self): configs = load_configs("Configs.json") value_configs = load_configs("Configs_ValueNN.json") policy_configs = load_configs("Configs_PolicyNN.json") #logging.basicConfig(format='%(asctime)s %(message)s', filename=configs["log_name"], level=logging.WARNING, #datefmt="%Y-%m-%d %H:%M:%S") #logging.info("Starting Optimizer.") self.width = configs["width"] self.height = configs["height"] self.buffer_size = configs["buffer_size"] self.filename = configs["data_filename"] self.experiment_name = "Deviations/" + configs["val_filename"] self.buffer = Buffer(self.buffer_size) self.env = BlockRelocation(self.height, self.width) #self.model = ValueNetwork(configs=value_configs) #self.policy_network = PolicyNetwork(configs=policy_configs) #self.combined_model = CombinedModel(configs=configs) #self.value_wrapper = EstimatorWrapper(self.model) #self.policy_wrapper = EstimatorWrapper(self.policy_network) self.value_net = ValueNetworkKeras(value_configs) self.policy_net = PolicyNetworkKeras(policy_configs) self.tree_searcher = TreeSearch( self.value_net, BlockRelocation(self.height, self.width), self.policy_net) self.tree_searcher.std_vals = load_obj(self.experiment_name) self.baseline_params = { "search_depth": 5, "epsilon": 0.1, "threshold": 0.01, "drop_percent": 0.25, "factor": 0.01 } self.current_search_params = { "search_depth": 5, "epsilon": 0.1, "threshold": 0.05, "drop_percent": 0.3, "factor": 0.05 } self.dfs_params_hq = {"stop_param": 4, "k": 12} self.dfs_params_fast = {"stop_param": 1, "k": 12}
class ExperienceReplay(object): def __init__(self, maxCapacity): self.maxCapacity = maxCapacity self.buffer = Buffer(self.maxCapacity) def pushItem(self, item): self.buffer.pushItem(item) def sample(self, batchSize): availableIndexes = self.getLength() sampledIndexes = sample(range(0, availableIndexes), batchSize) return [self.buffer.getItem(index) for index in sampledIndexes] def getLength(self): return self.buffer.getLength()
def test_continous_reading_negative_case(my_config): my_config['opcua']['number_of_reconnections'] = -1 my_config['metrics'][0]['interval'] = 500 server_thread = threading.Thread(target=run_simulation_server, args=[20]) server_thread.start() my_buffer = Buffer(my_config) ua_listener = OPCUAListener(my_config, my_buffer) time.sleep(3) ua_listener.connect() time.sleep(10) ua_listener.exit() server_thread.join() time.sleep(5) list_of_metrics = [metric['metric_id'] for metric in my_config['metrics']] metric_values = [[] for _ in range(len(list_of_metrics))] for buffer_entity in my_buffer.buffer: metric_idx = list_of_metrics.index( buffer_entity.data['node'].metric_id) metric_values[metric_idx].append( buffer_entity.data['data_variant'].Value.Value) increments_equal_one = True for metric_value_set in metric_values: for i in range(1, len(metric_value_set), 1): if metric_value_set[i] != metric_value_set[i - 1] + 1: increments_equal_one = False break assert increments_equal_one == False
def __init__(self,maxSize, alpha=0.6): self.maxSize = maxSize self.buffer = Buffer(self.maxSize) self.heap = Heap() self.weights = None #Add two flags to indicate whether alpha or queue size has changed self.prevAlpha = alpha self.prevSize =0 # Variables to store current alpha and exp replay size self.alpha = alpha self.curSize = 0 #Weightings to each experience self.endPoints = []
def eachSegment(self): length = self.numberOfSegments() for i in range(0, length): samples = self.samplesForSegment(i) buf = Buffer(samples=samples, size=self.sizeForWindow, sampleRate=self.buf.sampleRate) yield (buf, i)
def __init__(self, argv=None): if notSupported: raise notSupported if argv is not None: self._argv = argv if self._argv is _marker: self._get_login_shell() (pid, fd) = pty.fork() if pid == 0: self._spawn() self._ptyfd = fd self._ptypid = pid self._new_text = None fcntl.fcntl(self._ptyfd, fcntl.F_SETFL, os.O_NDELAY) self._termProcess = True Buffer.__init__(self, name=self._argv[0], text='')
def my_buffer(): cfg_file = '../src_test/config_test.yaml' with open(cfg_file) as config_file: cfg = yaml.safe_load(config_file) test_buffer = Buffer(cfg) for i in range(test_buffer.max_buffer_size - 1): test_buffer.buffer.append(random() * 1000) return test_buffer
def __add_buffer(self, log_item): buf = Buffer(log_item, self.handler, maxsize=self.buffer_size) if log_item.channel not in self.buffers: self.buffers[log_item.channel] = {log_item.level: buf} else: if log_item.level not in self.buffers[log_item.channel]: self.buffers[log_item.channel][log_item.level] = buf return buf
class ExperienceReplay(object): def __init__(self,maxSize): self.maxSize = maxSize self.buffer = Buffer(self.maxSize) self.curSize = 0 def addExperience(self, *experience): self.buffer.insert(Transition(*experience)) self.curSize = min(self.curSize+1,self.maxSize) def sample(self, samplesAmount): sampledPoints = np.random.choice(self.curSize, samplesAmount, replace=False).tolist() expList = [] for a in sampledPoints : expList.append(self.buffer.getItem(a)) return expList
class Agent(): def __init__(self, Env_dim, Nb_action): self.memory = Buffer(Memory_size) self.eval_nn = Network(Env_dim, Nb_action) self.target_nn = Network(Env_dim, Nb_action) self.optimizer = torch.optim.Adam(self.eval_nn.parameters(), lr=Learning_rate) self.criterion = nn.MSELoss(reduction='sum') self.counter = 0 self.target_nn.fc1 = self.eval_nn.fc1 self.target_nn.fc2 = self.eval_nn.fc2 self.target_nn.out = self.eval_nn.out def choose_action(self, s): s = torch.unsqueeze(torch.FloatTensor(s), 0) return self.eval_nn(s)[0].detach() # ae(s) def getSample(self): return self.memory.sample(Batch_size) def optimize_model(self, file): if self.memory.get_nb_elements() >= Batch_size: batch = self.memory.sample(Batch_size) for s, a, s_, r, done in batch: qValues = (self.eval_nn(torch.tensor(s).float()))[a] qValues_ = self.target_nn(torch.tensor(s_).float()) qValues_target = Gamma * torch.max(qValues_) JO = pow(qValues - (r + (qValues_target * (1 - done))), 2) loss = self.criterion(qValues, JO) self.optimizer.zero_grad() # if i != Batch_size - 1: # loss.backward(retain_graph=True) # else: # loss.backward() loss.backward() self.optimizer.step() self.counter += 1 if self.counter % Refresh_gap == 0: torch.save(self.eval_nn, file) self.target_nn.fc1 = self.eval_nn.fc1 self.target_nn.fc2 = self.eval_nn.fc2 self.target_nn.out = self.eval_nn.out def store_transition(self, value): self.memory.insert(value)
class ExperienceReplay(object): def __init__(self,maxSize, alpha=0.6): self.maxSize = maxSize self.buffer = Buffer(self.maxSize) self.curSize = 0 def addExperience(self, experience): self.buffer.insert(experience) self.curSize = min(self.curSize+1,self.maxSize) def sample(self, samplesAmount): sampledPoints = np.random.choice(self.curSize, samplesAmount, replace=False).tolist() expList = [] weightList = [] for a in sampledPoints : expList.append(self.buffer.getItem(a)) weightList.append(1.0/samplesAmount) return np.asarray(expList), weightList, None
def run(cmd, quiet = False, abandon_output = True): proc = Popen(split(cmd), stdout = PIPE, stderr = STDOUT) buf = Buffer(abandon_output = abandon_output) line = proc.stdout.readline() while len(line): buf.put(line) if not quiet: print(line, end = '') line = proc.stdout.readline() # Process could probably close the descriptor before exiting. proc.wait() if proc.returncode != 0: raise Exception('Process exited with a non-zero return code. ' + 'Last output of the program:\n\n' + '---------- Start of exception log --\n' + buf.get_short().strip() + '\n---------- End of exception log --\n') return buf.get_long()
def analysis(self): Buff = Buffer() Stat = State() #Buff.add_buff(self.Current) #print(Buff.get_buff()) #print(self.Current) self.Current = self.get_next() while Stat.get_state() != 'final': if Stat.get_state() == 'start': if self.Current in self.Separators: self.Current = self.get_next() elif self.Current.isalpha(): Buff.clear_buff() Buff.add_buff(self.Current) Stat.set_state('ident') self.Current = self.get_next() #print(Buff.get_buff()) elif self.Current.isdigit(): Buff.clear_buff() Buff.add_buff(self.Current) Stat.set_state('numb') self.Current = self.get_next() elif self.Current == '{': #ЗАПИСАТЬ КОММЕНТАРИЙ В БУФЕР Stat.set_state('comment') self.Current = self.get_next() elif self.Current == '.': Lexems_arr.append( Lexer.Lexer(self.get_pos(), 'Разделитель', self.Current, 'имя')) Stat.set_state('final') elif self.Current in self.Delimiters: Stat.set_state('delimit') self.Current = self.get_next()
def __init__(self,h,w): self.win = Window(top=0,x=w,y=h) self.buffer = Buffer() self.filename = "" self.cursor = Cursor() self.mode = "normal" self.exit = False self.handlers = Handler() self.command = "" self.message = ""
def __init__(self, sock_family, sock_type, remote_address, fd=None): Dispatcher.__init__(self, fd) self.sock_family = sock_family self.sock_type = sock_type self.remote_address = remote_address self.read_buffer = Buffer(4096) self.write_buffer = Buffer(4096) self.uid = None self.gids = set() # For AF_UNIX sockets find out the uid/gids of the caller. if self.fd is not None and self.sock_family == socket.AF_UNIX: self.uid = utils.get_peer_uid(fd) self.gids = utils.uid_gids.get(self.uid, []) # Connect if this is client side. if self.client: self.handle_connect() print("uid:", self.uid, self.gids)
def __init__(self,config): self.myBuffer = Buffer(config) self.out_csv = '' self.in_io_count = 0 self.out_io_count = 0 self.csv_row = ['-','-',0,0,0,0,0,'hadoop','bin/spark'] self.in_ran_io_count = 0 self.in_seq_size_count = 0 self.out_ran_io_count = 0 self.out_seq_size= 0 self.write_seq_threshold = config['write_seq_threshold'] self.print_input_config(config)
def connect(self): if (self.csv_loaded == False): self.plotWidget.clear() line = 1 rfile = ReadCSV.read("test.csv") for row in rfile: if(line == 1): line = line+1 continue self.upload_data.append(row) Fixer.fixTimeAndAlt(self.upload_data) buff = Buffer() if os.path.exists('data.txt'): os.remove('data.txt') for i in self.upload_data: buff.sendToBuffer(i) buff.sendData() self.data = (DataLoader.read('data.txt')) self.csv_loaded = True self.infocsv.setText(_fromUtf8("Zaladowano")) self.info.setText("") self.wys = self.kat = self.dyst = self.pred = self.odchyl = 0
def __init__(self, lexer, error_handler=None): self.input = Buffer(lexer.tokenize()) self.error_handler = error_handler self.is_eof = lambda t: isinstance(t, EOFToken) self.is_def = lambda t: isinstance(t, DefToken) self.is_extern = lambda t: isinstance(t, ExternToken) self.is_if = lambda t: isinstance(t, IfToken) self.is_then = lambda t: isinstance(t, ThenToken) self.is_else = lambda t: isinstance(t, ElseToken) self.is_for = lambda t: isinstance(t, ForToken) self.is_in = lambda t: isinstance(t, InToken) self.is_var = lambda t: isinstance(t, VarToken) self.is_identifer = lambda t: isinstance(t, IdentifierToken) self.is_number = lambda t: isinstance(t, NumberToken) self.is_character = lambda t: isinstance(t, CharacterToken) self.is_binary = lambda t: isinstance(t, BinaryToken) self.is_unary = lambda t: isinstance(t, UnaryToken) self.is_binop = lambda t: OperatorManager.is_binop(t) self.is_unop = lambda t: OperatorManager.is_unop(t)
def check(self): m = Buffer() m.put_int(MUX_C_ALIVE_CHECK) rid = self.rid self.rid += 1 m.put_int(rid) if self._write_packet(m) is not None: return (False, 'Can\'t send ALIVE-CHECK request') m = self._read_packet() rep_msg = m.get_int() if rep_msg != MUX_S_ALIVE: return (False, 'Expected ALIVE reply, got %u' % (rep_msg,)) rep_rid = m.get_int() if rep_rid != rid: return (False, 'Got unexpected request id %u, expected %u' % (rep_rid, rid)) rep_pid = m.get_int() return (True, rep_pid)
def __init__(self): self.program = Program('Shader/Quad.prog') # Prepare quad geometry self.vbuffer = Buffer(gl.GL_ARRAY_BUFFER, gl.GL_STATIC_DRAW, Vertex) self.vbuffer.push(Vertex(-.5, -.5, 0, 0, 1)) self.vbuffer.push(Vertex(-.5, .5, 0, 0, 0)) self.vbuffer.push(Vertex( .5, -.5, 0, 1, 1)) self.vbuffer.push(Vertex( .5, .5, 0, 1, 0)) # Set up vertex array object for mapping structs => shader inputs vao = (ctypes.c_int * 1)() gl.glGenVertexArrays(1, vao) self.vao = vao[0] stride = ctypes.sizeof(Vertex) gl.glBindVertexArray(self.vao) gl.glBindBuffer(gl.GL_ARRAY_BUFFER, self.vbuffer.id) gl.glEnableVertexAttribArray(Attr.POSITION) offset = ctypes.c_void_p(0) gl.glVertexAttribPointer(Attr.POSITION, 3, gl.GL_FLOAT, gl.GL_FALSE, stride, offset) gl.glEnableVertexAttribArray(Attr.TEXCOORD) offset = ctypes.c_void_p(12) gl.glVertexAttribPointer(Attr.TEXCOORD, 2, gl.GL_FLOAT, gl.GL_FALSE, stride, offset) gl.glBindVertexArray(0)
def _write_packet(self, buf): pkt = Buffer() pkt.put_str(buf) return self.fd.sendall(pkt)
class MinerProtocol(Protocol): """ implementation of the Minecraft protocol """ def __init__(self, name): global debug self.log = debug self.bot = MinerBot(name, self.send) self.buffer = Buffer() self.world = World() self.packets = { # ID: (function, format 0x00: (self.onKeepAlive, Standard("Bi")), # Keep alive 0x01: (self.onLoginRequest, String("BiSqibbBB")), # Login request 0x02: (self.onHandshake, String("BS")), # Handshake 0x03: (self.onChat, String("BS")), # Chat message 0x04: (None, Standard("Bq")), # Time update 0x05: (None, Standard("Bihhh")), # Entity Equipment 0x06: (self.onSpawnPosition, Standard("Biii")), # Spawn position 0x07: (None, Standard("Bii?")), # Use entity (C --> S) 0x08: (self.onUpdateHealth, Standard("Bhhf")), # Update health 0x09: (None, Standard("Bbbbhl")), # Respawn (C --> S) 0x0A: (None, Standard("B?")), # Player (C --> S) 0x0B: (None, Standard("Bdddd?")), # Player position (C --> S) 0x0C: (None, Standard("Bff?")), # Player look (C --> S) 0x0D: (self.onPLayerPosition, Standard("Bddddff?")), # Player position & look 0x0E: (None, Standard("Bbibib")), # Player digging (C --> S) 0x0F: (None, BlockSlot("BibibW")), # Player block placement (C --> S) 0x10: (None, Standard("Bh")), # Holding change (C --> S) 0x11: (None, Standard("Bibibi")), # Use bed 0x12: (None, Standard("Bib")), # Animation 0x13: (None, Standard("Bib")), # Entity action (C --> S) 0x14: (self.onNamedEntitySpawn, String("BiSiiibbh")), # Named entity spawn 0x15: (None, Standard("Bihbhiiibbb")), # Pickup spawn 0x16: (None, Standard("Bii")), # Collect item 0x17: (self.onAddObject, Standard("Bibiiiihhh")), # Add object/vehicle 0x18: (None, MetaEntity("BibiiibbE")), # Mob spawn 0x19: (None, String("BiSiiii")), # Entity: painting 0x1A: (None, Standard("Biiiih")), # Experience Orb 0x1B: (None, Standard("Bffff??")), # Stance update 0x1C: (None, Standard("Bihhh")), # Entity velocity 0x1D: (None, Standard("Bi")), # Destroy entity 0x1E: (None, Standard("Bi")), # Entity 0x1F: (self.onEntityRelativeMove, Standard("Bibbb")), # Entity relative move 0x20: (None, Standard("Bibb")), # Entity look 0x21: (None, Standard("Bibbbbb")), # Entity look and relative move 0x22: (None, Standard("Biiiibb")), # Entity teleport 0x26: (None, Standard("Bib")), # Entity status 0x27: (None, Standard("Bii")), # Attach entity 0x28: (None, MetaEntity("BiE")), # Entity metadata 0x29: (None, Standard("Bibbh")), # Entity Effect 0x2A: (None, Standard("Bib")), # Remove Entity Effect 0x2B: (None, Standard("Bfhh")), # Experience 0x32: (self.onPreChunk, Standard("Bii?")), # Pre chunk 0x33: (self.onChunk, MetaChunk()), # Map chunk 0x34: (None, BlockArray()), # Multi-block change 0x35: (None, Standard("Bibibb")), # Block change 0x36: (None, Standard("Bihibb")), # Block action 0x3C: (None, BlockExplosion()), # Explosion 0x3D: (None, Standard("Biibii")), # Sound effect 0x46: (None, Standard("Bbb")), # New/invalid state 0x47: (None, Standard("Bi?iii")), # Thunderbolt 0x64: (None, String("BbbSb")), # Open window 0x65: (None, Standard("Bb")), # Close window 0x66: (None, BlockSlot("Bbhbh?W")), # Window click 0x67: (None, BlockSlot("BbhW")), # Set slot 0x68: (self.onWindowItems, Window()), # Window items 0x69: (None, Standard("Bbhh")), # Update window property 0x6A: (self.onTransaction, Standard("Bbh?")), # Transaction 0x6B: (None, BlockSlot("BhW")), # Creative inventory action 0x6C: (None, Standard("Bbb")), # Enchant Item 0x82: (None, String("BihiSSSS")), # Update sign 0x83: (None, ByteArray()), # item data 0xC8: (None, Standard("Bib")), # Increment statistic 0xC9: (self.onPlayerList, String("BS?h")), # Player List Item 0xFF: (self.onDisconnected, String("BS")), # Disconnect } def connectionMade(self): """ called when a connection has been established """ self.log.info("<%s> connected" % self.bot.getName()) # self.writer = PacketWriter(self.transport) self.log.debug("ATTEMPTING TO HANDSHAKE") self.sendHandshake(self.bot.getName()) def connectionLost(self, reason): self.log.error("connection lost %s" % reason) def dataReceived(self, data): """ new data are received """ self.buffer.append(data) while True: # get the packet Type try: data = self.buffer.peek()[0] packetType = ord(data) except IOError: # if empty buffer stop the loop break self.log.received(packetType) # get the packet information if packetType not in self.packets: self.log.error("Unknown packet 0x%02X" % packetType) self.transport.loseConnection() fct, fmt = self.packets[packetType] try: packet = fmt.unpack(self.buffer) except IOError: break # do action on packet received if fct!= None: fct(self.bot, packet) def onDisconnected(self, bot, info): self.log.info("Disconnected by server. Reason=%s" % info[1]) def onKeepAlive(self, bot, info): self.send(0x00, 0) def onUpdateHealth(self, bot, info): bot.setHealth(health=info[1], food=info[2], foodSaturation=info[3]) def onLoginRequest(self, bot, info): bot.setUID(info[1]) #bot.setMaxSeed(info[3]) pass def onHandshake(self, bot, info): #bot.setConnectionHash(info[1]) self.sendLoginRequest(22, self.bot.getName()) self.bot.nameToUpper() def onSpawnPosition(self, bot, info): bot.setSpawnPosition(info) def onPLayerPosition(self, bot, info): cmd, x, stance, y, z, yaw, pitch, onGround = info bot.setPlayerPosition(x, y, z, stance, yaw, pitch, onGround) self.send(0x0D, x, y, stance, z, yaw, pitch, onGround) def onEntityRelativeMove(self, bot, info): bot.setEntityRelativeMove(info) def onWindowItems(self, bot, info): pass # if info[1] == 0: # # inventory window # bot.setInventory(info[2], info[3]) # else: # logging.error("Unknown window number=%d" % info[1]) # def onSetSlot(self, bot, info): # if info[1] == 0: # bot.setSlotItem(info[1], info[2], info[3], info[4], info[5]) # logging.debug("SetSlot %d, slot %d" %(info[1], info[2])) def onNamedEntitySpawn(self, bot, info): bot.setPlayerSpawn(info) def onAddObject(self): pass def onPreChunk(self, bot, info): pass def onChunk(self, bot, info): cmd, x, y, z, sx, sy, sz, compSize, chunkData = info # correct the real size sx += 1 sy += 1 sz += 1 self.log.chunk("Chunk (x,y,z)=(%d, %d, %d) (sx,sy,sz)=(%d, %d, %d), len=%d" % (x, y, z, sx, sy, sz, len(chunkData))) self.world.addChunk(x, y, z, sx, sy, sz, chunkData) def onTransaction(self, bot, info): pass def send(self, cmd, *data): if cmd not in self.packets: self.log.error("Unknown packet to send 0x%02X" % cmd) self.transport.loseConnection() fct, pack = self.packets[cmd] res = pack.pack(cmd, *data) deb = "" for x in res: deb += "%02X " % ord(x) self.transport.write(res) self.log.send(cmd) def sendDisconnect(self, reason): self.send(0xFF, "Disconnection by client") self.transport.loseConnection() def sendHandshake(self, userName): self.send(0x02, userName) def sendLoginRequest(self, protocol_version, username): self.send(0x01, protocol_version, username, 0, 0, 0, 0, 0, 0) def onChat(self, bot, info): ''' Chat received. Analyse it to detect a message from the player ''' info = info[1].upper() if ord(info[0]) == 0xA7: #skip color code info = info[2:] # if the message is from a player it starts with <PlayerName> result = re.match("<(.*)>(.*)", info) if result == None: name = "" msg = info else: name = result.group(1) msg = result.group(2) result = msg.split() if BOT_NAME_NEEDED: # Command starts with the targeted Bot name if result[0] == bot.getName(): bot.parseCommand(name, result[1:]) else: if name <> bot.getName(): # do not compute self message bot.parseCommand(name, result) def onPlayerList(self, bot, info): bot.updatePlayerList(info[1], info[2])
def __init__(self, name): global debug self.log = debug self.bot = MinerBot(name, self.send) self.buffer = Buffer() self.world = World() self.packets = { # ID: (function, format 0x00: (self.onKeepAlive, Standard("Bi")), # Keep alive 0x01: (self.onLoginRequest, String("BiSqibbBB")), # Login request 0x02: (self.onHandshake, String("BS")), # Handshake 0x03: (self.onChat, String("BS")), # Chat message 0x04: (None, Standard("Bq")), # Time update 0x05: (None, Standard("Bihhh")), # Entity Equipment 0x06: (self.onSpawnPosition, Standard("Biii")), # Spawn position 0x07: (None, Standard("Bii?")), # Use entity (C --> S) 0x08: (self.onUpdateHealth, Standard("Bhhf")), # Update health 0x09: (None, Standard("Bbbbhl")), # Respawn (C --> S) 0x0A: (None, Standard("B?")), # Player (C --> S) 0x0B: (None, Standard("Bdddd?")), # Player position (C --> S) 0x0C: (None, Standard("Bff?")), # Player look (C --> S) 0x0D: (self.onPLayerPosition, Standard("Bddddff?")), # Player position & look 0x0E: (None, Standard("Bbibib")), # Player digging (C --> S) 0x0F: (None, BlockSlot("BibibW")), # Player block placement (C --> S) 0x10: (None, Standard("Bh")), # Holding change (C --> S) 0x11: (None, Standard("Bibibi")), # Use bed 0x12: (None, Standard("Bib")), # Animation 0x13: (None, Standard("Bib")), # Entity action (C --> S) 0x14: (self.onNamedEntitySpawn, String("BiSiiibbh")), # Named entity spawn 0x15: (None, Standard("Bihbhiiibbb")), # Pickup spawn 0x16: (None, Standard("Bii")), # Collect item 0x17: (self.onAddObject, Standard("Bibiiiihhh")), # Add object/vehicle 0x18: (None, MetaEntity("BibiiibbE")), # Mob spawn 0x19: (None, String("BiSiiii")), # Entity: painting 0x1A: (None, Standard("Biiiih")), # Experience Orb 0x1B: (None, Standard("Bffff??")), # Stance update 0x1C: (None, Standard("Bihhh")), # Entity velocity 0x1D: (None, Standard("Bi")), # Destroy entity 0x1E: (None, Standard("Bi")), # Entity 0x1F: (self.onEntityRelativeMove, Standard("Bibbb")), # Entity relative move 0x20: (None, Standard("Bibb")), # Entity look 0x21: (None, Standard("Bibbbbb")), # Entity look and relative move 0x22: (None, Standard("Biiiibb")), # Entity teleport 0x26: (None, Standard("Bib")), # Entity status 0x27: (None, Standard("Bii")), # Attach entity 0x28: (None, MetaEntity("BiE")), # Entity metadata 0x29: (None, Standard("Bibbh")), # Entity Effect 0x2A: (None, Standard("Bib")), # Remove Entity Effect 0x2B: (None, Standard("Bfhh")), # Experience 0x32: (self.onPreChunk, Standard("Bii?")), # Pre chunk 0x33: (self.onChunk, MetaChunk()), # Map chunk 0x34: (None, BlockArray()), # Multi-block change 0x35: (None, Standard("Bibibb")), # Block change 0x36: (None, Standard("Bihibb")), # Block action 0x3C: (None, BlockExplosion()), # Explosion 0x3D: (None, Standard("Biibii")), # Sound effect 0x46: (None, Standard("Bbb")), # New/invalid state 0x47: (None, Standard("Bi?iii")), # Thunderbolt 0x64: (None, String("BbbSb")), # Open window 0x65: (None, Standard("Bb")), # Close window 0x66: (None, BlockSlot("Bbhbh?W")), # Window click 0x67: (None, BlockSlot("BbhW")), # Set slot 0x68: (self.onWindowItems, Window()), # Window items 0x69: (None, Standard("Bbhh")), # Update window property 0x6A: (self.onTransaction, Standard("Bbh?")), # Transaction 0x6B: (None, BlockSlot("BhW")), # Creative inventory action 0x6C: (None, Standard("Bbb")), # Enchant Item 0x82: (None, String("BihiSSSS")), # Update sign 0x83: (None, ByteArray()), # item data 0xC8: (None, Standard("Bib")), # Increment statistic 0xC9: (self.onPlayerList, String("BS?h")), # Player List Item 0xFF: (self.onDisconnected, String("BS")), # Disconnect }
class Parser(object): def __init__(self, lexer, error_handler=None): self.input = Buffer(lexer.tokenize()) self.error_handler = error_handler self.is_eof = lambda t: isinstance(t, EOFToken) self.is_def = lambda t: isinstance(t, DefToken) self.is_extern = lambda t: isinstance(t, ExternToken) self.is_if = lambda t: isinstance(t, IfToken) self.is_then = lambda t: isinstance(t, ThenToken) self.is_else = lambda t: isinstance(t, ElseToken) self.is_for = lambda t: isinstance(t, ForToken) self.is_in = lambda t: isinstance(t, InToken) self.is_var = lambda t: isinstance(t, VarToken) self.is_identifer = lambda t: isinstance(t, IdentifierToken) self.is_number = lambda t: isinstance(t, NumberToken) self.is_character = lambda t: isinstance(t, CharacterToken) self.is_binary = lambda t: isinstance(t, BinaryToken) self.is_unary = lambda t: isinstance(t, UnaryToken) self.is_binop = lambda t: OperatorManager.is_binop(t) self.is_unop = lambda t: OperatorManager.is_unop(t) def collect(self): return self.input.accept() def look(self, condition=None): return self.input.peek(condition) def expect(self, condition=None): return self.input.move(condition) def try_number_expr(self): """ number_expr ::= number """ number = self.expect(self.is_number) if number is None: return None return NumberExprNode(number) def try_paren_expr(self): """ paren_expr ::= '(' expr ')' """ left_paren = self.expect(lambda t: t.name == '(') if left_paren is None: return None expr = self.try_expr() right_paren = self.expect(lambda t: t.name == ')') if right_paren is None: raise ExpectedRightParen(left_paren.line if expr is None else expr.line) return expr def try_identifier_expr(self): """ identifier_expr ::= identifier ::= identifier '(' ')' ::= identifier '(' expr (',' expr)* ')' """ identifier = self.expect(self.is_identifer) if identifier is None: return None left_paren = self.expect(lambda t: t.name == '(') if left_paren is None: return VariableExprNode(identifier) else: args = [] while True: arg = self.try_expr() if arg is not None: args.append(arg) self.expect(lambda t: t.name == ',') else: right_paren = self.expect(lambda t: t.name == ')') if right_paren is None: raise ExpectedRightParen(left_paren.line if len(args) == 0 else args[-1].line) return CallExprNode(identifier, args) def try_if_expr(self): """ if_expr ::= if expr then expr else expr """ token = self.expect(self.is_if) if token is None: return None condition = self.try_expr() if condition is None: raise ExpectedExpr(token.line) token = self.expect(self.is_then) if token is None: raise ExpectedThen(condition.line) true = self.try_expr() if true is None: raise ExpectedExpr(token.line) token = self.expect(self.is_else) if token is None: raise ExpectedElse(true.line) false = self.try_expr() if false is None: raise ExpectedExpr(token.line) return IfExprNode(condition, true, false) def try_for_expr(self): """ for_expr ::= for identifier '=' expr ',' expr (',' expr)? in expr """ token = self.expect(self.is_for) if token is None: return None variable = self.expect(self.is_identifer) if variable is None: raise ExpectedIdentifier(token.line) token = self.expect(lambda t: t.name == '=') if token is None: raise ExpectedEqualSign(variable.line) begin = self.try_expr() if begin is None: raise ExpectedExpr(token.line) token = self.expect(lambda t: t.name == ',') if token is None: raise ExpectedComma(begin.line) end = self.try_expr() if end is None: raise ExpectedExpr(token.line) token = self.expect(lambda t: t.name == ',') if token is not None: step = self.try_expr() if step is None: raise ExpectedExpr(token.line) else: step = None token = self.expect(self.is_in) if token is None: raise ExpectedIn(end.line if step is None else step.line) body = self.try_expr() if body is None: raise ExpectedExpr(token.line) return ForExprNode(variable, begin, end, step, body) def try_var_expr(self): """ var_expr ::= var identifier ('=' expr)? (',' identifier ('=' expr)?)* in expr """ var_token = self.expect(self.is_var) if var_token is None: return None variables = {} while True: identifier = self.expect(self.is_identifer) if identifier is None: break assignment = self.expect(lambda t: t.name == '=') if assignment is not None: expr = self.try_expr() if expr is None: raise ExpectedExpr(assignment.line) variables[identifier] = expr else: variables[identifier] = None self.expect(lambda t: t.name == ',') if len(variables) == 0: raise ExpectedVariableList(var_token.line) in_token = self.expect(self.is_in) if in_token is None: raise ExpectedIn(max([var.line for var in variables.keys()])) body = self.try_expr() if body is None: raise ExpectedExpr(in_token.line) return VarExprNode(variables, body) def try_primary_expr(self): """ primary_expr ::= number_expr ::= paren_expr ::= identifier_expr ::= if_expr ::= for_expr ::= var_expr """ expr = self.try_number_expr() if expr is not None: return expr expr = self.try_paren_expr() if expr is not None: return expr expr = self.try_identifier_expr() if expr is not None: return expr expr = self.try_if_expr() if expr is not None: return expr expr = self.try_for_expr() if expr is not None: return expr return self.try_var_expr() def try_unary_expr(self): """ unary_expr ::= primary_expr ::= unop unary_expr """ unop = self.expect(self.is_unop) if unop is None: return self.try_primary_expr() operand = self.try_unary_expr() if operand is None: raise ExpectedOperand(unop.line) return UnaryExprNode(unop, operand) def try_binop_rhs(self, lhs, lhs_prec): """ binop_rhs ::= (binop unary_expr)* """ while True: binop = self.look(self.is_binop) if binop is None: return lhs prec = OperatorManager.get_binop_precedence(binop) if prec < lhs_prec: return lhs self.expect(self.is_binop) # eat binop rhs = self.try_unary_expr() if rhs is None: raise ExpectedUnaryExpr(binop.line) next_binop = self.look(self.is_binop) if next_binop is not None and \ OperatorManager.get_binop_precedence(next_binop) > prec: rhs = self.try_binop_rhs(rhs, prec + 1) lhs = BinaryExprNode(binop, lhs, rhs) def try_expr(self): """ expr ::= unary_expr binop_rhs """ lhs = self.try_unary_expr() if lhs is None: return None return self.try_binop_rhs(lhs, 0) def try_normal_prototype(self): """ normal_prototype ::= identifier '(' identifier* ')' """ identifier = self.expect(self.is_identifer) if identifier is None: return None left_paren = self.expect(lambda t: t.name == '(') if left_paren is None: raise ExpectedLeftParen(identifier.line) args = [] while True: arg = self.expect(self.is_identifer) if arg is not None: args.append(arg) else: right_paren = self.expect(lambda t: t.name == ')') if right_paren is None: raise ExpectedRightParen(left_paren.line if len(args) == 0 else args[-1].line) return PrototypeNode(identifier, args) def try_binary_prototype(self): """ binary_prototype ::= binary character number? '(' identifier identifier ')' """ binary = self.expect(self.is_binary) if binary is None: return None operator = self.expect(self.is_character) if operator is None: raise ExpectedOperator(binary.line) precedence = self.expect(self.is_number) left_paren = self.expect(lambda t: t.name == '(') if left_paren is None: raise ExpectedLeftParen(operator.line if precedence is None else precedence.line) arg1 = self.expect(self.is_identifer) if arg1 is None: raise ExpectedIdentifier(left_paren.line) arg2 = self.expect(self.is_identifer) if arg2 is None: raise ExpectedIdentifier(arg1.line) right_paren = self.expect(lambda t: t.name == ')') if right_paren is None: raise ExpectedRightParen(arg2.line) return BinOpPrototypeNode(IdentifierToken(binary.name + operator.name, binary.line), precedence, [arg1, arg2]) def try_unary_prototype(self): """ unary_prototype ::= unary character '(' identifier ')' """ unary = self.expect(self.is_unary) if unary is None: return None operator = self.expect(self.is_character) if operator is None: raise ExpectedOperator(unary.line) left_paren = self.expect(lambda t: t.name == '(') if left_paren is None: raise ExpectedLeftParen(operator.line) arg = self.expect(self.is_identifer) if arg is None: raise ExpectedIdentifier(left_paren.line) right_paren = self.expect(lambda t: t.name == ')') if right_paren is None: raise ExpectedRightParen(arg.line) return UnOpPrototypeNode(IdentifierToken(unary.name + operator.name, unary.line), [arg]) def try_prototype(self): """ prototype ::= normal_prototype ::= binary_prototype ::= unary_prototype """ prototype = self.try_normal_prototype() if prototype is not None: return prototype prototype = self.try_binary_prototype() if prototype is not None: return prototype prototype = self.try_unary_prototype() if prototype is not None: return prototype def try_function(self): """ function ::= def prototype expr """ keyword = self.expect(self.is_def) if keyword is None: return None prototype = self.try_prototype() if prototype is None: raise ExpectedPrototype(keyword.line) expr = self.try_expr() if expr is None: raise ExpectedExpr(prototype.line) return FunctionNode(prototype, expr) def try_declaration(self): """ declaration ::= extern prototype """ keyword = self.expect(self.is_extern) if keyword is None: return None prototype = self.try_prototype() if prototype is None: raise ExpectedPrototype(keyword.line) return prototype def try_toplevel_expr(self): """ toplevel_expr ::= expr we make an anonymous prototype to represent a top-level expr """ expr = self.try_expr() if expr is None: return None return TopLevelExpr(expr) def try_negligible_character(self): # ignore top-level semicolons return self.expect(lambda t: t.name == ';') def try_eof(self): return self.expect(self.is_eof) def try_unknown(self): # try to recovery from syntax errors by eating an unknown token token = self.expect() if token is not None: raise UnknownToken(token) def parse(self): while True: try: node = self.try_function() if node is not None: yield node continue node = self.try_declaration() if node is not None: yield node continue node = self.try_toplevel_expr() if node is not None: yield node continue if self.try_negligible_character() is not None: continue if self.try_eof() is not None: break self.try_unknown() except BoidaeSyntaxError as e: if self.error_handler is not None: self.error_handler(e)
def forward(self, mode, ftype, listen_host, listen_port, connect_host, connect_port): m = Buffer() if mode: n = 'OPEN' m.put_int(MUX_C_OPEN_FWD) else: n = 'CLOSE' m.put_int(MUX_C_CLOSE_FWD) rid = self.rid self.rid += 1 m.put_int(rid) if isinstance(ftype, basestring): assert ftype in _fwd_names, 'Invalid forward type %s' % (ftype,) ftype = _fwd_names[ftype] m.put_int(ftype) m.put_str(listen_host) m.put_int(listen_port) m.put_str(connect_host) m.put_int(connect_port) if self._write_packet(m) is not None: return (False, 'Can\'t send %s-FORWARD request' % (n,)) m = self._read_packet() rep_msg = m.get_int() rep_rid = m.get_int() if rep_rid != rid: return (False, 'Got unexpected request id %u, expected %u' % (rep_rid, rid)) if rep_msg != MUX_S_OK and rep_msg != MUX_S_REMOTE_PORT: rep_reason = m.get_str() if rep_msg == MUX_S_FAILURE: return (False, 'Failure in %s-FORWARD request: %s' % (n, rep_reason,)) elif rep_msg == MUX_S_PERMISSION_DENIED: return (False, 'Permission denied for %s-FORWARD request: %s' % (n, rep_reason,)) return (False, 'Unexpected server reply, got %u' % (rep_msg,)) if ftype == MUX_FWD_REMOTE and listen_port == 0: if rep_msg != MUX_S_REMOTE_PORT: return (False, 'Expected remote port reply, got %u' % (rep_msg,)) rep_port = m.get_int() return (True, rep_port) return (True, None)
def __init__(self,config): self.myBuffer = Buffer(config) self.out_csv = '' self.in_io_count = 0 self.out_io_count = 0 self.csv_row = ['-','-',0,0,0,0,0,'hadoop','bin/spark']
class Shaper(): def __init__(self,config): self.myBuffer = Buffer(config) self.out_csv = '' self.in_io_count = 0 self.out_io_count = 0 self.csv_row = ['-','-',0,0,0,0,0,'hadoop','bin/spark'] def read_csv(self,in_file): colnames = ['STIME','TIME','UID','PID','D','BLOCK','SIZE','COMM','PATHNAME'] data = pandas.read_csv(in_file,names = colnames) data_dir = data.D.tolist() data_addr = map(lambda x: x//8, data.BLOCK.tolist()) data_size = map(lambda x: x//4096, data.SIZE.tolist()) arrs = [] length = len(data_dir) self.start_time = time.time() for i in range(0,length): arrs.append([data_dir[i],data_addr[i],data_size[i]]) return arrs def run(self,in_file,out_file): arrs = self.read_csv(in_file) self.in_io_count = len(arrs) self.out_csv = open(out_file,'wb') for each in arrs: if each[0]=='W': #TODO: big file, dont put into cache, need to invalidate the data in cache if cache hit while not self.myBuffer.add([each[1],each[1]+each[2]]): ios = self.myBuffer.get_cold() self.gen_write_io(ios) else: #TODO: check cache hit, if hit, return data immediately instead of generate read io ios = [[each[1],each[2]]] self.gen_read_io(ios) #clear all data in Buffer ios = self.myBuffer.get_all(); self.gen_write_io(ios) arrs = self.read_csv(out_file) self.out_io_count = len(arrs) def gen_write_io(self,ios): for each in ios: self.write_csv(['W',each[0],each[1]]) def gen_read_io(self,ios): for each in ios: self.write_csv(['R',each[0],each[1]]) def write_csv(self,item): content_list = [] self.csv_row[4] = item[0] self.csv_row[5] = item[1]*8 self.csv_row[6] = item[2]*4096 content_list.append(self.csv_row) csv_writer = csv.writer(self.out_csv) csv_writer.writerows(content_list) def print_io_count(self): print "==========================================================" print "I/O count before optimization:", self.in_io_count print "I/O count after optimization:", self.out_io_count print "=========================================================="
def forwards(self): m = Buffer() m.put_int(MUX_C_LIST_FWDS) rid = self.rid self.rid += 1 m.put_int(rid) if self._write_packet(m) is not None: return (False, 'Can\'t send LIST-FORWARDS request') m = self._read_packet() rep_msg = m.get_int() rep_rid = m.get_int() if rep_rid != rid: return (False, 'Got unexpected request id %u, expected %u' % (rep_rid, rid)) if rep_msg != MUX_S_RESULT: rep_reason = m.get_str() if rep_msg == MUX_S_FAILURE: return (False, 'Failure in LIST-FORWARDS request: %s' % (rep_reason,)) elif rep_msg == MUX_S_PERMISSION_DENIED: return (False, 'Permission denied for LIST-FORWARDS request: %s' % (rep_reason,)) return (False, 'Unexpected server reply, got %u' % (rep_msg,)) fwds = [] while len(m): fid = m.get_int() ftype = m.get_int() listen_host = m.get_str() listen_port = m.get_int() connect_host = m.get_str() connect_port = m.get_int() if ftype == MUX_FWD_REMOTE and listen_port == 0: listen_port = m.get_int() fwds.append((fid, _fwd_types[ftype], listen_host, listen_port, connect_host, connect_port)) return (True, fwds)
def _read_packet(self): m = Buffer(self.fd.recv(4)) pktlen = m.get_int() return Buffer(self.fd.recv(pktlen))
def sessions(self): m = Buffer() m.put_int(MUX_C_LIST_SESSIONS) rid = self.rid self.rid += 1 m.put_int(rid) if self._write_packet(m) is not None: return (False, 'Can\'t send LIST-SESSIONS request') m = self._read_packet() rep_msg = m.get_int() rep_rid = m.get_int() if rep_rid != rid: return (False, 'Got unexpected request id %u, expected %u' % (rep_rid, rid)) if rep_msg != MUX_S_RESULT: rep_reason = m.get_str() if rep_msg == MUX_S_FAILURE: return (False, 'Failure in LIST-SESSIONS request: %s' % (rep_reason,)) elif rep_msg == MUX_S_PERMISSION_DENIED: return (False, 'Permission denied for LIST-SESSIONS request: %s' % (rep_reason,)) return (False, 'Unexpected server reply, got %u' % (rep_msg,)) sessions = [] while len(m): sid = m.get_int(); stype = m.get_int(); rid = m.get_int(); cid = m.get_int(); tname = m.get_str(); rname = m.get_str(); sessions.append((sid, stype, rid, cid, tname, rname)) return (True, sessions)
#!/usr/bin/env python from Context import Context from Buffer import Buffer from Shader import Shader from Program import Program from Texture import Texture from Mat4 import Mat4 import OpenGL.GL as gl import ctypes c = Context() b = Buffer(gl.GL_ARRAY_BUFFER, gl.GL_STATIC_DRAW, ctypes.c_int) b.sync() b.push(1) b.push(2) b.push(3) b.push(4) b.push(5) b[12] = 3 assert(b[0]==1) assert(b[1]==2) assert(b[2]==3) assert(b[3]==4) assert(b[4]==5) assert(b[5]==0) assert(b[13]==None)
def info(self, fmt): m = Buffer() m.put_int(MUX_C_INFO) rid = self.rid self.rid += 1 m.put_int(rid) m.put_str(fmt) if self._write_packet(m) is not None: return (False, 'Can\'t send INFO request') m = self._read_packet() rep_msg = m.get_int() rep_rid = m.get_int() if rep_rid != rid: return (False, 'Got unexpected request id %u, expected %u' % (rep_rid, rid)) if rep_msg != MUX_S_RESULT: rep_reason = m.get_str() if rep_msg == MUX_S_FAILURE: return (False, rep_reason) return (False, 'Expected INFO reply, got %x' % (rep_msg,)) rep_str = m.get_str() return (True, rep_str)
class Renderer(object): def __init__(self): self.program = Program('Shader/Quad.prog') # Prepare quad geometry self.vbuffer = Buffer(gl.GL_ARRAY_BUFFER, gl.GL_STATIC_DRAW, Vertex) self.vbuffer.push(Vertex(-.5, -.5, 0, 0, 1)) self.vbuffer.push(Vertex(-.5, .5, 0, 0, 0)) self.vbuffer.push(Vertex( .5, -.5, 0, 1, 1)) self.vbuffer.push(Vertex( .5, .5, 0, 1, 0)) # Set up vertex array object for mapping structs => shader inputs vao = (ctypes.c_int * 1)() gl.glGenVertexArrays(1, vao) self.vao = vao[0] stride = ctypes.sizeof(Vertex) gl.glBindVertexArray(self.vao) gl.glBindBuffer(gl.GL_ARRAY_BUFFER, self.vbuffer.id) gl.glEnableVertexAttribArray(Attr.POSITION) offset = ctypes.c_void_p(0) gl.glVertexAttribPointer(Attr.POSITION, 3, gl.GL_FLOAT, gl.GL_FALSE, stride, offset) gl.glEnableVertexAttribArray(Attr.TEXCOORD) offset = ctypes.c_void_p(12) gl.glVertexAttribPointer(Attr.TEXCOORD, 2, gl.GL_FLOAT, gl.GL_FALSE, stride, offset) gl.glBindVertexArray(0) def render(self, g): # Render all objects attached to the graphics context. for quad in g.quad: self.quad(g, quad) for text in g.text: self.text(g, text) def quad(self, g, quad): # Enable alpha blending/transparency self.vbuffer.sync() gl.glUseProgram(self.program.id) gl.glEnable(gl.GL_BLEND) gl.glEnable(gl.GL_DEPTH_TEST) gl.glBlendFunc(gl.GL_SRC_ALPHA, gl.GL_ONE_MINUS_SRC_ALPHA) # Bind texture gl.glUniform1i(self.program.tex, 0) gl.glBindTexture(gl.GL_TEXTURE_2D, quad.texture.id) # Set up geometry transforms worldMatrix = Matrix.scale(quad.width, quad.height, 1) worldMatrix = Matrix.translate(quad.x, quad.y, 0) * worldMatrix worldViewProjectionMatrix = g.viewProjectionMatrix * worldMatrix #worldViewProjectionMatrix = g.viewProjectionMatrix gl.glUniformMatrix4fv(self.program.worldViewProjectionMatrix, 1, 0, worldViewProjectionMatrix.data) # Draw geometry gl.glBindVertexArray(self.vao) gl.glDrawArrays(gl.GL_TRIANGLE_STRIP, 0, 4) gl.glBindVertexArray(0) def text(self, g, text): pass