def exit(self):
m = Buffer()
m.put_int(MUX_C_TERMINATE)
rid = self.rid
self.rid += 1
m.put_int(rid)
if self._write_packet(m) is not None:
return (False, 'Can\'t send TERMINATE request')
# ignore reply
# m = self._read_packet()
#
# rep_msg = m.get_int()
#
# rep_rid = m.get_int()
# if rep_rid != rid:
# return (False, 'Got unexpected request id %u, expected %u' % (rep_rid, rid))
#
# if rep_msg != MUX_S_OK:
# rep_reason = m.get_str()
# if rep_msg == MUX_S_FAILURE:
# return (False, 'Failure in EXIT message: %s' % (rep_reason,))
# elif rep_msg == MUX_S_PERMISSION_DENIED:
# return (False, 'Permission denied for EXIT message: %s' % (rep_reason,))
# return (False, 'Unexpected server reply, got %u' % (rep_msg,))
return (True, None)
def __init__(self,path):
self.filePath = path
print "Flight initialisation"
self.reader = csv.reader(open(self.filePath, 'rb'))
self.writer = csv.writer(open('SimulationOutput.csv', 'wb'))
flight_initialized = True
self.buffer = Buffer()
def analyzation(self):
buff = Buffer()
stat = State()
dict_state = dict_of_states()
self.curr_p = -1
self.curr_symbs[1] = self.text[0]
while stat.get_state()!=3:
self.get_next()
print(self.curr_p,' ', self.curr_symbs[0], self.define_symb(self.curr_symbs[0]), stat.get_state() )
if stat.get_state() == 1:
next_state = dict_state.find_state( stat.get_state(), self.define_symb(self.curr_symbs[0]), self.define_symb(self.curr_symbs[0]))
self.state_act(stat.get_state(),buff, next_state )
stat.set_state( next_state )
print('->' ,stat.get_state())
next_state = dict_state.find_state( stat.get_state(), self.define_symb(self.curr_symbs[0]), self.define_symb(self.curr_symbs[1]))
print("BUFFER",buff.get_buff())
self.state_act(stat.get_state(),buff, next_state)
stat.set_state( next_state )
print('->' ,stat.get_state())
self.state_act(stat.get_state(),buff, 3)
for i in range(len(self.Lexems_arr)):
print(self.Lexems_arr[i].get())
def __init__(self, symbols,period_start,period_end,mode,CASH_BIAS = 0,WINDOW_SIZE = 48,LR = 2e-5,MODEL_NAME = 'CNN',reward = 'avg_log_cum_return'):
tf.compat.v1.disable_eager_execution() # permet l'utilisation complète de Tensorflow 1
self.symbols = symbols
self.period_start = period_start
self.period_end = period_end
self.symbols_num = len(symbols)
self.mode = mode
self.nb_ep = 0
self.MODEL_NAME = MODEL_NAME
#HYPER PARAMETERS
self.BUFFER_SIZE = 200
self.BATCH_SIZE = 10
self.SHOW_EVERY = WINDOW_SIZE*7*4 #Affichage des résultats tous les MOIS (30jours)
self.WINDOW_SIZE = WINDOW_SIZE # Une journée
self.CASH_BIAS = CASH_BIAS
self.NB_FEATURES = 9
self.SAMPLE_BIAS = 1.05
self.state_dim = (self.symbols_num,self.WINDOW_SIZE,self.NB_FEATURES)
self.action_size = self.symbols_num +1
self.LR_list = {'train':2e-5,'test':9e-5,'valid':9e-5}
self.ROLLING_STEPS_dic = {'train':1,'test':0,'valid':0}
self.ROLLING_STEPS = self.ROLLING_STEPS_dic[mode]
if LR in [2e-5,9e-5]:
self.LR = self.LR_list[mode]
else:
self.LR = LR
#Initialisation
self.episode_reward = []
self.total_step = 0
self.session = self.tf_session()
np.random.seed(4)
self.agent = Agent(self.session,self.state_dim,self.action_size,self.BATCH_SIZE,self.LR,reward,MODEL_NAME)
self.buffer = Buffer(self.BUFFER_SIZE, self.SAMPLE_BIAS)
def prepare_loaded_data(self, file_name):
"""
Przygotowuje i wczytuje dane z pliku
:param file_name: plik z danymi
:return:
"""
#utworzenie bufora
buff = Buffer()
data = CsvReader.read_from_file(file_name, 1)
#zmiana czasu
data = DataOperation.change_time_relative(data)
#zmiana wysokosci na metry
data = DataOperation.change_altitude_cm_m(data)
#stworzenie zapisywacza
saver = FileSaver("saved_data/dane.txt")
#kazda linijke z pliku csv buforujemy osobno
for d in data:
buff.set_data(d)
buffered_data = buff.get_data()
#sprawdzamy czy kazda linijka jest poprawnie zapisana
if InputValidator.input_val(buffered_data):
#zapisujemy kazda linijke do nowego pliku
saver.save_data(buffered_data)
#odczyt danych z pliku csv i wizualizacja
r_data = CsvReader.read_from_file(saver.get_file_name())
#tworzymy wizualizator, drugi parametr do interwal czasowy
self.visualizer = PlotCreator(r_data, 1)
self.data_loaded = True
print "Dane zaladowane"
def runSimulation(self):
"""
:return:
"""
sensors = self.sensors
sensorsList = SensorFactory.createSensorList(sensors, self.plane)
header = encoder(sensorsList)
buffer = Buffer(10, sensorsList)
self.plane.takeoff() # start the simulation
while self.plane.isFlying():
self.plane.update()
buffer.readData()
data = buffer.returnDataTr()
n = 0
k = 0
for s in self.sensors.keys():
self.getChart(n).clear()
self.getChart(n).set_title(s)
for i in range(self.sensors[s]):
self.getChart(n).plot(data[i+k])
k += self.sensors[s]
n += 1
self.canvas.show()
return self.plane.getName(), header, buffer.returnDataCopy()
def Test1():
env=simpy.Environment()
B = Buffer()
B.capacity=5
Source_Geo(env,"Source",0.5,B)
Server_Geo(env,"Server",0.6,B)
env.run(until=20)
def getANSIC():
import_file_path = filedialog.askopenfilename(
title="Seleccionar Archivo", filetypes=((".c Files", "*.c"), ))
Buffer.cargar_buffer(import_file_path)
# Listas para cada lista devuelta lista de la función tokenizar
token = []
lexema = []
fila = []
columna = []
total = []
# tokenizar y recargando el buffer
for i in Buffer.cargar_buffer(import_file_path):
t, lex, lin, col, tot = Analizador.tokenizar(i)
token += t
lexema += lex
fila += lin
columna += col
total += [str(tot)]
S = tk.Scrollbar(root)
T = tk.Text(root, height=20, width=92)
S.pack(side=tk.RIGHT, fill=tk.Y)
T.pack(side=tk.LEFT, fill=tk.Y)
S.config(command=T.yview)
T.config(yscrollcommand=S.set)
res = '\nTokens reconocidos:\n\n' + str(
token) + '\n\n\nLISTA DE TOKENS DETALLADA: \n\n' + '\n'.join(
[str(elem) for elem in total])
T.insert(tk.END, res)
def prepare_loaded_data(self, file_name):
"""
Przygotowuje i wczytuje dane z pliku
:param file_name: plik z danymi
:return:
"""
#utworzenie bufora
buff = Buffer()
data = CsvReader.read_from_file(file_name, 1)
#zmiana czasu
data = DataOperation.change_time_relative(data)
#zmiana wysokosci na metry
data = DataOperation.change_altitude_cm_m(data)
#stworzenie zapisywacza
saver = FileSaver("saved_data/dane.txt")
#kazda linijke z pliku csv buforujemy osobno
for d in data:
buff.set_data(d)
buffered_data = buff.get_data()
#sprawdzamy czy kazda linijka jest poprawnie zapisana
if InputValidator.input_val(buffered_data):
#zapisujemy kazda linijke do nowego pliku
saver.save_data(buffered_data)
#odczyt danych z pliku csv i wizualizacja
r_data = CsvReader.read_from_file(saver.get_file_name())
#tworzymy wizualizator, drugi parametr do interwal czasowy
self.visualizer = PlotCreator(r_data, 1)
self.data_loaded = True
print "Dane zaladowane"
class WeightBasedExpReplay(object):
def __init__(self, maxSize, alpha=0.6, epsilon=0.000001):
self.maxSize = maxSize
self.buffer = Buffer(self.maxSize)
self.sumTree = SumTree(self.maxSize)
self.weights = {}
self.alpha = 0.6
self.curSize = 0
self.epsilon = epsilon
self.heap = Heap()
def addExperience(self, experience):
weight = self.heap.getMaxPriority()
index = self.buffer.getPointer()
self.buffer.insert(experience)
prevWeight = 0
if index in self.weights:
prevWeight = self.weights[index]
diffWeight = weight - prevWeight
self.weights[index] = weight
self.sumTree.insert(diffWeight, index)
self.heap.add(index, weight)
self.curSize = min(self.curSize + 1, self.maxSize)
def modifyExperience(self, weight, index):
weight = weight + self.epsilon
weight = weight**self.alpha
prevWeight = 0
if index in self.weights:
prevWeight = self.weights[index]
diffWeight = weight - prevWeight
self.weights[index] = weight
self.sumTree.insert(diffWeight, index)
self.heap.add(index, weight)
def sample(self, samplesAmount):
startPoints = np.linspace(0, self.sumTree.getAllSum(),
samplesAmount + 1).tolist()
expList = []
weightList = []
indexList = []
for a in range(len(startPoints) - 1):
start = startPoints[a]
end = startPoints[a + 1]
sampledNum = np.random.uniform(start, end)
retrIndex = self.sumTree.search(sampledNum)
expList.append(self.buffer.getItem(retrIndex))
weightList.append(self.weights[retrIndex] /
self.sumTree.getAllSum())
indexList.append(retrIndex)
return np.asarray(expList), np.asarray(weightList), np.asarray(
indexList)
def getMaxPriority(self):
if self.heap.size == 0:
return sys.float_info.max
return self.heap.p2w[1]
def __init__(self, maxSize, alpha=0.6, epsilon=0.000001):
self.maxSize = maxSize
self.buffer = Buffer(self.maxSize)
self.sumTree = SumTree(self.maxSize)
self.weights = {}
self.alpha = 0.6
self.curSize = 0
self.epsilon = epsilon
self.heap = Heap()
def __init__(self, papaiNoel, listaElfo, listaRena, tamanhoGrupoElfos):
self.listaRena = listaRena
self.listaElfo = listaElfo
self.papaiNoel = papaiNoel
self.bufferElfo = Buffer(tamanhoGrupoElfos)
self.bufferRena = Buffer(len(self.listaRena))
self.contadorElfo = 0
self.contadorRena = 0
self.porta = Lock()
self.anel = Lock()
def __init__(self, h, w):
self.win = Window(top=0, x=w, y=h)
self.buffer = Buffer()
self.filename = ""
self.cursor = Cursor()
self.mode = "normal"
self.exit = False
self.handlers = Handler()
self.command = ""
self.message = ""
def __init__(self, env_dict, params):
"""
option_num, state_dim, action_dim, action_bound, gamma, learning_rate, replacement,
buffer_capacity, epsilon
gamma: (u_gamma, l_gamma)
learning_rate: (lr_u_policy, lr_u_critic, lr_option, lr_termin, lr_l_critic)
"""
# session
self.sess = tf.Session()
# environment parameters
self.sd = env_dict['state_dim']
self.ad = env_dict['action_dim']
a_bound = env_dict['action_scale']
assert a_bound.shape == (self.ad,), 'Action bound does not match action dimension!'
# hyper parameters
self.on = params['option_num']
epsilon = params['epsilon']
u_gamma = params['upper_gamma']
l_gamma = params['lower_gamma']
u_capac = params['upper_capacity']
l_capac = params['lower_capacity']
u_lrcri = params['upper_learning_rate_critic']
l_lrcri = params['lower_learning_rate_critic']
l_lrpol = params['lower_learning_rate_policy']
l_lrter = params['lower_learning_rate_termin']
# the frequency of training termination function
if params['delay'] == 'inf':
self.delay = -1
else:
self.delay = params['delay']
# Upper critic and buffer
self.u_critic = UCritic(session=self.sess, state_dim=self.sd, option_num=self.on,
gamma=u_gamma, epsilon=epsilon, learning_rate=u_lrcri)
self.u_buffer = Buffer(state_dim=self.sd, action_dim=1, capacity=u_capac)
# Lower critic, options and buffer HER
self.l_critic = LCritic(session=self.sess, state_dim=self.sd, action_dim=self.ad,
gamma=l_gamma, learning_rate=l_lrcri)
self.l_options = [Option(session=self.sess, state_dim=self.sd, action_dim=self.ad,
ordinal=i, learning_rate=[l_lrpol, l_lrter])
for i in range(self.on)]
self.l_buffers = [Buffer(state_dim=self.sd, action_dim=self.ad, capacity=l_capac)
for i in range(self.on)]
# Initialize all coefficients and saver
self.sess.run(tf.global_variables_initializer())
self.saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=100)
self.tc = 0 # counter for training termination
self.mc = 0 # counter for model
def connect(self, blocking=1):
self.fd.connect(self.path)
self.fd.setblocking(blocking)
m = Buffer()
m.put_int(MUX_MSG_HELLO)
m.put_int(SSHMUX_VER)
if self._write_packet(m) is not None:
return (False, 'Can\'t send HELLO message')
m = self._read_packet()
mux_msg = m.get_int()
if mux_msg != MUX_MSG_HELLO:
return (False, 'Expected HELLO reply, got %u' % (mux_msg,))
mux_ver = m.get_int()
if mux_ver != SSHMUX_VER:
return (False, 'Unsupported multiplexing protocol version %d (expected %d)' % (mux_ver, SSHMUX_VER))
extensions = {}
while len(m):
name = m.get_str()
value = m.get_str()
extensions[name] = value
return (True, extensions)
def connect_online_disconnect_offline(my_config):
server_thread = threading.Thread(target=run_simulation_server, args=[15])
server_thread.start()
my_buffer = Buffer(my_config)
ua_listener = OPCUAListener(my_config, my_buffer)
time.sleep(3)
ua_listener.connect()
time.sleep(30)
ua_listener.exit()
time.sleep(15)
assert my_buffer.len() >= 0
def __init__(self, Env_dim, Nb_action):
self.memory = Buffer(Memory_size)
self.eval_nn = Network(Env_dim, Nb_action)
self.target_nn = Network(Env_dim, Nb_action)
self.optimizer = torch.optim.Adam(self.eval_nn.parameters(),
lr=Learning_rate)
self.criterion = nn.MSELoss(reduction='sum')
self.counter = 0
self.target_nn.fc1 = self.eval_nn.fc1
self.target_nn.fc2 = self.eval_nn.fc2
self.target_nn.out = self.eval_nn.out
def __init__(self, connection: socket):
self.time_start = time.time()
self.connection = connection
self.buffer = Buffer(connection)
self.step = 0
self.username = None
self.stop_event = threading.Event()
self.username_wrong = None
self.start_time = time.time()
t = threading.Timer(30, self.after_done)
t.start()
super().__init__()
def Buffers(self, j):
o = flatbuffers.number_types.UOffsetTFlags.py_type(
self._tab.Offset(12))
if o != 0:
x = self._tab.Vector(o)
x += flatbuffers.number_types.UOffsetTFlags.py_type(j) * 4
x = self._tab.Indirect(x)
from Buffer import Buffer
obj = Buffer()
obj.Init(self._tab.Bytes, x)
return obj
return None
def __init__(self, config):
self.myBuffer = Buffer(config)
self.out_csv = ''
self.in_io_count = 0
self.out_io_count = 0
self.csv_row = ['-', '-', 0, 0, 0, 0, 0, 'hadoop', 'bin/spark']
self.in_ran_io_count = 0
self.in_seq_size_count = 0
self.out_ran_io_count = 0
self.out_seq_size = 0
self.write_seq_threshold = config['write_seq_threshold']
self.print_input_config(config)
def __init__(self, number: int, gel: "Global Event List"):
self.name = f"Host {number}"
self.status = "idle"
self.buffer = Buffer()
self.channel = gel.channel
self.GEL = gel
self.arrivalRate = 0.8 # lambda
self.senseTime = 0.01 # 0.01 ms
self.DIFS = 0.1 # 0.10 ms
self.SIFS = 0.05 # 0.05 ms
self.notACKedDict = {}
self.ackId = 0
def test_polling_interval(my_config):
server_thread = threading.Thread(target=run_simulation_server, args=[20])
server_thread.start()
my_buffer = Buffer(my_config)
ua_listener = OPCUAListener(my_config, my_buffer)
time.sleep(3)
ua_listener.connect()
time.sleep(5)
ua_listener.exit()
time.sleep(5)
server_thread.join()
print('Buffer length=', my_buffer.len())
assert 9 <= my_buffer.len() <= 11
def stop(self):
m = Buffer()
m.put_int(MUX_C_STOP_LISTENING)
rid = self.rid
self.rid += 1
m.put_int(rid)
if self._write_packet(m) is not None:
return (False, 'Can\'t send STOP-LISTENING request')
m = self._read_packet()
rep_msg = m.get_int()
rep_rid = m.get_int()
if rep_rid != rid:
return (False, 'Got unexpected request id %u, expected %u' % (rep_rid, rid))
if rep_msg != MUX_S_OK:
rep_reason = m.get_str()
if rep_msg == MUX_S_FAILURE:
return (False, 'Failure in STOP message: %s' % (rep_reason,))
elif rep_msg == MUX_S_PERMISSION_DENIED:
return (False, 'Permission denied for STOP message: %s' % (rep_reason,))
return (False, 'Unexpected server reply, got %u' % (rep_msg,))
return (True, None)
def convertToCgrom2(self, str): buf = Buffer() buf.buffer = str for i in range(len(buf)): ch = ord(buf[i]) if ch == 0x5d: buf[i] = chr(252) elif ch == 0x5b: buf[i] = chr(250) elif ch == 0x24: buf[i] = chr(162) elif ch == 0x40: buf[i] = chr(160) elif ch == 0x5c: buf[i] = chr(251) elif ch == 0x7b: buf[i] = chr(253) elif ch == 0x7d: buf[i] = chr(255) elif ch == 0x7c: buf[i] = chr(254) elif ch == 0x27 or ch == 0x60 or ch == 0xB4: buf[i] = chr(39) elif ch == 0xe8: buf[i] = chr(164) elif ch == 0xe9: buf[i] = chr(165) elif ch == 0xc8: buf[i] = chr(197) elif ch == 0xc9: buf[i] = chr(207) elif ch == 0xe4: buf[i] = chr(123) elif ch == 0xc4: buf[i] = chr(91) elif ch == 0xf6: buf[i] = chr(124) elif ch == 0xd6: buf[i] = chr(92) elif ch == 0xfc: buf[i] = chr(126) elif ch == 0xdc: buf[i] = chr(94) elif ch == 0x5e: buf[i] = chr(253) elif ch == 0x5f: buf[i] = chr(254) return buf.buffer
class StreamSocketDispatcher (Dispatcher):
def __init__(self, sock_family, sock_type, remote_address, fd=None):
Dispatcher.__init__(self, fd)
self.sock_family = sock_family
self.sock_type = sock_type
self.remote_address = remote_address
self.read_buffer = Buffer(4096)
self.write_buffer = Buffer(4096)
self.uid = None
self.gids = set()
# For AF_UNIX sockets find out the uid/gids of the caller.
if self.fd is not None and self.sock_family == socket.AF_UNIX:
self.uid = utils.get_peer_uid(fd)
self.gids = utils.uid_gids.get(self.uid, [])
# Connect if this is client side.
if self.client:
self.handle_connect()
print("uid:", self.uid, self.gids)
def readable(self):
return not self.read_buffer.full()
def writeable(self):
return not self.write_buffer.empty()
def handle_connect(self):
Dispatcher.handle_connect(self)
try:
fd = socket.socket(self.sock_family, self.sock_type)
fd.connect(self.remote_address)
self.fd = fd
except socket.error as e:
print("ERROR could not open socket %s, %s" % (e, self), file=sys.stderr)
def handle_read(self):
received = self.read_buffer.recv_into(self.fd)
if received == 0:
print("read: close")
self.handle_close()
else:
print("read:", self.read_buffer)
def handle_write(self):
sent = self.write_buffer.send_outoff(self.fd)
if sent == 0:
self.handle_close()
def __init__(self):
configs = load_configs("Configs.json")
value_configs = load_configs("Configs_ValueNN.json")
policy_configs = load_configs("Configs_PolicyNN.json")
#logging.basicConfig(format='%(asctime)s %(message)s', filename=configs["log_name"], level=logging.WARNING,
#datefmt="%Y-%m-%d %H:%M:%S")
#logging.info("Starting Optimizer.")
self.width = configs["width"]
self.height = configs["height"]
self.buffer_size = configs["buffer_size"]
self.filename = configs["data_filename"]
self.experiment_name = "Deviations/" + configs["val_filename"]
self.buffer = Buffer(self.buffer_size)
self.env = BlockRelocation(self.height, self.width)
#self.model = ValueNetwork(configs=value_configs)
#self.policy_network = PolicyNetwork(configs=policy_configs)
#self.combined_model = CombinedModel(configs=configs)
#self.value_wrapper = EstimatorWrapper(self.model)
#self.policy_wrapper = EstimatorWrapper(self.policy_network)
self.value_net = ValueNetworkKeras(value_configs)
self.policy_net = PolicyNetworkKeras(policy_configs)
self.tree_searcher = TreeSearch(
self.value_net, BlockRelocation(self.height, self.width),
self.policy_net)
self.tree_searcher.std_vals = load_obj(self.experiment_name)
self.baseline_params = {
"search_depth": 5,
"epsilon": 0.1,
"threshold": 0.01,
"drop_percent": 0.25,
"factor": 0.01
}
self.current_search_params = {
"search_depth": 5,
"epsilon": 0.1,
"threshold": 0.05,
"drop_percent": 0.3,
"factor": 0.05
}
self.dfs_params_hq = {"stop_param": 4, "k": 12}
self.dfs_params_fast = {"stop_param": 1, "k": 12}
class ExperienceReplay(object):
def __init__(self, maxCapacity):
self.maxCapacity = maxCapacity
self.buffer = Buffer(self.maxCapacity)
def pushItem(self, item):
self.buffer.pushItem(item)
def sample(self, batchSize):
availableIndexes = self.getLength()
sampledIndexes = sample(range(0, availableIndexes), batchSize)
return [self.buffer.getItem(index) for index in sampledIndexes]
def getLength(self):
return self.buffer.getLength()
def test_continous_reading_negative_case(my_config):
my_config['opcua']['number_of_reconnections'] = -1
my_config['metrics'][0]['interval'] = 500
server_thread = threading.Thread(target=run_simulation_server, args=[20])
server_thread.start()
my_buffer = Buffer(my_config)
ua_listener = OPCUAListener(my_config, my_buffer)
time.sleep(3)
ua_listener.connect()
time.sleep(10)
ua_listener.exit()
server_thread.join()
time.sleep(5)
list_of_metrics = [metric['metric_id'] for metric in my_config['metrics']]
metric_values = [[] for _ in range(len(list_of_metrics))]
for buffer_entity in my_buffer.buffer:
metric_idx = list_of_metrics.index(
buffer_entity.data['node'].metric_id)
metric_values[metric_idx].append(
buffer_entity.data['data_variant'].Value.Value)
increments_equal_one = True
for metric_value_set in metric_values:
for i in range(1, len(metric_value_set), 1):
if metric_value_set[i] != metric_value_set[i - 1] + 1:
increments_equal_one = False
break
assert increments_equal_one == False
def __init__(self,maxSize, alpha=0.6):
self.maxSize = maxSize
self.buffer = Buffer(self.maxSize)
self.heap = Heap()
self.weights = None
#Add two flags to indicate whether alpha or queue size has changed
self.prevAlpha = alpha
self.prevSize =0
# Variables to store current alpha and exp replay size
self.alpha = alpha
self.curSize = 0
#Weightings to each experience
self.endPoints = []
def eachSegment(self):
length = self.numberOfSegments()
for i in range(0, length):
samples = self.samplesForSegment(i)
buf = Buffer(samples=samples,
size=self.sizeForWindow,
sampleRate=self.buf.sampleRate)
yield (buf, i)
def __init__(self, argv=None):
if notSupported:
raise notSupported
if argv is not None:
self._argv = argv
if self._argv is _marker:
self._get_login_shell()
(pid, fd) = pty.fork()
if pid == 0:
self._spawn()
self._ptyfd = fd
self._ptypid = pid
self._new_text = None
fcntl.fcntl(self._ptyfd, fcntl.F_SETFL, os.O_NDELAY)
self._termProcess = True
Buffer.__init__(self, name=self._argv[0], text='')
def my_buffer():
cfg_file = '../src_test/config_test.yaml'
with open(cfg_file) as config_file:
cfg = yaml.safe_load(config_file)
test_buffer = Buffer(cfg)
for i in range(test_buffer.max_buffer_size - 1):
test_buffer.buffer.append(random() * 1000)
return test_buffer
def __add_buffer(self, log_item):
buf = Buffer(log_item, self.handler, maxsize=self.buffer_size)
if log_item.channel not in self.buffers:
self.buffers[log_item.channel] = {log_item.level: buf}
else:
if log_item.level not in self.buffers[log_item.channel]:
self.buffers[log_item.channel][log_item.level] = buf
return buf
class ExperienceReplay(object):
def __init__(self,maxSize):
self.maxSize = maxSize
self.buffer = Buffer(self.maxSize)
self.curSize = 0
def addExperience(self, *experience):
self.buffer.insert(Transition(*experience))
self.curSize = min(self.curSize+1,self.maxSize)
def sample(self, samplesAmount):
sampledPoints = np.random.choice(self.curSize, samplesAmount, replace=False).tolist()
expList = []
for a in sampledPoints :
expList.append(self.buffer.getItem(a))
return expList
class Agent():
def __init__(self, Env_dim, Nb_action):
self.memory = Buffer(Memory_size)
self.eval_nn = Network(Env_dim, Nb_action)
self.target_nn = Network(Env_dim, Nb_action)
self.optimizer = torch.optim.Adam(self.eval_nn.parameters(),
lr=Learning_rate)
self.criterion = nn.MSELoss(reduction='sum')
self.counter = 0
self.target_nn.fc1 = self.eval_nn.fc1
self.target_nn.fc2 = self.eval_nn.fc2
self.target_nn.out = self.eval_nn.out
def choose_action(self, s):
s = torch.unsqueeze(torch.FloatTensor(s), 0)
return self.eval_nn(s)[0].detach() # ae（s）
def getSample(self):
return self.memory.sample(Batch_size)
def optimize_model(self, file):
if self.memory.get_nb_elements() >= Batch_size:
batch = self.memory.sample(Batch_size)
for s, a, s_, r, done in batch:
qValues = (self.eval_nn(torch.tensor(s).float()))[a]
qValues_ = self.target_nn(torch.tensor(s_).float())
qValues_target = Gamma * torch.max(qValues_)
JO = pow(qValues - (r + (qValues_target * (1 - done))), 2)
loss = self.criterion(qValues, JO)
self.optimizer.zero_grad()
# if i != Batch_size - 1:
# loss.backward(retain_graph=True)
# else:
# loss.backward()
loss.backward()
self.optimizer.step()
self.counter += 1
if self.counter % Refresh_gap == 0:
torch.save(self.eval_nn, file)
self.target_nn.fc1 = self.eval_nn.fc1
self.target_nn.fc2 = self.eval_nn.fc2
self.target_nn.out = self.eval_nn.out
def store_transition(self, value):
self.memory.insert(value)
class ExperienceReplay(object):
def __init__(self,maxSize, alpha=0.6):
self.maxSize = maxSize
self.buffer = Buffer(self.maxSize)
self.curSize = 0
def addExperience(self, experience):
self.buffer.insert(experience)
self.curSize = min(self.curSize+1,self.maxSize)
def sample(self, samplesAmount):
sampledPoints = np.random.choice(self.curSize, samplesAmount, replace=False).tolist()
expList = []
weightList = []
for a in sampledPoints :
expList.append(self.buffer.getItem(a))
weightList.append(1.0/samplesAmount)
return np.asarray(expList), weightList, None
def run(cmd, quiet = False, abandon_output = True):
proc = Popen(split(cmd), stdout = PIPE, stderr = STDOUT)
buf = Buffer(abandon_output = abandon_output)
line = proc.stdout.readline()
while len(line):
buf.put(line)
if not quiet:
print(line, end = '')
line = proc.stdout.readline()
# Process could probably close the descriptor before exiting.
proc.wait()
if proc.returncode != 0:
raise Exception('Process exited with a non-zero return code. ' +
'Last output of the program:\n\n' +
'---------- Start of exception log --\n' +
buf.get_short().strip() +
'\n---------- End of exception log --\n')
return buf.get_long()
def analysis(self):
Buff = Buffer()
Stat = State()
#Buff.add_buff(self.Current)
#print(Buff.get_buff())
#print(self.Current)
self.Current = self.get_next()
while Stat.get_state() != 'final':
if Stat.get_state() == 'start':
if self.Current in self.Separators:
self.Current = self.get_next()
elif self.Current.isalpha():
Buff.clear_buff()
Buff.add_buff(self.Current)
Stat.set_state('ident')
self.Current = self.get_next()
#print(Buff.get_buff())
elif self.Current.isdigit():
Buff.clear_buff()
Buff.add_buff(self.Current)
Stat.set_state('numb')
self.Current = self.get_next()
elif self.Current == '{':
#ЗАПИСАТЬ КОММЕНТАРИЙ В БУФЕР
Stat.set_state('comment')
self.Current = self.get_next()
elif self.Current == '.':
Lexems_arr.append(
Lexer.Lexer(self.get_pos(), 'Разделитель',
self.Current, 'имя'))
Stat.set_state('final')
elif self.Current in self.Delimiters:
Stat.set_state('delimit')
self.Current = self.get_next()
def __init__(self,h,w): self.win = Window(top=0,x=w,y=h) self.buffer = Buffer() self.filename = "" self.cursor = Cursor() self.mode = "normal" self.exit = False self.handlers = Handler() self.command = "" self.message = ""
def __init__(self, sock_family, sock_type, remote_address, fd=None):
Dispatcher.__init__(self, fd)
self.sock_family = sock_family
self.sock_type = sock_type
self.remote_address = remote_address
self.read_buffer = Buffer(4096)
self.write_buffer = Buffer(4096)
self.uid = None
self.gids = set()
# For AF_UNIX sockets find out the uid/gids of the caller.
if self.fd is not None and self.sock_family == socket.AF_UNIX:
self.uid = utils.get_peer_uid(fd)
self.gids = utils.uid_gids.get(self.uid, [])
# Connect if this is client side.
if self.client:
self.handle_connect()
print("uid:", self.uid, self.gids)
def __init__(self,config):
self.myBuffer = Buffer(config)
self.out_csv = ''
self.in_io_count = 0
self.out_io_count = 0
self.csv_row = ['-','-',0,0,0,0,0,'hadoop','bin/spark']
self.in_ran_io_count = 0
self.in_seq_size_count = 0
self.out_ran_io_count = 0
self.out_seq_size= 0
self.write_seq_threshold = config['write_seq_threshold']
self.print_input_config(config)
def connect(self):
if (self.csv_loaded == False):
self.plotWidget.clear()
line = 1
rfile = ReadCSV.read("test.csv")
for row in rfile:
if(line == 1):
line = line+1
continue
self.upload_data.append(row)
Fixer.fixTimeAndAlt(self.upload_data)
buff = Buffer()
if os.path.exists('data.txt'):
os.remove('data.txt')
for i in self.upload_data:
buff.sendToBuffer(i)
buff.sendData()
self.data = (DataLoader.read('data.txt'))
self.csv_loaded = True
self.infocsv.setText(_fromUtf8("Zaladowano"))
self.info.setText("")
self.wys = self.kat = self.dyst = self.pred = self.odchyl = 0
def __init__(self, lexer, error_handler=None):
self.input = Buffer(lexer.tokenize())
self.error_handler = error_handler
self.is_eof = lambda t: isinstance(t, EOFToken)
self.is_def = lambda t: isinstance(t, DefToken)
self.is_extern = lambda t: isinstance(t, ExternToken)
self.is_if = lambda t: isinstance(t, IfToken)
self.is_then = lambda t: isinstance(t, ThenToken)
self.is_else = lambda t: isinstance(t, ElseToken)
self.is_for = lambda t: isinstance(t, ForToken)
self.is_in = lambda t: isinstance(t, InToken)
self.is_var = lambda t: isinstance(t, VarToken)
self.is_identifer = lambda t: isinstance(t, IdentifierToken)
self.is_number = lambda t: isinstance(t, NumberToken)
self.is_character = lambda t: isinstance(t, CharacterToken)
self.is_binary = lambda t: isinstance(t, BinaryToken)
self.is_unary = lambda t: isinstance(t, UnaryToken)
self.is_binop = lambda t: OperatorManager.is_binop(t)
self.is_unop = lambda t: OperatorManager.is_unop(t)
def check(self):
m = Buffer()
m.put_int(MUX_C_ALIVE_CHECK)
rid = self.rid
self.rid += 1
m.put_int(rid)
if self._write_packet(m) is not None:
return (False, 'Can\'t send ALIVE-CHECK request')
m = self._read_packet()
rep_msg = m.get_int()
if rep_msg != MUX_S_ALIVE:
return (False, 'Expected ALIVE reply, got %u' % (rep_msg,))
rep_rid = m.get_int()
if rep_rid != rid:
return (False, 'Got unexpected request id %u, expected %u' % (rep_rid, rid))
rep_pid = m.get_int()
return (True, rep_pid)
def __init__(self):
self.program = Program('Shader/Quad.prog')
# Prepare quad geometry
self.vbuffer = Buffer(gl.GL_ARRAY_BUFFER, gl.GL_STATIC_DRAW, Vertex)
self.vbuffer.push(Vertex(-.5, -.5, 0, 0, 1))
self.vbuffer.push(Vertex(-.5, .5, 0, 0, 0))
self.vbuffer.push(Vertex( .5, -.5, 0, 1, 1))
self.vbuffer.push(Vertex( .5, .5, 0, 1, 0))
# Set up vertex array object for mapping structs => shader inputs
vao = (ctypes.c_int * 1)()
gl.glGenVertexArrays(1, vao)
self.vao = vao[0]
stride = ctypes.sizeof(Vertex)
gl.glBindVertexArray(self.vao)
gl.glBindBuffer(gl.GL_ARRAY_BUFFER, self.vbuffer.id)
gl.glEnableVertexAttribArray(Attr.POSITION)
offset = ctypes.c_void_p(0)
gl.glVertexAttribPointer(Attr.POSITION, 3, gl.GL_FLOAT, gl.GL_FALSE, stride, offset)
gl.glEnableVertexAttribArray(Attr.TEXCOORD)
offset = ctypes.c_void_p(12)
gl.glVertexAttribPointer(Attr.TEXCOORD, 2, gl.GL_FLOAT, gl.GL_FALSE, stride, offset)
gl.glBindVertexArray(0)
def _write_packet(self, buf):
pkt = Buffer()
pkt.put_str(buf)
return self.fd.sendall(pkt)
class MinerProtocol(Protocol):
""" implementation of the Minecraft protocol """
def __init__(self, name):
global debug
self.log = debug
self.bot = MinerBot(name, self.send)
self.buffer = Buffer()
self.world = World()
self.packets = {
# ID: (function, format
0x00: (self.onKeepAlive, Standard("Bi")), # Keep alive
0x01: (self.onLoginRequest, String("BiSqibbBB")), # Login request
0x02: (self.onHandshake, String("BS")), # Handshake
0x03: (self.onChat, String("BS")), # Chat message
0x04: (None, Standard("Bq")), # Time update
0x05: (None, Standard("Bihhh")), # Entity Equipment
0x06: (self.onSpawnPosition, Standard("Biii")), # Spawn position
0x07: (None, Standard("Bii?")), # Use entity (C --> S)
0x08: (self.onUpdateHealth, Standard("Bhhf")), # Update health
0x09: (None, Standard("Bbbbhl")), # Respawn (C --> S)
0x0A: (None, Standard("B?")), # Player (C --> S)
0x0B: (None, Standard("Bdddd?")), # Player position (C --> S)
0x0C: (None, Standard("Bff?")), # Player look (C --> S)
0x0D: (self.onPLayerPosition, Standard("Bddddff?")), # Player position & look
0x0E: (None, Standard("Bbibib")), # Player digging (C --> S)
0x0F: (None, BlockSlot("BibibW")), # Player block placement (C --> S)
0x10: (None, Standard("Bh")), # Holding change (C --> S)
0x11: (None, Standard("Bibibi")), # Use bed
0x12: (None, Standard("Bib")), # Animation
0x13: (None, Standard("Bib")), # Entity action (C --> S)
0x14: (self.onNamedEntitySpawn, String("BiSiiibbh")), # Named entity spawn
0x15: (None, Standard("Bihbhiiibbb")), # Pickup spawn
0x16: (None, Standard("Bii")), # Collect item
0x17: (self.onAddObject, Standard("Bibiiiihhh")), # Add object/vehicle
0x18: (None, MetaEntity("BibiiibbE")), # Mob spawn
0x19: (None, String("BiSiiii")), # Entity: painting
0x1A: (None, Standard("Biiiih")), # Experience Orb
0x1B: (None, Standard("Bffff??")), # Stance update
0x1C: (None, Standard("Bihhh")), # Entity velocity
0x1D: (None, Standard("Bi")), # Destroy entity
0x1E: (None, Standard("Bi")), # Entity
0x1F: (self.onEntityRelativeMove, Standard("Bibbb")), # Entity relative move
0x20: (None, Standard("Bibb")), # Entity look
0x21: (None, Standard("Bibbbbb")), # Entity look and relative move
0x22: (None, Standard("Biiiibb")), # Entity teleport
0x26: (None, Standard("Bib")), # Entity status
0x27: (None, Standard("Bii")), # Attach entity
0x28: (None, MetaEntity("BiE")), # Entity metadata
0x29: (None, Standard("Bibbh")), # Entity Effect
0x2A: (None, Standard("Bib")), # Remove Entity Effect
0x2B: (None, Standard("Bfhh")), # Experience
0x32: (self.onPreChunk, Standard("Bii?")), # Pre chunk
0x33: (self.onChunk, MetaChunk()), # Map chunk
0x34: (None, BlockArray()), # Multi-block change
0x35: (None, Standard("Bibibb")), # Block change
0x36: (None, Standard("Bihibb")), # Block action
0x3C: (None, BlockExplosion()), # Explosion
0x3D: (None, Standard("Biibii")), # Sound effect
0x46: (None, Standard("Bbb")), # New/invalid state
0x47: (None, Standard("Bi?iii")), # Thunderbolt
0x64: (None, String("BbbSb")), # Open window
0x65: (None, Standard("Bb")), # Close window
0x66: (None, BlockSlot("Bbhbh?W")), # Window click
0x67: (None, BlockSlot("BbhW")), # Set slot
0x68: (self.onWindowItems, Window()), # Window items
0x69: (None, Standard("Bbhh")), # Update window property
0x6A: (self.onTransaction, Standard("Bbh?")), # Transaction
0x6B: (None, BlockSlot("BhW")), # Creative inventory action
0x6C: (None, Standard("Bbb")), # Enchant Item
0x82: (None, String("BihiSSSS")), # Update sign
0x83: (None, ByteArray()), # item data
0xC8: (None, Standard("Bib")), # Increment statistic
0xC9: (self.onPlayerList, String("BS?h")), # Player List Item
0xFF: (self.onDisconnected, String("BS")), # Disconnect
}
def connectionMade(self):
""" called when a connection has been established """
self.log.info("<%s> connected" % self.bot.getName())
# self.writer = PacketWriter(self.transport)
self.log.debug("ATTEMPTING TO HANDSHAKE")
self.sendHandshake(self.bot.getName())
def connectionLost(self, reason):
self.log.error("connection lost %s" % reason)
def dataReceived(self, data):
""" new data are received """
self.buffer.append(data)
while True:
# get the packet Type
try:
data = self.buffer.peek()[0]
packetType = ord(data)
except IOError: # if empty buffer stop the loop
break
self.log.received(packetType)
# get the packet information
if packetType not in self.packets:
self.log.error("Unknown packet 0x%02X" % packetType)
self.transport.loseConnection()
fct, fmt = self.packets[packetType]
try:
packet = fmt.unpack(self.buffer)
except IOError:
break
# do action on packet received
if fct!= None:
fct(self.bot, packet)
def onDisconnected(self, bot, info):
self.log.info("Disconnected by server. Reason=%s" % info[1])
def onKeepAlive(self, bot, info):
self.send(0x00, 0)
def onUpdateHealth(self, bot, info):
bot.setHealth(health=info[1], food=info[2], foodSaturation=info[3])
def onLoginRequest(self, bot, info):
bot.setUID(info[1])
#bot.setMaxSeed(info[3])
pass
def onHandshake(self, bot, info):
#bot.setConnectionHash(info[1])
self.sendLoginRequest(22, self.bot.getName())
self.bot.nameToUpper()
def onSpawnPosition(self, bot, info):
bot.setSpawnPosition(info)
def onPLayerPosition(self, bot, info):
cmd, x, stance, y, z, yaw, pitch, onGround = info
bot.setPlayerPosition(x, y, z, stance, yaw, pitch, onGround)
self.send(0x0D, x, y, stance, z, yaw, pitch, onGround)
def onEntityRelativeMove(self, bot, info):
bot.setEntityRelativeMove(info)
def onWindowItems(self, bot, info):
pass
# if info[1] == 0:
# # inventory window
# bot.setInventory(info[2], info[3])
# else:
# logging.error("Unknown window number=%d" % info[1])
# def onSetSlot(self, bot, info):
# if info[1] == 0:
# bot.setSlotItem(info[1], info[2], info[3], info[4], info[5])
# logging.debug("SetSlot %d, slot %d" %(info[1], info[2]))
def onNamedEntitySpawn(self, bot, info):
bot.setPlayerSpawn(info)
def onAddObject(self):
pass
def onPreChunk(self, bot, info):
pass
def onChunk(self, bot, info):
cmd, x, y, z, sx, sy, sz, compSize, chunkData = info
# correct the real size
sx += 1
sy += 1
sz += 1
self.log.chunk("Chunk (x,y,z)=(%d, %d, %d) (sx,sy,sz)=(%d, %d, %d), len=%d" % (x, y, z, sx, sy, sz, len(chunkData)))
self.world.addChunk(x, y, z, sx, sy, sz, chunkData)
def onTransaction(self, bot, info):
pass
def send(self, cmd, *data):
if cmd not in self.packets:
self.log.error("Unknown packet to send 0x%02X" % cmd)
self.transport.loseConnection()
fct, pack = self.packets[cmd]
res = pack.pack(cmd, *data)
deb = ""
for x in res:
deb += "%02X " % ord(x)
self.transport.write(res)
self.log.send(cmd)
def sendDisconnect(self, reason):
self.send(0xFF, "Disconnection by client")
self.transport.loseConnection()
def sendHandshake(self, userName):
self.send(0x02, userName)
def sendLoginRequest(self, protocol_version, username):
self.send(0x01, protocol_version, username, 0, 0, 0, 0, 0, 0)
def onChat(self, bot, info):
'''
Chat received. Analyse it to detect a message from the player
'''
info = info[1].upper()
if ord(info[0]) == 0xA7: #skip color code
info = info[2:]
# if the message is from a player it starts with <PlayerName>
result = re.match("<(.*)>(.*)", info)
if result == None:
name = ""
msg = info
else:
name = result.group(1)
msg = result.group(2)
result = msg.split()
if BOT_NAME_NEEDED: # Command starts with the targeted Bot name
if result[0] == bot.getName():
bot.parseCommand(name, result[1:])
else:
if name <> bot.getName(): # do not compute self message
bot.parseCommand(name, result)
def onPlayerList(self, bot, info):
bot.updatePlayerList(info[1], info[2])
def __init__(self, name):
global debug
self.log = debug
self.bot = MinerBot(name, self.send)
self.buffer = Buffer()
self.world = World()
self.packets = {
# ID: (function, format
0x00: (self.onKeepAlive, Standard("Bi")), # Keep alive
0x01: (self.onLoginRequest, String("BiSqibbBB")), # Login request
0x02: (self.onHandshake, String("BS")), # Handshake
0x03: (self.onChat, String("BS")), # Chat message
0x04: (None, Standard("Bq")), # Time update
0x05: (None, Standard("Bihhh")), # Entity Equipment
0x06: (self.onSpawnPosition, Standard("Biii")), # Spawn position
0x07: (None, Standard("Bii?")), # Use entity (C --> S)
0x08: (self.onUpdateHealth, Standard("Bhhf")), # Update health
0x09: (None, Standard("Bbbbhl")), # Respawn (C --> S)
0x0A: (None, Standard("B?")), # Player (C --> S)
0x0B: (None, Standard("Bdddd?")), # Player position (C --> S)
0x0C: (None, Standard("Bff?")), # Player look (C --> S)
0x0D: (self.onPLayerPosition, Standard("Bddddff?")), # Player position & look
0x0E: (None, Standard("Bbibib")), # Player digging (C --> S)
0x0F: (None, BlockSlot("BibibW")), # Player block placement (C --> S)
0x10: (None, Standard("Bh")), # Holding change (C --> S)
0x11: (None, Standard("Bibibi")), # Use bed
0x12: (None, Standard("Bib")), # Animation
0x13: (None, Standard("Bib")), # Entity action (C --> S)
0x14: (self.onNamedEntitySpawn, String("BiSiiibbh")), # Named entity spawn
0x15: (None, Standard("Bihbhiiibbb")), # Pickup spawn
0x16: (None, Standard("Bii")), # Collect item
0x17: (self.onAddObject, Standard("Bibiiiihhh")), # Add object/vehicle
0x18: (None, MetaEntity("BibiiibbE")), # Mob spawn
0x19: (None, String("BiSiiii")), # Entity: painting
0x1A: (None, Standard("Biiiih")), # Experience Orb
0x1B: (None, Standard("Bffff??")), # Stance update
0x1C: (None, Standard("Bihhh")), # Entity velocity
0x1D: (None, Standard("Bi")), # Destroy entity
0x1E: (None, Standard("Bi")), # Entity
0x1F: (self.onEntityRelativeMove, Standard("Bibbb")), # Entity relative move
0x20: (None, Standard("Bibb")), # Entity look
0x21: (None, Standard("Bibbbbb")), # Entity look and relative move
0x22: (None, Standard("Biiiibb")), # Entity teleport
0x26: (None, Standard("Bib")), # Entity status
0x27: (None, Standard("Bii")), # Attach entity
0x28: (None, MetaEntity("BiE")), # Entity metadata
0x29: (None, Standard("Bibbh")), # Entity Effect
0x2A: (None, Standard("Bib")), # Remove Entity Effect
0x2B: (None, Standard("Bfhh")), # Experience
0x32: (self.onPreChunk, Standard("Bii?")), # Pre chunk
0x33: (self.onChunk, MetaChunk()), # Map chunk
0x34: (None, BlockArray()), # Multi-block change
0x35: (None, Standard("Bibibb")), # Block change
0x36: (None, Standard("Bihibb")), # Block action
0x3C: (None, BlockExplosion()), # Explosion
0x3D: (None, Standard("Biibii")), # Sound effect
0x46: (None, Standard("Bbb")), # New/invalid state
0x47: (None, Standard("Bi?iii")), # Thunderbolt
0x64: (None, String("BbbSb")), # Open window
0x65: (None, Standard("Bb")), # Close window
0x66: (None, BlockSlot("Bbhbh?W")), # Window click
0x67: (None, BlockSlot("BbhW")), # Set slot
0x68: (self.onWindowItems, Window()), # Window items
0x69: (None, Standard("Bbhh")), # Update window property
0x6A: (self.onTransaction, Standard("Bbh?")), # Transaction
0x6B: (None, BlockSlot("BhW")), # Creative inventory action
0x6C: (None, Standard("Bbb")), # Enchant Item
0x82: (None, String("BihiSSSS")), # Update sign
0x83: (None, ByteArray()), # item data
0xC8: (None, Standard("Bib")), # Increment statistic
0xC9: (self.onPlayerList, String("BS?h")), # Player List Item
0xFF: (self.onDisconnected, String("BS")), # Disconnect
}
class Parser(object):
def __init__(self, lexer, error_handler=None):
self.input = Buffer(lexer.tokenize())
self.error_handler = error_handler
self.is_eof = lambda t: isinstance(t, EOFToken)
self.is_def = lambda t: isinstance(t, DefToken)
self.is_extern = lambda t: isinstance(t, ExternToken)
self.is_if = lambda t: isinstance(t, IfToken)
self.is_then = lambda t: isinstance(t, ThenToken)
self.is_else = lambda t: isinstance(t, ElseToken)
self.is_for = lambda t: isinstance(t, ForToken)
self.is_in = lambda t: isinstance(t, InToken)
self.is_var = lambda t: isinstance(t, VarToken)
self.is_identifer = lambda t: isinstance(t, IdentifierToken)
self.is_number = lambda t: isinstance(t, NumberToken)
self.is_character = lambda t: isinstance(t, CharacterToken)
self.is_binary = lambda t: isinstance(t, BinaryToken)
self.is_unary = lambda t: isinstance(t, UnaryToken)
self.is_binop = lambda t: OperatorManager.is_binop(t)
self.is_unop = lambda t: OperatorManager.is_unop(t)
def collect(self):
return self.input.accept()
def look(self, condition=None):
return self.input.peek(condition)
def expect(self, condition=None):
return self.input.move(condition)
def try_number_expr(self):
"""
number_expr ::= number
"""
number = self.expect(self.is_number)
if number is None:
return None
return NumberExprNode(number)
def try_paren_expr(self):
"""
paren_expr ::= '(' expr ')'
"""
left_paren = self.expect(lambda t: t.name == '(')
if left_paren is None:
return None
expr = self.try_expr()
right_paren = self.expect(lambda t: t.name == ')')
if right_paren is None:
raise ExpectedRightParen(left_paren.line if expr is None else expr.line)
return expr
def try_identifier_expr(self):
"""
identifier_expr
::= identifier
::= identifier '(' ')'
::= identifier '(' expr (',' expr)* ')'
"""
identifier = self.expect(self.is_identifer)
if identifier is None:
return None
left_paren = self.expect(lambda t: t.name == '(')
if left_paren is None:
return VariableExprNode(identifier)
else:
args = []
while True:
arg = self.try_expr()
if arg is not None:
args.append(arg)
self.expect(lambda t: t.name == ',')
else:
right_paren = self.expect(lambda t: t.name == ')')
if right_paren is None:
raise ExpectedRightParen(left_paren.line if len(args) == 0 else args[-1].line)
return CallExprNode(identifier, args)
def try_if_expr(self):
"""
if_expr ::= if expr then expr else expr
"""
token = self.expect(self.is_if)
if token is None:
return None
condition = self.try_expr()
if condition is None:
raise ExpectedExpr(token.line)
token = self.expect(self.is_then)
if token is None:
raise ExpectedThen(condition.line)
true = self.try_expr()
if true is None:
raise ExpectedExpr(token.line)
token = self.expect(self.is_else)
if token is None:
raise ExpectedElse(true.line)
false = self.try_expr()
if false is None:
raise ExpectedExpr(token.line)
return IfExprNode(condition, true, false)
def try_for_expr(self):
"""
for_expr ::= for identifier '=' expr ',' expr (',' expr)? in expr
"""
token = self.expect(self.is_for)
if token is None:
return None
variable = self.expect(self.is_identifer)
if variable is None:
raise ExpectedIdentifier(token.line)
token = self.expect(lambda t: t.name == '=')
if token is None:
raise ExpectedEqualSign(variable.line)
begin = self.try_expr()
if begin is None:
raise ExpectedExpr(token.line)
token = self.expect(lambda t: t.name == ',')
if token is None:
raise ExpectedComma(begin.line)
end = self.try_expr()
if end is None:
raise ExpectedExpr(token.line)
token = self.expect(lambda t: t.name == ',')
if token is not None:
step = self.try_expr()
if step is None:
raise ExpectedExpr(token.line)
else:
step = None
token = self.expect(self.is_in)
if token is None:
raise ExpectedIn(end.line if step is None else step.line)
body = self.try_expr()
if body is None:
raise ExpectedExpr(token.line)
return ForExprNode(variable, begin, end, step, body)
def try_var_expr(self):
"""
var_expr ::= var identifier ('=' expr)? (',' identifier ('=' expr)?)* in expr
"""
var_token = self.expect(self.is_var)
if var_token is None:
return None
variables = {}
while True:
identifier = self.expect(self.is_identifer)
if identifier is None:
break
assignment = self.expect(lambda t: t.name == '=')
if assignment is not None:
expr = self.try_expr()
if expr is None:
raise ExpectedExpr(assignment.line)
variables[identifier] = expr
else:
variables[identifier] = None
self.expect(lambda t: t.name == ',')
if len(variables) == 0:
raise ExpectedVariableList(var_token.line)
in_token = self.expect(self.is_in)
if in_token is None:
raise ExpectedIn(max([var.line for var in variables.keys()]))
body = self.try_expr()
if body is None:
raise ExpectedExpr(in_token.line)
return VarExprNode(variables, body)
def try_primary_expr(self):
"""
primary_expr
::= number_expr
::= paren_expr
::= identifier_expr
::= if_expr
::= for_expr
::= var_expr
"""
expr = self.try_number_expr()
if expr is not None:
return expr
expr = self.try_paren_expr()
if expr is not None:
return expr
expr = self.try_identifier_expr()
if expr is not None:
return expr
expr = self.try_if_expr()
if expr is not None:
return expr
expr = self.try_for_expr()
if expr is not None:
return expr
return self.try_var_expr()
def try_unary_expr(self):
"""
unary_expr
::= primary_expr
::= unop unary_expr
"""
unop = self.expect(self.is_unop)
if unop is None:
return self.try_primary_expr()
operand = self.try_unary_expr()
if operand is None:
raise ExpectedOperand(unop.line)
return UnaryExprNode(unop, operand)
def try_binop_rhs(self, lhs, lhs_prec):
"""
binop_rhs ::= (binop unary_expr)*
"""
while True:
binop = self.look(self.is_binop)
if binop is None:
return lhs
prec = OperatorManager.get_binop_precedence(binop)
if prec < lhs_prec:
return lhs
self.expect(self.is_binop) # eat binop
rhs = self.try_unary_expr()
if rhs is None:
raise ExpectedUnaryExpr(binop.line)
next_binop = self.look(self.is_binop)
if next_binop is not None and \
OperatorManager.get_binop_precedence(next_binop) > prec:
rhs = self.try_binop_rhs(rhs, prec + 1)
lhs = BinaryExprNode(binop, lhs, rhs)
def try_expr(self):
"""
expr ::= unary_expr binop_rhs
"""
lhs = self.try_unary_expr()
if lhs is None:
return None
return self.try_binop_rhs(lhs, 0)
def try_normal_prototype(self):
"""
normal_prototype ::= identifier '(' identifier* ')'
"""
identifier = self.expect(self.is_identifer)
if identifier is None:
return None
left_paren = self.expect(lambda t: t.name == '(')
if left_paren is None:
raise ExpectedLeftParen(identifier.line)
args = []
while True:
arg = self.expect(self.is_identifer)
if arg is not None:
args.append(arg)
else:
right_paren = self.expect(lambda t: t.name == ')')
if right_paren is None:
raise ExpectedRightParen(left_paren.line if len(args) == 0 else args[-1].line)
return PrototypeNode(identifier, args)
def try_binary_prototype(self):
"""
binary_prototype ::= binary character number? '(' identifier identifier ')'
"""
binary = self.expect(self.is_binary)
if binary is None:
return None
operator = self.expect(self.is_character)
if operator is None:
raise ExpectedOperator(binary.line)
precedence = self.expect(self.is_number)
left_paren = self.expect(lambda t: t.name == '(')
if left_paren is None:
raise ExpectedLeftParen(operator.line if precedence is None else precedence.line)
arg1 = self.expect(self.is_identifer)
if arg1 is None:
raise ExpectedIdentifier(left_paren.line)
arg2 = self.expect(self.is_identifer)
if arg2 is None:
raise ExpectedIdentifier(arg1.line)
right_paren = self.expect(lambda t: t.name == ')')
if right_paren is None:
raise ExpectedRightParen(arg2.line)
return BinOpPrototypeNode(IdentifierToken(binary.name + operator.name, binary.line),
precedence, [arg1, arg2])
def try_unary_prototype(self):
"""
unary_prototype ::= unary character '(' identifier ')'
"""
unary = self.expect(self.is_unary)
if unary is None:
return None
operator = self.expect(self.is_character)
if operator is None:
raise ExpectedOperator(unary.line)
left_paren = self.expect(lambda t: t.name == '(')
if left_paren is None:
raise ExpectedLeftParen(operator.line)
arg = self.expect(self.is_identifer)
if arg is None:
raise ExpectedIdentifier(left_paren.line)
right_paren = self.expect(lambda t: t.name == ')')
if right_paren is None:
raise ExpectedRightParen(arg.line)
return UnOpPrototypeNode(IdentifierToken(unary.name + operator.name, unary.line), [arg])
def try_prototype(self):
"""
prototype
::= normal_prototype
::= binary_prototype
::= unary_prototype
"""
prototype = self.try_normal_prototype()
if prototype is not None:
return prototype
prototype = self.try_binary_prototype()
if prototype is not None:
return prototype
prototype = self.try_unary_prototype()
if prototype is not None:
return prototype
def try_function(self):
"""
function ::= def prototype expr
"""
keyword = self.expect(self.is_def)
if keyword is None:
return None
prototype = self.try_prototype()
if prototype is None:
raise ExpectedPrototype(keyword.line)
expr = self.try_expr()
if expr is None:
raise ExpectedExpr(prototype.line)
return FunctionNode(prototype, expr)
def try_declaration(self):
"""
declaration ::= extern prototype
"""
keyword = self.expect(self.is_extern)
if keyword is None:
return None
prototype = self.try_prototype()
if prototype is None:
raise ExpectedPrototype(keyword.line)
return prototype
def try_toplevel_expr(self):
"""
toplevel_expr ::= expr
we make an anonymous prototype to represent a top-level expr
"""
expr = self.try_expr()
if expr is None:
return None
return TopLevelExpr(expr)
def try_negligible_character(self):
# ignore top-level semicolons
return self.expect(lambda t: t.name == ';')
def try_eof(self):
return self.expect(self.is_eof)
def try_unknown(self):
# try to recovery from syntax errors by eating an unknown token
token = self.expect()
if token is not None:
raise UnknownToken(token)
def parse(self):
while True:
try:
node = self.try_function()
if node is not None:
yield node
continue
node = self.try_declaration()
if node is not None:
yield node
continue
node = self.try_toplevel_expr()
if node is not None:
yield node
continue
if self.try_negligible_character() is not None:
continue
if self.try_eof() is not None:
break
self.try_unknown()
except BoidaeSyntaxError as e:
if self.error_handler is not None:
self.error_handler(e)
def forward(self, mode, ftype, listen_host, listen_port, connect_host, connect_port):
m = Buffer()
if mode:
n = 'OPEN'
m.put_int(MUX_C_OPEN_FWD)
else:
n = 'CLOSE'
m.put_int(MUX_C_CLOSE_FWD)
rid = self.rid
self.rid += 1
m.put_int(rid)
if isinstance(ftype, basestring):
assert ftype in _fwd_names, 'Invalid forward type %s' % (ftype,)
ftype = _fwd_names[ftype]
m.put_int(ftype)
m.put_str(listen_host)
m.put_int(listen_port)
m.put_str(connect_host)
m.put_int(connect_port)
if self._write_packet(m) is not None:
return (False, 'Can\'t send %s-FORWARD request' % (n,))
m = self._read_packet()
rep_msg = m.get_int()
rep_rid = m.get_int()
if rep_rid != rid:
return (False, 'Got unexpected request id %u, expected %u' % (rep_rid, rid))
if rep_msg != MUX_S_OK and rep_msg != MUX_S_REMOTE_PORT:
rep_reason = m.get_str()
if rep_msg == MUX_S_FAILURE:
return (False, 'Failure in %s-FORWARD request: %s' % (n, rep_reason,))
elif rep_msg == MUX_S_PERMISSION_DENIED:
return (False, 'Permission denied for %s-FORWARD request: %s' % (n, rep_reason,))
return (False, 'Unexpected server reply, got %u' % (rep_msg,))
if ftype == MUX_FWD_REMOTE and listen_port == 0:
if rep_msg != MUX_S_REMOTE_PORT:
return (False, 'Expected remote port reply, got %u' % (rep_msg,))
rep_port = m.get_int()
return (True, rep_port)
return (True, None)
def __init__(self,config):
self.myBuffer = Buffer(config)
self.out_csv = ''
self.in_io_count = 0
self.out_io_count = 0
self.csv_row = ['-','-',0,0,0,0,0,'hadoop','bin/spark']
class Shaper():
def __init__(self,config):
self.myBuffer = Buffer(config)
self.out_csv = ''
self.in_io_count = 0
self.out_io_count = 0
self.csv_row = ['-','-',0,0,0,0,0,'hadoop','bin/spark']
def read_csv(self,in_file):
colnames = ['STIME','TIME','UID','PID','D','BLOCK','SIZE','COMM','PATHNAME']
data = pandas.read_csv(in_file,names = colnames)
data_dir = data.D.tolist()
data_addr = map(lambda x: x//8, data.BLOCK.tolist())
data_size = map(lambda x: x//4096, data.SIZE.tolist())
arrs = []
length = len(data_dir)
self.start_time = time.time()
for i in range(0,length):
arrs.append([data_dir[i],data_addr[i],data_size[i]])
return arrs
def run(self,in_file,out_file):
arrs = self.read_csv(in_file)
self.in_io_count = len(arrs)
self.out_csv = open(out_file,'wb')
for each in arrs:
if each[0]=='W':
#TODO: big file, dont put into cache, need to invalidate the data in cache if cache hit
while not self.myBuffer.add([each[1],each[1]+each[2]]):
ios = self.myBuffer.get_cold()
self.gen_write_io(ios)
else:
#TODO: check cache hit, if hit, return data immediately instead of generate read io
ios = [[each[1],each[2]]]
self.gen_read_io(ios)
#clear all data in Buffer
ios = self.myBuffer.get_all();
self.gen_write_io(ios)
arrs = self.read_csv(out_file)
self.out_io_count = len(arrs)
def gen_write_io(self,ios):
for each in ios:
self.write_csv(['W',each[0],each[1]])
def gen_read_io(self,ios):
for each in ios:
self.write_csv(['R',each[0],each[1]])
def write_csv(self,item):
content_list = []
self.csv_row[4] = item[0]
self.csv_row[5] = item[1]*8
self.csv_row[6] = item[2]*4096
content_list.append(self.csv_row)
csv_writer = csv.writer(self.out_csv)
csv_writer.writerows(content_list)
def print_io_count(self):
print "=========================================================="
print "I/O count before optimization:", self.in_io_count
print "I/O count after optimization:", self.out_io_count
print "=========================================================="
def forwards(self):
m = Buffer()
m.put_int(MUX_C_LIST_FWDS)
rid = self.rid
self.rid += 1
m.put_int(rid)
if self._write_packet(m) is not None:
return (False, 'Can\'t send LIST-FORWARDS request')
m = self._read_packet()
rep_msg = m.get_int()
rep_rid = m.get_int()
if rep_rid != rid:
return (False, 'Got unexpected request id %u, expected %u' % (rep_rid, rid))
if rep_msg != MUX_S_RESULT:
rep_reason = m.get_str()
if rep_msg == MUX_S_FAILURE:
return (False, 'Failure in LIST-FORWARDS request: %s' % (rep_reason,))
elif rep_msg == MUX_S_PERMISSION_DENIED:
return (False, 'Permission denied for LIST-FORWARDS request: %s' % (rep_reason,))
return (False, 'Unexpected server reply, got %u' % (rep_msg,))
fwds = []
while len(m):
fid = m.get_int()
ftype = m.get_int()
listen_host = m.get_str()
listen_port = m.get_int()
connect_host = m.get_str()
connect_port = m.get_int()
if ftype == MUX_FWD_REMOTE and listen_port == 0:
listen_port = m.get_int()
fwds.append((fid, _fwd_types[ftype], listen_host, listen_port, connect_host, connect_port))
return (True, fwds)
def _read_packet(self):
m = Buffer(self.fd.recv(4))
pktlen = m.get_int()
return Buffer(self.fd.recv(pktlen))
def sessions(self):
m = Buffer()
m.put_int(MUX_C_LIST_SESSIONS)
rid = self.rid
self.rid += 1
m.put_int(rid)
if self._write_packet(m) is not None:
return (False, 'Can\'t send LIST-SESSIONS request')
m = self._read_packet()
rep_msg = m.get_int()
rep_rid = m.get_int()
if rep_rid != rid:
return (False, 'Got unexpected request id %u, expected %u' % (rep_rid, rid))
if rep_msg != MUX_S_RESULT:
rep_reason = m.get_str()
if rep_msg == MUX_S_FAILURE:
return (False, 'Failure in LIST-SESSIONS request: %s' % (rep_reason,))
elif rep_msg == MUX_S_PERMISSION_DENIED:
return (False, 'Permission denied for LIST-SESSIONS request: %s' % (rep_reason,))
return (False, 'Unexpected server reply, got %u' % (rep_msg,))
sessions = []
while len(m):
sid = m.get_int();
stype = m.get_int();
rid = m.get_int();
cid = m.get_int();
tname = m.get_str();
rname = m.get_str();
sessions.append((sid, stype, rid, cid, tname, rname))
return (True, sessions)
#!/usr/bin/env python from Context import Context from Buffer import Buffer from Shader import Shader from Program import Program from Texture import Texture from Mat4 import Mat4 import OpenGL.GL as gl import ctypes c = Context() b = Buffer(gl.GL_ARRAY_BUFFER, gl.GL_STATIC_DRAW, ctypes.c_int) b.sync() b.push(1) b.push(2) b.push(3) b.push(4) b.push(5) b[12] = 3 assert(b[0]==1) assert(b[1]==2) assert(b[2]==3) assert(b[3]==4) assert(b[4]==5) assert(b[5]==0) assert(b[13]==None)
def info(self, fmt):
m = Buffer()
m.put_int(MUX_C_INFO)
rid = self.rid
self.rid += 1
m.put_int(rid)
m.put_str(fmt)
if self._write_packet(m) is not None:
return (False, 'Can\'t send INFO request')
m = self._read_packet()
rep_msg = m.get_int()
rep_rid = m.get_int()
if rep_rid != rid:
return (False, 'Got unexpected request id %u, expected %u' % (rep_rid, rid))
if rep_msg != MUX_S_RESULT:
rep_reason = m.get_str()
if rep_msg == MUX_S_FAILURE:
return (False, rep_reason)
return (False, 'Expected INFO reply, got %x' % (rep_msg,))
rep_str = m.get_str()
return (True, rep_str)
class Renderer(object):
def __init__(self):
self.program = Program('Shader/Quad.prog')
# Prepare quad geometry
self.vbuffer = Buffer(gl.GL_ARRAY_BUFFER, gl.GL_STATIC_DRAW, Vertex)
self.vbuffer.push(Vertex(-.5, -.5, 0, 0, 1))
self.vbuffer.push(Vertex(-.5, .5, 0, 0, 0))
self.vbuffer.push(Vertex( .5, -.5, 0, 1, 1))
self.vbuffer.push(Vertex( .5, .5, 0, 1, 0))
# Set up vertex array object for mapping structs => shader inputs
vao = (ctypes.c_int * 1)()
gl.glGenVertexArrays(1, vao)
self.vao = vao[0]
stride = ctypes.sizeof(Vertex)
gl.glBindVertexArray(self.vao)
gl.glBindBuffer(gl.GL_ARRAY_BUFFER, self.vbuffer.id)
gl.glEnableVertexAttribArray(Attr.POSITION)
offset = ctypes.c_void_p(0)
gl.glVertexAttribPointer(Attr.POSITION, 3, gl.GL_FLOAT, gl.GL_FALSE, stride, offset)
gl.glEnableVertexAttribArray(Attr.TEXCOORD)
offset = ctypes.c_void_p(12)
gl.glVertexAttribPointer(Attr.TEXCOORD, 2, gl.GL_FLOAT, gl.GL_FALSE, stride, offset)
gl.glBindVertexArray(0)
def render(self, g):
# Render all objects attached to the graphics context.
for quad in g.quad:
self.quad(g, quad)
for text in g.text:
self.text(g, text)
def quad(self, g, quad):
# Enable alpha blending/transparency
self.vbuffer.sync()
gl.glUseProgram(self.program.id)
gl.glEnable(gl.GL_BLEND)
gl.glEnable(gl.GL_DEPTH_TEST)
gl.glBlendFunc(gl.GL_SRC_ALPHA, gl.GL_ONE_MINUS_SRC_ALPHA)
# Bind texture
gl.glUniform1i(self.program.tex, 0)
gl.glBindTexture(gl.GL_TEXTURE_2D, quad.texture.id)
# Set up geometry transforms
worldMatrix = Matrix.scale(quad.width, quad.height, 1)
worldMatrix = Matrix.translate(quad.x, quad.y, 0) * worldMatrix
worldViewProjectionMatrix = g.viewProjectionMatrix * worldMatrix
#worldViewProjectionMatrix = g.viewProjectionMatrix
gl.glUniformMatrix4fv(self.program.worldViewProjectionMatrix, 1, 0,
worldViewProjectionMatrix.data)
# Draw geometry
gl.glBindVertexArray(self.vao)
gl.glDrawArrays(gl.GL_TRIANGLE_STRIP, 0, 4)
gl.glBindVertexArray(0)
def text(self, g, text):
pass
