def __init__(self,
root,
window_size,
audio_loader,
examples_per_file=8,
blacklist_patterns=None,
loaded_files_buffer=10,
file_usages=10):
assert (isinstance(root, str))
if blacklist_patterns is None:
blacklist_patterns = []
self.root = root
self._window_size = window_size
self._audio_loader = audio_loader
self._loaded_files_buffer = loaded_files_buffer
self._file_usages = file_usages
self._index = 0
self.filenames = []
for ext in ['*.wav', '*.mp3']:
self.filenames.extend(
[filename for filename in Path(root).rglob(ext)])
for pattern in blacklist_patterns:
self.filenames = self.blacklist(self.filenames, pattern)
self._loaded_files = dict()
Worker.call(self._populateLoadedFiles).asDaemon.start()
np.random.shuffle(self.filenames)
self._examples_per_file = examples_per_file
def _selectFile(self):
loaded_files = list(self._loaded_files.keys())
while len(list(self._loaded_files.keys())) == 0:
self._loadNewFile()
Worker.call(self._populateLoadedFiles).asDaemon.start()
loaded_files = list(self._loaded_files.keys())
return np.random.choice(loaded_files)
def start_stop(self):
"""
Start the GOL simulation on a separate thread or stop it if it was already running
"""
if not self._gol_model.get_running():
self._gol_model.set_running(True)
self._worker = Worker(self.single_step,
1 / self._gol_model.get_fps())
self._worker.start()
else:
self._worker.stop()
self._gol_model.set_running(False)
def test_many_download(self):
q = Queue()
q2 = Queue()
selects = {}
for i in range(10):
choice = random.choice(list(test_urls.keys()))
if choice in selects.keys():
selects[choice] += 1
else:
selects[choice] = 1
q.put((i, {"url": test_urls[choice]["url"], "dest": test_dest}))
worker = Worker({"wait_task": 1, "wait_retry": 0, "max_retry": 1}, q, q2)
worker.start()
worker.join()
self.assertTrue(q.empty())
for key, value in selects.items():
path = test_urls[key]["path"]
dirname = FileManager.get_dirname(path)
basename = FileManager.get_basename(path)
for i in range(value):
filepath = os.path.join(dirname, "{}_{}".format(i, basename)) if i != 0 else path
self.assertTrue(os.path.exists(filepath))
with open(filepath, "rb") as f:
data = f.read()
self.assertEqual(hashlib.md5(data).hexdigest(), test_urls[key]["md5"])
FileManager.remove_file(filepath)
def test_sftp_download(self):
q = Queue()
q2 = Queue()
q.put((0, {"url": test_urls["sftp_small"]["url"], "dest": test_dest}))
q.put((1, {"url": test_urls["sftp_pkey_small"]["url"], "dest": test_dest,
"key_filename": test_urls["sftp_pkey_small"]["key_filename"],
"passphrase": test_urls["sftp_pkey_small"]["key_filename"]}))
worker = Worker({"wait_task": 0}, q, q2)
worker.start()
worker.join()
self.assertTrue(q.empty())
self.assertTrue(os.path.exists(test_urls["sftp_small"]["path"]))
self.assertTrue(os.path.exists(test_urls["sftp_pkey_small"]["path"]))
with open(test_urls["sftp_small"]["path"], "rb") as f:
data = f.read()
self.assertEqual(hashlib.md5(data).hexdigest(), test_urls["sftp_small"]["md5"])
with open(test_urls["sftp_pkey_small"]["path"], "rb") as f:
data = f.read()
self.assertEqual(hashlib.md5(data).hexdigest(), test_urls["sftp_pkey_small"]["md5"])
FileManager.remove_file(test_urls["sftp_small"]["path"])
FileManager.remove_file(test_urls["sftp_pkey_small"]["path"])
def test_fail_download(self):
works = Queue(maxsize=0)
progresses = Queue(maxsize=0)
for i, key in enumerate(test_urls):
works.put((i + 1, {
"url": test_urls[key]["url"],
"dest": test_dest
}))
for i in range(4):
worker = Worker({
"wait_task": 1,
"wait_retry": 0,
"max_retry": 1
},
works,
progresses,
test_net=True,
name="worker{}".format(i))
worker.setDaemon(True)
worker.start()
visualizer = Visualizer(4, progresses, name="visualizer")
visualizer.start()
works.join()
visualizer.join()
self.assertTrue(works.empty())
self.assertTrue(progresses.empty())
self.assertEqual(visualizer.fail, 4)
self.assertEqual(visualizer.fail, visualizer.task)
self.assertEqual(len(visualizer.results), 4)
self.assertTrue(len(os.listdir(test_dest)) == 0)
def ssh_connect(self, args):
ssh = SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
LOG.debug("ssh connect args: {}".format(args))
dst_addr = args[:2]
LOG.info("Connecting to {}:{}".format(*dst_addr))
try:
ssh.connect(*args, timeout=6)
except socket.error:
raise ValueError('Unable to connect to {}:{}'.format(*dst_addr))
except paramiko.BadAuthenticationType:
raise ValueError('Bad authentication type.')
except paramiko.AuthenticationException:
raise ValueError('Authentication failed.')
except paramiko.BadHostKeyException:
raise ValueError('Bad host key.')
chan = ssh.invoke_shell(term='xterm')
chan.setblocking(0)
worker = Worker(self.loop, ssh, chan, dst_addr)
worker.encoding = self.get_default_encoding(ssh)
return worker
def test_404_download(self):
q = Queue()
q2 = Queue()
q.put((0, {"url": "http://google.com/blah", "dest": test_dest}))
worker = Worker({"wait_task": 0, "wait_retry": 0}, q, q2)
worker.start()
worker.join()
self.assertTrue(q.empty())
q2.get()
self.assertIsInstance(q2.get()[1]["error"], requests.exceptions.HTTPError)
def test_no_dest(self):
q = Queue()
q2 = Queue()
q.put((0, {"url": "url"}))
q.put((1, {"url": "url", "dest": ""}))
worker = Worker({"wait_task": 0}, q, q2)
worker.start()
worker.join()
self.assertTrue(q.empty())
self.assertIsInstance(q2.get()[1]["error"], NoDestinationPathException)
self.assertIsInstance(q2.get()[1]["error"], NoDestinationPathException)
def test_unsupport_protocol(self):
q = Queue()
q2 = Queue()
q.put((0, {"url": "url", "dest": test_dest}))
q.put((1, {"url": "abc://path", "dest": test_dest}))
worker = Worker({"wait_task": 0}, q, q2)
worker.start()
worker.join()
self.assertTrue(q.empty())
q2.get()
self.assertIsInstance(q2.get()[1]["error"], UnsupportedProtocolException)
q2.get()
self.assertIsInstance(q2.get()[1]["error"], UnsupportedProtocolException)
def test_success_download(self):
works = Queue(maxsize=0)
progresses = Queue(maxsize=0)
for i, key in enumerate(test_urls):
works.put((i + 1, {
"url": test_urls[key]["url"],
"dest": test_dest
}))
for i in range(4):
worker = Worker({
"wait_task": 1,
"wait_retry": 0,
"max_retry": 1
},
works,
progresses,
name="worker{}".format(i))
worker.setDaemon(True)
worker.start()
visualizer = Visualizer(4, progresses, name="visualizer")
visualizer.start()
works.join()
visualizer.join()
self.assertTrue(works.empty())
self.assertTrue(progresses.empty())
self.assertEqual(visualizer.success, 4)
self.assertEqual(visualizer.success, visualizer.task)
self.assertTrue(not visualizer.results)
for key in test_urls:
self.assertTrue(os.path.exists(test_urls[key]["path"]))
with open(test_urls[key]["path"], "rb") as f:
data = f.read()
self.assertEqual(
hashlib.md5(data).hexdigest(), test_urls[key]["md5"])
FileManager.remove_file(test_urls[key]["path"])
def test_fail_download(self):
q = Queue()
q2 = Queue()
selects = {}
for i in range(10):
choice = random.choice(list(test_urls.keys()))
if choice in selects.keys():
selects[choice] += 1
else:
selects[choice] = 1
q.put((i, {"url": test_urls[choice]["url"], "dest": test_dest}))
worker = Worker({"wait_task": 1, "wait_retry": 0, "max_retry": 1}, q, q2, test_net=True)
worker.start()
worker.join()
self.assertTrue(q.empty())
self.assertTrue(len(os.listdir(test_dest)) == 0)
def _task_action(self, f):
w = Worker(driver=self.driver,
device=self.device.get("deviceName"),
reset=self.device.get("reset", None))
w.execute(f=f)
saver = tf.train.Saver(max_to_keep=3, var_list=GLOBAL_AC.getVars+[global_step])
GLOBAL_AC.InitializeVariablesFromFile(saver,MODEL_PATH)
progbar = tf.keras.utils.Progbar(None, unit_name='Training',stateful_metrics=["Reward"])
writer = tf.summary.FileWriter(LOG_PATH,graph=sess.graph)
# Create workers
workers = []
for i in range(settings["NumberENV"]):
i_name = 'W_%i' % i # worker name
network = Network(settings["NetworkConfig"],nActions,netConfigOverride,scope=i_name)
Method = GetFunction(settings["Method"])
localNetwork = Method(network,sess,stateShape=dFeatures,actionSize=nActions,scope=i_name,HPs=settings["NetworkHPs"],globalAC=GLOBAL_AC,nTrajs=nTrajs)
localNetwork.InitializeVariablesFromFile(saver,MODEL_PATH)
env,_,_,_ = CreateEnvironment(envSettings,multiprocessing=1)
workers.append(Worker(localNetwork,env,sess,global_step,global_step_next,settings,progbar,writer,MODEL_PATH,saver))
InitializeVariables(sess) #Included to catch if there are any uninitalized variables.
COORD = tf.train.Coordinator()
worker_threads = []
for i,worker in enumerate(workers):
if i==0:
job = lambda: worker.work(COORD,render=args.render)
else:
job = lambda: worker.work(COORD)
t = threading.Thread(target=job)
t.start()
worker_threads.append(t)
COORD.join(worker_threads)
def _usedFilename(self, filename):
self._loaded_files[filename][0] = self._loaded_files[filename][0] + 1
if self._loaded_files[filename][0] >= self._file_usages:
del self._loaded_files[filename]
Worker.call(self._loadNewFile).asDaemon.start()
last_minute = (work_hours + start_hour) * 60 prob_incoming_clients = [0.71, 0.23, 0.05, 0.01] prob_service_type = [0.5, 0.4, 0.1] avg_service_duration = [12, 15, 25] stdev_service_duration = [0.8, 1.0, 3.0] num_of_workers = 4 N = 0 clients_arrival_interval = 5 maxrow = 8 EVENTS = [] ROW = [] WORKERS = [Worker() for _ in range(num_of_workers)] time = start_hour * 60 ncl = 0 N = 0 Nserved = 0 Nwaited = 0 Twaited = 0 EVENTS = [] ev = Event(time, 1, (1, )) EVENTS = fill_events(EVENTS, ev) ev = Event(last_minute, 8, (1, )) EVENTS = fill_events(EVENTS, ev) t = time + int(round(pr.exponential(clients_arrival_interval)))
# Quit if inputs is empty
if not inputs:
raise Exception("No inputs given")
works = Queue(maxsize=0)
progresses = Queue(maxsize=0)
# Put work to Queue
for i, info in enumerate(inputs.values()):
works.put((i + 1, info))
# Setup workers
num_threads = min(config.get("max_worker", 5), len(inputs))
for i in range(num_threads):
worker = Worker(config,
works,
progresses,
name="worker{}".format(i))
worker.setDaemon(True)
worker.start()
# Setup visualizer
visualizer = Visualizer(len(inputs), progresses, name="visualizer")
visualizer.start()
# Wait until works Queue and visualizer finished
works.join()
visualizer.join()
except FileNotFoundError as errf:
print(errf)
except Exception as e:
class MainController:
"""
Class representing the controller of the MVC pattern. It connects to the GUI signals to catch the user interactions
and modify the GOL state model accordingly.
"""
def __init__(self, application: QApplication, main_window: MainWindow,
gol_model: GOLModel):
self._application = application
application.aboutToQuit.connect(self._stop_worker_on_app_closing)
self._main_window = main_window
main_window.connect_to_button_clear(self.clear_grid)
main_window.connect_to_button_load(self.load_custom_pattern)
main_window.connect_to_button_play(self.start_stop)
main_window.connect_to_button_save(self.save_pattern)
main_window.connect_to_button_step(self.single_step)
main_window.connect_to_combo_patterns(self.select_example_pattern)
main_window.connect_to_radio_age(self.toggle_show_cell_age)
main_window.connect_to_slider_speed(self.set_speed)
main_window.grid_widget.connect_to_cell_clicked(self.toggle_cell)
self._gol_model = gol_model
self._worker = None
# Variables for the grid update
self._conv_filter = np.ones((3, 3))
self._conv_filter[1, 1] = 0
def clear_grid(self):
"""
Clear the GOL grid bringing it back to its initial state (depending on the chosen pattern)
:return:
"""
self.select_example_pattern(self._gol_model.get_base_pattern())
self._main_window.show_message_on_status_bar("Grid cleared")
def load_custom_pattern(self):
"""
Load a pattern from a chosen file into the current GOL state
:return:
"""
file_path = QFileDialog.getOpenFileName(
self._main_window,
"Load pattern file",
filter="Pattern File (*.cells)")[0]
if file_path:
self._main_window.reset_combo_patterns()
if self._load_file(file_path):
self._main_window.show_message_on_status_bar("Pattern loaded")
def _load_file(self, file_path: str):
"""
Helper method to load a pattern from a .cells file (plain text format).
:param file_path: Path of the pattern file
:return: False if the file is invalid or the pattern do not fit the current grid, otherwise True
"""
grid_pattern = patterns.read_pattern_file(file_path)
if grid_pattern is None:
self._main_window.show_error_message("Invalid pattern file")
return False
else:
grid_height, grid_width = self._gol_model.get_grid_size()
pattern_height, pattern_width = grid_pattern.shape
# If the pattern is bigger than the grid show an error
if pattern_height > grid_height or pattern_width > grid_width:
self._main_window.show_error_message(
"The loaded pattern is bigger than the available grid")
return False
else:
# Copy the pattern at the center of a blank grid
new_grid = np.zeros(self._gol_model.get_grid_size(), np.uint8)
v_margin = (grid_height - pattern_height) // 2
h_margin = (grid_width - pattern_width) // 2
new_grid[v_margin:v_margin + pattern_height,
h_margin:h_margin + pattern_width] = grid_pattern
self._gol_model.set_grid_as_numpy(new_grid)
return True
def save_pattern(self):
"""
Save the current grid state in a .cells file as a reloadable state
:return:
"""
file_path = QFileDialog.getSaveFileName(
self._main_window,
"Save pattern file",
filter="Pattern File (*.cells)")[0]
if file_path:
patterns.save_pattern_file(file_path,
self._gol_model.get_grid_as_numpy())
self._main_window.show_message_on_status_bar("Pattern saved")
def select_example_pattern(self, pattern_name):
"""
Load a predefined pattern chosen from the provided list
:param pattern_name: The name of the pattern to load (it is the same of its file name)
:return:
"""
self._gol_model.set_base_pattern(pattern_name)
# The selected pattern is the custom one: restart from a blank grid
if pattern_name == "Custom":
new_grid = np.zeros(self._gol_model.get_grid_size(), np.uint8)
self._gol_model.set_grid_as_numpy(new_grid)
else:
file_path = os.path.join(config.ROOT_PATH, config.FOLDER_PATTERNS,
pattern_name + ".cells")
if not self._load_file(file_path):
# Something went wrong during the pattern loading: select the Custom pattern
self._main_window.reset_combo_patterns()
def set_speed(self, speed):
"""
Change the simulation speed
:param speed: The simulation speed in FPS
:return:
"""
self._gol_model.set_fps(speed)
if self._gol_model.get_running():
self._worker.set_wait_time(1 / self._gol_model.get_fps())
def single_step(self):
"""
Performs an update step of the grid applying the Game of Life rules.
Besides calculating dead and living cells at the next time step, it also calculates the age of each cell
(how many time steps they are alive). The age ranges from 0 (dead) to 255 (ancient)
"""
grid_curr_age = self._gol_model.get_grid_as_numpy()
grid_curr_alive = grid_curr_age.astype(np.bool).astype(np.uint8)
# Use convolution to calculate the number of neighbors for each cell
grid_neighbors = ndimage.convolve(grid_curr_alive,
self._conv_filter,
mode="constant",
cval=0)
# Calculate which cells to give birth: a dead cell is born when it has exactly three neighbors
grid_newborns = grid_neighbors == 3
grid_newborns = np.logical_and(grid_newborns,
np.logical_not(grid_curr_alive))
# Calculate which cells survive: a living cell survive when it has two or three neighbors
grid_survived = np.logical_and(grid_neighbors >= 2,
grid_neighbors <= 3)
grid_survived = np.logical_and(grid_survived, grid_curr_alive)
# Calculate the living cells at the next step merging survived and newborn cells
grid_next = np.logical_or(grid_newborns,
grid_survived).astype(np.uint8)
# Calculate the age of the cells in the new grid
grid_next = grid_curr_age * grid_next + grid_next
np.putmask(
grid_next, grid_curr_age == 255, 255
) # This is necessary to avoid the overflow handling by numpy and cap the array values to 255
self._gol_model.set_grid_as_numpy(grid_next)
def start_stop(self):
"""
Start the GOL simulation on a separate thread or stop it if it was already running
"""
if not self._gol_model.get_running():
self._gol_model.set_running(True)
self._worker = Worker(self.single_step,
1 / self._gol_model.get_fps())
self._worker.start()
else:
self._worker.stop()
self._gol_model.set_running(False)
def _stop_worker_on_app_closing(self):
if self._worker is not None:
self._worker.stop()
def toggle_show_cell_age(self, show_cell_age: bool):
self._gol_model.set_show_cell_age(show_cell_age)
def toggle_cell(self, cell_coord: tuple):
"""
Toggle the state of the specified cell
:param cell_coord: A tuple containing the cell coordinates as (row, column)
"""
grid = self._gol_model.get_grid_as_numpy()
row, col = cell_coord
grid[row, col] = 0 if grid[row, col] else 1
self._gol_model.set_grid_as_numpy(grid)
import time
import json
import gmqtt
from utils.utils import run_event_loop, STOP, init_client
from utils.consts import WORKER_REGISTRED_TOPIC, WORKER_REGISTRATION_TOPIC, WORKER_UNREGISTER_TOPIC, \
WORKER_RESULT_TOPIC, BALANCER_WORKER_TOPIC
from utils.worker import Worker
worker = Worker()
async def on_message(client, topic, payload, qos, properties):
if topic == WORKER_REGISTRED_TOPIC:
data = json.loads(payload.decode('utf-8'))
worker_hex = data.get("worker_hex")
worker_num = data.get('worker_num')
if not worker.is_registered() and worker.worker_hex == worker_hex:
worker.register(worker_num)
client.subscribe(f'{worker.balancer_topic}', qos=1)
elif topic == worker.balancer_topic and worker.is_registered():
print(f'Worker {worker.number}. Publish.',
f"Topic: '{worker.result_topic}'.", 'Payload:', payload)
client.publish(worker.result_topic, payload, qos=1)
return 0
async def main(broker_host, token):
will_message = gmqtt.Message(WORKER_UNREGISTER_TOPIC,
worker.worker_hex,
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.
import os, sys
# Add the wmg_agent dir to the system path.
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
# No need for argparse. All settings are contained in the spec file.
num_args = len(sys.argv) - 1
if num_args != 1:
print('run.py accepts a single argument specifying the runspec.')
exit(1)
# Read the runspec.
from utils.spec_reader import SpecReader
SpecReader(sys.argv[1])
# Execute the runspec.
from utils.worker import Worker
worker = Worker()
worker.execute()