def parse_config(self, config_file: str):
# extended interpolation for complex variables
config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation())
config.read(config_file)
# first part of the config file is a Global section
GlobalConfig = namedtuple('GlobalConfig', config.options(config.sections()[0]))
global_config = GlobalConfig(*config[config.sections()[0]].values())
# second and further sections of the config file are Sessions sections
SessionConfig = namedtuple('SessionConfig', config.options(config.sections()[1]))
sessions = [session for session in config.sections() if str(session).split('_')[0] == 'SESSION']
sessions_values = [config[session].values() for session in sessions]
session_config = [SessionConfig(*value) for value in sessions_values]
# return a config tuple
return (global_config, session_config)
def hwpcontrol(confilename):
""" Run the HWP control
"""
# Load config file
config = configparser.ConfigParser()
config.read(confilename)
# Make interfaces and agents
logctrl = LoggerControl(config, 'Log')
inusr = InterUser(config, 'User')
insock = InterSocket(config, 'Socket')
inweb = InterWeb(config, 'Web')
opwat = WatchdogOper(config, 'Watch')
agconf = ConfigAgent(config, 'Conf')
agresp = AgentParent(config, 'Echo')
aggal = GalilAgent(config, 'Galil')
readou = ReadoutAgent(config, 'Readout')
# Register agents with interfaces
for agent in [agresp, aggal, agconf, opwat, readou]:
inusr.addagent(agent)
insock.addagent(agent)
inweb.addagent(agent)
opwat.addagent(agent)
insock.addagent(agent)
# Run items as threads (as daemons such that they shut down on exit)
threads = {}
for item in [
logctrl, agresp, aggal, agconf, opwat, inusr, insock, inweb, readou
]:
thread = threading.Thread(target=item)
thread.daemon = True
thread.start()
threads[item.name] = thread
# Wait and do some stuff
time.sleep(2)
agresp.comqueue.put(('Do It', inusr.respqueue))
# Join with User Interface thread
threads['User'].join()
def getvalue(self, key, value):
config = configparser.ConfigParser()
config.read(self.pathvar)
smtp_para = config[key][value]
return smtp_para
def getpassword(self):
config = configparser.ConfigParser()
config.read(self.pathvar)
smtp_password = config['smtp.cloud']['password']
return str(smtp_password)
def getSMTPtoken(self):
config = configparser.ConfigParser()
config.read(self.pathvar)
token = config['smtp.cloud']['token']
return token
def getSMTPtoAddr(self):
config = configparser.ConfigParser()
config.read(self.pathvar)
smtp_toaddress = config['smtp.cloud']['toAddr']
return str(smtp_toaddress)
def getSMTPfromAddr(self):
config = configparser.ConfigParser()
config.read(self.pathvar)
smtp_from = config['smtp.cloud']['fromAddr']
return str(smtp_from)
def getSMTPport(self):
config = configparser.ConfigParser()
config.read(self.pathvar)
smtp_port = config['smtp.cloud']['port']
return str(smtp_port)
def getSMTPhost(self):
config = configparser.ConfigParser()
config.read(self.pathvar)
smtp_host = config['smtp.cloud']['host']
return str(smtp_host)
class ConfigUtil(object):
'''
classdocs
'''
config = configparser.ConfigParser()
dirname = os.path.abspath(
'A:\git-repos\workspace\iot-device\config\ConnectedDevicesConfig.props'
)
config.read(dirname)
pathvar = dirname
# config['smtp.cloud']['port']
#
def __init__(self):
return
def getdirectory(self):
return self.pathvar
def getSMTPhost(self):
config = configparser.ConfigParser()
config.read(self.pathvar)
smtp_host = config['smtp.cloud']['host']
return str(smtp_host)
def getSMTPport(self):
config = configparser.ConfigParser()
config.read(self.pathvar)
smtp_port = config['smtp.cloud']['port']
return str(smtp_port)
def getSMTPfromAddr(self):
config = configparser.ConfigParser()
config.read(self.pathvar)
smtp_from = config['smtp.cloud']['fromAddr']
return str(smtp_from)
def getSMTPtoAddr(self):
config = configparser.ConfigParser()
config.read(self.pathvar)
smtp_toaddress = config['smtp.cloud']['toAddr']
return str(smtp_toaddress)
def getSMTPtoken(self):
config = configparser.ConfigParser()
config.read(self.pathvar)
token = config['smtp.cloud']['token']
return token
def getpassword(self):
config = configparser.ConfigParser()
config.read(self.pathvar)
smtp_password = config['smtp.cloud']['password']
return str(smtp_password)
def getvalue(self, key, value):
config = configparser.ConfigParser()
config.read(self.pathvar)
smtp_para = config[key][value]
return smtp_para
def loadconfig(self):
# path_var="A:\git-repos\workspace\iot-device\config\ConnectedDevicesConfig.props"
load = os.path.isfile(self.pathvar)
if load is True:
return True
else:
return False
def hasconfig(self, key, value):
load = self.getvalue(key, value)
if load is not None:
return True
else:
return False
def run_tasks(
task_to_run: Union[str, List[str]] = None,
workspace: str = None,
access_key: str = None,
num_tasks: int = None,
low_pri_nodes: int = 9,
dedicated_nodes: int = 1,
pool_name: str = None,
job_name: str = None,
use_service_principal: bool = False,
vm_sku: str = None,
config_file: str = user_config,
log_iterations: Union[bool, str] = False,
workdir: str = "/src",
image_name: str = None,
image_version: str = None,
platform: str = None,
show_price: bool = True,
wait_time: int = 10,
time_delay: int = 0,
app_insights: bool = True,
):
"""Run simulators in Azure Batch.
Parameters
----------
task_to_run : Union[str, List[str]], optional
Task(s) to run, by default None, which forces user to enter (a single task)
If you want to run distinct tasks provide a List of tasks to run.
In the case whena list is provided, note that len(task_to_run) must equal num_tasks.
workspace : str, optional
[description], by default None
access_key : str, optional
[description], by default None
num_tasks : str, mandatory
Number of simulators to run as separate tasks
low_pri_nodes : int, optional
Number of low priority to create in pool, by default 9
dedicated_nodes : int, optional
Number of dedicated to create in pool, by default 1
pool_name : str, optional
Name of the pool to create for simulation scaling, by default None
job_name : str, optional
Job name for simulation scaling job, by default None
use_service_principal : bool, optional
[description], by default False
vm_sku : str, optional
[description], by default None
config_file : str, optional
Location of configuration file containing ACR and Batch parameters, by default user_config
log_iterations : Union[bool, str], optional
[description], by default False
workdir : str, optional
[description], by default None
image_name : str, optional
[description], by default None
image_version : str, optional
[description], by default None
platform : str, optional
[description], by default None
show_price : bool, optional
[description], by default True
wait_time : int, optional
[description], by default 10
time_delay: int, optional
time to delay next task, by default 0
app_insights: bool, optional
whether to use application_insights to monitor azure batch pools
"""
if not os.path.exists(config_file):
raise ValueError(f"No configuration file found at {config_file}")
config = configparser.ConfigParser()
config.read(config_file)
platform = config["ACR"]["PLATFORM"]
if not task_to_run:
task_to_run = input(
"Please enter task to run from container (e.g., python main.py): ")
if not num_tasks:
num_tasks = input("Number of simulators to run as tasks on Batch: ")
if type(task_to_run) == List:
assert len(task_to_run) == int(
num_tasks), "Number of tasks must equal number of tasks to run"
total_nodes = low_pri_nodes + dedicated_nodes
tasks_per_node = max(ceil(float(num_tasks) / total_nodes), 1)
if platform:
logger.info(
f"Writing {platform} to {config_file}'s platform argument.")
config["ACR"]["PLATFORM"] = platform
config["POOL"]["NUM_TASKS"] = str(num_tasks)
config["POOL"]["TASKS_PER_NODE"] = str(tasks_per_node)
config["POOL"]["LOW_PRI_NODES"] = str(low_pri_nodes)
config["POOL"]["DEDICATED_NODES"] = str(dedicated_nodes)
logger.info(
f"Requested pool size low-priority nodes: {low_pri_nodes}, dedicated nodes: {dedicated_nodes}"
)
if not vm_sku:
vm_sku = input(
"What VM Name / SKU do you want to use? (if you don't know leave this empty): "
)
if vm_sku.lower() == "none" or vm_sku.lower() == "":
if tasks_per_node <= 8:
vm_sku = "Standard_E2s_v3"
elif tasks_per_node <= 16:
vm_sku = "Standard_E8s_v3"
elif tasks_per_node <= 32:
vm_sku = "Standard_E16s_v3"
elif tasks_per_node <= 75:
vm_sku = "Standard_E32s_v3"
elif tasks_per_node > 75:
vm_sku = "Standard_E64s_v3"
logger.info(
"Running {0} tasks per node, please check if VM Size is compatible"
.format(tasks_per_node))
logger.warning(
f"Auto-selecting [bold green]{vm_sku}[/bold green] for your pool based on calculated tasks per node.",
)
if tasks_per_node > 8:
logger.warning(
f"You have asked to run {tasks_per_node} tasks per node! You also did not provide a VM SKU. Based on this we selected {vm_sku} as your VM, which may be costly! Please confirm with [bold magenta]yes[/bold magenta] in the next prompt or choose a different VM. Our calculator https://share.streamlit.io/akzaidi/bonsai-cost-calculator/main/st-azure-pricing.py may be helpful for your calculations.",
)
confirm_sku = input(
f"Confirm with yes if you want to use {vm_sku} for your pool, or type in a new VM SKU: "
)
if confirm_sku.lower() != "yes":
vm_sku = confirm_sku
config["POOL"]["VM_SIZE"] = vm_sku
config["POOL"]["TASK_START_DIR"] = workdir
if not image_name:
image_name = config["ACR"]["IMAGE_NAME"]
if not image_version:
image_version = config["ACR"]["IMAGE_VERSION"]
if not pool_name:
config["POOL"]["POOL_ID"] = image_name + "pool" + str(total_nodes)
else:
config["POOL"]["POOL_ID"] = pool_name
if not job_name:
config["POOL"]["JOB_NAME"] = image_name + "job" + str(num_tasks)
else:
config["POOL"]["JOB_NAME"] = job_name
if platform.lower() == "windows":
win_config = configparser.ConfigParser()
win_config.read(windows_config)
config["POOL"]["PUBLISHER"] = win_config["POOL"]["PUBLISHER"]
config["POOL"]["OFFER"] = win_config["POOL"]["OFFER"]
config["POOL"]["SKU"] = win_config["POOL"]["SKU"]
config["POOL"]["VERSION"] = win_config["POOL"]["VERSION"]
config["POOL"]["AGENT_SKU"] = win_config["POOL"]["AGENT_SKU"]
config["POOL"]["PYTHON_EXEC"] = "python"
config["ACR"]["IMAGE_NAME"] = image_name
config["ACR"]["IMAGE_VERSION"] = image_version
with open(config_file, "w") as conf_file:
config.write(conf_file)
batch_run = AzureBatchContainers(
config_file=config_file,
service_principal=use_service_principal,
workspace=workspace,
access_key=access_key,
)
if image_name:
batch_run.config["ACR"]["IMAGE_NAME"] = image_name
if image_version:
batch_run.config["ACR"]["IMAGE_VERSION"] = image_version
if type(log_iterations) == str:
log_iterations = bool(strtobool(log_iterations))
batch_run.batch_main(
command=task_to_run,
log_iterations=log_iterations,
workdir=workdir,
show_price=show_price,
wait_time=wait_time,
delay_next=time_delay,
app_insights=app_insights,
)