def __init__(self, config_file, action, stage=None):
super(ProjectHandler, self).__init__()
self.config_file = config_file
self.stage = stage
self.action = action
self.stage_actions = [
'submit', 'clean', 'status', 'check', 'statistics'
]
self.project_actions = ['check', 'clean']
if stage is None and self.action not in self.project_actions:
raise Exception("Action {} not available".format(self.action))
elif stage is not None and self.action not in self.stage_actions:
raise Exception("Action {} not available".format(self.action))
# Build the configuration class:
self.config = ProjectConfig(config_file)
# Make sure the stage requested is in the file:
if stage is not None and stage not in self.config.stages:
raise Exception(
'Stage {0} not in configuration file.'.format(stage))
# Create the work directory:
self.project_work_dir = self.config['top_dir'] + '/work/'
if stage is not None:
self.stage_work_dir = self.project_work_dir + stage + '/'
def main():
project = ProjectConfig('example_project.yml')
project.larsoft().setup_larsoft()
db_file = project['top_dir'] +'/work/' + project['name'] + '.db'
a = JobRunner(project = project, stage=project.stage('generation'))
a.prepare_job()
a.run_job(db_util = DBUtil(db_file))
return
def part1():
# parse the configuration file:
config_file = '/home/cadams/Harvard-Production/yml-configs/bnb_plus_cosmics-multistage-test.yml'
config = ProjectConfig(config_file)
# Print out the stages to run over:
for name, stage in config.stages.iteritems():
# Create a handler to check files and such
handler = ProjectHandler(config_file, action='status', stage=name)
handler.build_directory()
handler.first_stage_check()
db = handler.project_db
print('Faking stage {0}'.format(stage.name))
print(' Input specified? {0}'.format(stage.has_input()))
#Make sure output directory exists:
out_dir = stage.output_directory()
handler.make_directory(out_dir)
# Generate fake output for this stage:
for i in range(stage.n_jobs()):
for fcl in stage.fcl():
fcl = os.path.basename(fcl)
_f = '/{0}_fake_{1}.root'.format(fcl, i)
# Input
if stage.has_input():
input_files, locations = stage.get_next_files(1, db)
print('Input files are: \n {0}'.format(input_files))
# Processing
# Nothing actually happens
# Output
with open(out_dir + _f, 'w') as file:
pass
# Declare the file to the database:
db.declare_file(filename=_f,
dataset=stage.output_dataset(),
location=out_dir,
stage=name,
status=0,
nevents=10,
ftype=0)
# Mark the input files as consumed:
if stage.has_input():
stage.finalize(input_files, db)
def main(config_file, stage):
print("Creating Project Config Object")
project = ProjectConfig(config_file)
print("Config created, setup software ...")
project.software().setup()
runner_class = RunnerTypes()[project.software()['type']]
runner = runner_class(project = project, stage=project.stage(stage))
print("Preparing job ...")
runner.prepare_job()
job_id = "{0}_{1}".format(os.environ['SLURM_ARRAY_JOB_ID'], os.environ['SLURM_ARRAY_TASK_ID'])
print("Job ID is {0}".format(job_id))
print("Running job ...")
runner.run_job(job_id)
return
def load(window):
"""Intelligently guess the appropriate .ensime file location for the
given window. Load the .ensime and parse as s-expression.
Return: (inferred project root directory, config sexp)
"""
for f in _locations(window):
try:
conf = ProjectConfig(f)
return conf
except Exception:
exc_type, exc_val, _ = os.sys.exc_info()
raise BadEnsimeConfig(
"""Ensime has failed to parse the .ensime configuration file at
{loc} because ofthe following error:
{typ} : {val}""".format(loc=str(f), typ=str(exc_type), val=str(exc_val)))
raise DotEnsimeNotFound(
errno.ENOENT,
"""Ensime has failed to find a .ensime file within this project.
Create a .ensime file by running'sbt ensimeConfig' or equivalent for your build tool.
We looked at """, window.folders())
def config(conffile):
return ProjectConfig(CONFROOT.join(conffile).strpath)
except Exception as e:
print(f'Error with reading from {self.get_name()}: {e}')
#print('Reconnecting...')
#self.disconnect()
#raise ConnectionError
def write(self, message):
try:
#message = str(message)
#byte_msg: bytes = str.encode(message + '\n')
json_str = json.dumps(message)
byte_msg = bytes(json_str, encoding='utf-8')
self.client.sendto(byte_msg, self.addr)
print(f'Sent to PC: {message}')
except Exception as e:
print(f'Error with writing {message} to {self.get_name()}: {e}')
#print('Reconnecting...')
#self.disconnect()
raise ConnectionError
if __name__ == '__main__':
server = PcConn(ProjectConfig(default=False))
server.connect()
Robot_Position = '{"MDP15":"SENSORS","SENSORS":"0;0;0;0;0;0"}' #to algo
RP = json.loads(Robot_Position)
while True:
server.write(RP)
server.read()
time.sleep(2)
def test_fails_when_given_invalid_config():
badconf = path.local(__file__).dirpath() / 'resources' / 'broken.conf'
with raises(sexpdata.ExpectClosingBracket):
ProjectConfig(badconf.strpath)
# coding: utf-8
from py import path
from pytest import raises
import sexpdata
from config import ProjectConfig
confpath = path.local(__file__).dirpath() / 'resources' / 'test.conf'
config = ProjectConfig(confpath.strpath)
def test_parses_dot_ensime():
assert config.get('scala-version') == '2.11.8'
assert config.get('list') == ["a", "b", "c", "d"]
assert len(config['nest']) == 2
assert config['nest'][0]['id'] == {'config': 'conf1', 'name': 'nested1'}
assert config['nest'][0]['targets'] == ['abc', 'xyz']
def test_is_immutable():
with raises(TypeError) as excinfo:
config['scala-version'] = 'bogus'
assert 'does not support item assignment' in str(excinfo.value)
def test_is_dict_like():
assert set(config.keys()) == set(['name', 'scala-version', 'list', 'nest'])
assert len(config) == 4
def build_script(engine, script, configuration, buildtype, build, platform,
clean, automated, buildexplicit):
"""
The Main call for build script execution.
:param engine: The desired engine path, absolute or relative.
:param script: The Project Script which defines the projects paths, build steps, and extra information.
:param configuration: Build configuration, e.g. Shipping
:param buildtype: Which type of build are you trying to create? Editor OR Package?
:param build: Which build steps to execute?
:param platform: Which platform to build for?
:param clean: Causes all actions to consider cleaning up their workspaces before executing their action.
:param automated: Configures the builder to recognize this build as being done by continuous integration and should
not manipulate the system environment.
:param buildexplicit: Should the build system only build what is requested? This prevents convienience cases like
the package build building the editor before trying to package. By setting this to true, it is
expected that the user has setup the proper state before building.
"""
# Fixup for old build type 'Game'.
if buildtype == 'Game':
buildtype = 'Editor'
global is_automated
if automated:
is_automated = automated
# Ensure Visual Studio is installed
if get_visual_studio_version() not in [2015, 2017]:
print_error('Cannot run build, valid visual studio install not found!')
return False
if not os.path.isfile(script):
error_exit('No build script defined! Use the -s arg', not is_automated)
with open(script, 'r') as fp:
try:
script_json = json.load(fp)
except Exception as jsonError:
error_exit('Build Script Syntax Error:\n{}'.format(jsonError),
not is_automated)
return
config = ProjectConfig(configuration, platform, False, clean, automated)
if not config.load_configuration(script_json, engine, buildexplicit):
error_exit('Failed to load configuration. See errors above.',
not config.automated)
print_title('Unreal Project Builder')
if config.automated:
click.secho('\nAutomated flag set!')
# Ensure the engine exists and we can build
if not buildexplicit:
ensure_engine(config, engine)
click.secho('\nProject File Path: {}\nEngine Path: {}'.format(
config.uproject_dir_path, config.UE4EnginePath))
# Ensure the unreal header tool exists. It is important for all Unreal projects
if not buildexplicit:
if not os.path.isfile(
os.path.join(config.UE4EnginePath,
'Engine\\Binaries\\Win64\\UnrealHeaderTool.exe')):
b = Build(config, build_name='UnrealHeaderTool')
if not b.run():
error_exit(b.error, not config.automated)
# Build required engine tools
if config.should_build_engine_tools and not buildexplicit:
clean_revert = config.clean
if buildtype == "Package":
config.clean = False # Don't clean if packaging, waste of time
b = Build(config, build_names=config.build_engine_tools)
if not b.run():
error_exit(b.error, not config.automated)
config.clean = clean_revert
# If a specific set of steps if being requested, only build those
if build != '':
steps = Buildsteps(config, steps_name=build)
if not steps.run():
error_exit(steps.error, not config.automated)
else:
if buildtype == "Editor":
if config.editor_running:
error_exit(
'Cannot build the Editor while the editor is running!',
not config.automated)
if 'game_editor_steps' in config.script:
steps = Buildsteps(config, steps_name='game_editor_steps')
if not steps.run():
error_exit(steps.error, not config.automated)
elif 'editor_steps' in config.script:
steps = Buildsteps(config, steps_name='editor_steps')
if not steps.run():
error_exit(steps.error, not config.automated)
else:
b = Build(config,
build_name='{}Editor'.format(config.uproject_name))
if not b.run():
error_exit(b.error, not config.automated)
elif buildtype == "Package":
# We need to build the editor before we can run any cook commands. This seems important for blueprints
# probably because it runs the engine and expects all of the native class RTTI to be up-to-date to be able
# to compile the blueprints. Usually you would be starting a package build from the editor, so it makes
# sense. Explicit builds ignore this however.
if not buildexplicit:
b = Build(config,
build_name='{}Editor'.format(config.uproject_name))
if not b.run():
error_exit(b.error, not config.automated)
if 'package_steps' in config.script:
steps = Buildsteps(config, steps_name='package_steps')
if not steps.run():
error_exit(steps.error, not config.automated)
else:
package = Package(config)
if not package.run():
error_exit(package.error, not config.automated)
print_action('SUCCESS!')
if not config.automated:
click.pause()
def move_files_to_neutrino():
pr = ProjectReader()
dr = DatasetReader()
remote_host = '*****@*****.**'
local_top_directory = '/n/holylfs02/LABS/guenette_lab/data/NEXT/NEXTNEW/MC/OtherForTransfer/'
remote_top_directory = '/lustre/neu/data4/NEXT/NEXTNEW/MC/Other/NEXUS_NEXT_v1_03_01/'
# We want to copy, for every project here (76 + Xenon, eventually)
# The files, configurations, and logs.
# The remote directory structure should be:
# remote_top_directory/nexus/{element}/{region}/output
# remote_top_directory/nexus/{element}/{region}/log
# remote_top_directory/nexus/{element}/{region}/config
# The config files, logs, and output all live in the same directory. So, what this script does
# is to generate a list files to/from for transfering. It creates symbolic links to
# the local files in the right directory structure as needed on neutrinos.
# The way this is done is to generate a file that will be used for creating symlinks
# A process is spawned to make the links
# Finally, a job is submitted to do the rsync command.
with open('transfer_protocol.txt', 'w') as _trnsf:
for isotope in isotopes:
element = isotope.split('-')[0]
for region in regions:
yml_name = '{element}/{region}/nexus_{element}_{region}.yml'.format(element=element, region=region)
# Read in the yml file:
pc = ProjectConfig(yml_name)
stage = pc.stage(element)
dataset = stage.output_dataset()
output_dir = stage.output_directory()
# Get the log files, config files, and output files
config_match = '/*config.mac'
init_match = '/*init.mac'
log_match = '/*.log'
output_file_list = dr.list_file_locations(dataset)
for _file in output_file_list:
_file = _file[0]
base = os.path.basename(_file)
destination = "{top}/nexus/{element}/{region}/output/{base}".format(
top = local_top_directory,
element = element,
region = region,
base = base
)
trnsf_str = "{}\t{}\n".format(_file, destination)
_trnsf.write(trnsf_str)
directory = os.path.dirname(_file)
# Get the config files:
init = glob.glob(directory + init_match)[0]
base = os.path.basename(init)
destination = "{top}/nexus/{element}/{region}/config/{base}".format(
top = local_top_directory,
element = element,
region = region,
base = base
)
trnsf_str = "{}\t{}\n".format(init, destination)
_trnsf.write(trnsf_str)
cfg = glob.glob(directory + config_match)[0]
base = os.path.basename(cfg)
destination = "{top}/nexus/{element}/{region}/config/{base}".format(
top = local_top_directory,
element = element,
region = region,
base = base
)
trnsf_str = "{}\t{}\n".format(cfg, destination)
_trnsf.write(trnsf_str)
# Get the log files:
logs = glob.glob(directory + log_match)
for log in logs:
base = os.path.basename(log)
destination = "{top}/nexus/{element}/{region}/log/{base}".format(
top = local_top_directory,
element = element,
region = region,
base = base
)
trnsf_str = "{}\t{}\n".format(log, destination)
_trnsf.write(trnsf_str)
break
print "Done making transfer list, creating symbolic links"
with open('transfer_protocol.txt', 'r') as _trnsf:
for line in _trnsf.readlines():
original, destination = line.rstrip('\n').split('\t')
destdir = os.path.dirname(destination)
try:
os.makedirs(destdir)
except:
pass
try:
os.symlink(original, destination)
except:
pass
print "Beginning file transfer."
with cd(local_top_directory):
command = ['rsync', '-rvL', 'nexus', '[email protected]:/lustre/neu/data4/NEXT/NEXTNEW/MC/Other/NEXUS_NEXT_v1_03_01/']
proc = subprocess.Popen(command,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE,
env = dict(os.environ))
retval=proc.poll()
# the loop executes to wait till the command finish running
stdout=''
stderr=''
while retval is None:
time.sleep(1.0)
# while waiting, fetch stdout (including STDERR) to avoid crogging the pipe
for line in iter(proc.stdout.readline, b''):
stdout += line
for line in iter(proc.stderr.readline, b''):
stderr += line
# update the return value
retval = proc.poll()
return_code = proc.returncode
if return_code != 0:
raise Exception("Failed")
else:
print stdout
def main(info_only):
# if action not in ['--submit', '--status', '--check']:
# raise Exception("action not supported.")
pr = ProjectReader()
dr = DatasetReader()
# Get the list of datasets that are in the production database:
datasets = pr.list_datasets()
datasets = [ ds for tupl in datasets for ds in tupl]
for isotope in isotopes:
element = isotope.split('-')[0]
for region in regions:
yml_name = '{element}/{region}/nexus_{element}_{region}.yml'.format(element=element, region=region)
# Read in the yml file:
pc = ProjectConfig(yml_name)
stage = pc.stage(element)
# print stage.output_dataset()
# First, check if this project is in the database:
if stage.output_dataset() in datasets:
# Check the output of this dataset.
# From the yml, get the number off jobs and the events per job:
total_events_submitted = stage.total_output_events()
total_events_produced = dr.sum(
dataset=stage.output_dataset(),
target='nevents',
type=0)
n_jobs = stage.n_jobs()
# From the database figure out how many jobs succeeded,
# and how many events were produced:
n_jobs_succeeded = dr.get_n_successful_jobs(stage.output_dataset())
# print "For dataset {}, {} of {} jobs completed".format(
# stage.output_dataset(),
# n_jobs_succeeded, n_jobs)
# print " {} of {} events passed the selection".format(
# total_events_produced,
# total_events_submitted)
# If the number of jobs completed equals the number of jobs submitted,
# it's done.
if n_jobs_succeeded >= 0.95*n_jobs:
print bcolors.OKGREEN + "{} - {} SUCCESS".format(element, region) + bcolors.ENDC
insertion_sql = '''
INSERT INTO next_new_bkg_summary(dataset, element, region, n_simulated, n_passed, events_per_job, n_jobs_submitted, n_jobs_succeeded)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
'''
conn = connect()
curr = conn.cursor()
tupl = (stage.output_dataset(), element, region, int(total_events_submitted), int(total_events_produced), int(stage.events_per_job()) int(n_jobs), int(n_jobs_succeeded))
curr.execute(insertion_sql, tupl)
conn.commit()
conn.close()
elif n_jobs_succeeded == 0:
print bcolors.WARNING + "{} - {} RESUBMIT".format(element, region) + bcolors.ENDC
# clean and resubmit
if not info_only:
ph = ProjectHandler(yml_name, action='clean', stage=element)
ph.act()
ph = ProjectHandler(yml_name, action='submit', stage=element)
ph.act()
else:
print bcolors.FAIL + "{} - {} MAKEUP NEEDED".format(element, region) + bcolors.ENDC
# Doing makeup jobs, just report it:
else:
# Need to submit it for the first time.
print bcolors.OKBLUE + "{} - {} SUBMITTING".format(element, region) + bcolors.ENDC
if not info_only:
ph = ProjectHandler(yml_name, action='submit', stage=element)
ph.act()
def build_script(engine, script, configuration, buildtype, build, platform, clean):
"""
The Main call for build script execution.
:param engine: The desired engine path, absolute or relative.
:param script: The Project Script which defines the projects paths, build steps, and extra information.
:param configuration: Build configuration, e.g. Shipping
:param buildtype: Which type of build are you trying to create? Editor OR Package?
:param build: Which build steps to execute?
:param platform: Which platform to build for?
:param clean: Causes all actions to consider cleaning up their workspaces before executing their action.
"""
# Fixup for old build type 'Game'.
if buildtype == 'Game':
buildtype = 'Editor'
# Ensure Visual Studio is installed
if get_visual_studio_version() not in [2015, 2017]:
print_error('Cannot run build, valid visual studio install not found!')
return False
if not os.path.isfile(script):
error_exit('No build script defined! Use the -s arg')
with open(script, 'r') as fp:
try:
script_json = json.load(fp)
except Exception as jsonError:
error_exit('Build Script Syntax Error:\n{}'.format(jsonError))
return
config = ProjectConfig(configuration, platform, False, clean)
if not config.load_configuration(script_json, engine, False):
error_exit('Failed to load configuration. See errors above.')
print_title('Unreal Project Builder')
build_meta = BuildMeta('project_build_meta')
if "meta" in config.script:
build_meta.insert_meta(**config.script["meta"])
# Ensure the engine exists and we can build
ensure_engine(config, engine)
click.secho('\nProject File Path: {}\nEngine Path: {}'.format(config.uproject_dir_path, config.UE4EnginePath))
# Ensure the unreal header tool exists. It is important for all Unreal projects
if not os.path.isfile(os.path.join(config.UE4EnginePath, 'Engine\\Binaries\\Win64\\UnrealHeaderTool.exe')):
b = Build(config, build_name='UnrealHeaderTool')
if not b.run():
error_exit(b.error)
# Build required engine tools
clean_revert = config.clean
if buildtype == "Package":
config.clean = False # Don't clean if packaging, waste of time
for tool_name in config.build_engine_tools:
b = Build(config, build_name=tool_name)
if not b.run():
error_exit(b.error)
config.clean = clean_revert
# If a specific set of steps if being requested, only build those
if build != '':
run_build_steps(config, build_meta, build, True)
else:
# Ensure engine is built
if not config.editor_running:
clean_revert = config.clean
if buildtype == "Package":
config.clean = False # Don't clean if packaging, waste of time
b = Build(config, build_name='UE4Editor')
if not b.run():
error_exit(b.error)
config.clean = clean_revert
else:
print_warning('Skipping engine build because engine is running!')
run_build_steps(config, build_meta, 'pre_build_steps')
if buildtype == "Editor":
if config.editor_running:
print_warning('Cannot build the Editor while the editor is running!')
click.pause()
sys.exit(1)
if 'game_editor_steps' in config.script:
run_build_steps(config, build_meta, 'game_editor_steps')
elif 'editor_steps' in config.script:
run_build_steps(config, build_meta, 'editor_steps')
else:
b = Build(config, build_name='{}Editor'.format(config.uproject_name))
if not b.run():
error_exit(b.error)
elif buildtype == "Package":
if 'package_steps' in config.script:
run_build_steps(config, build_meta, 'package_steps')
else:
package = Package(config)
if not package.run():
error_exit(package.error)
run_build_steps(config, build_meta, 'post_build_steps')
build_meta.save_meta()
print_action('SUCCESS!')
click.pause()
data = str(data.decode('utf-8')).strip()
# wrap data in a JSON format
if data[0].isnumeric():
data_dict = {'MDP15': 'SENSORS', 'SENSORS': data}
elif data == 'MC':
data_dict = {'MDP15': 'MC'}
else:
data_dict = {'MDP15': 'STATUS', 'STATUS': data}
print(f'Received from Arduino: {data}')
return self.format_data(data_dict)
except Exception as e:
print(f'Error with reading from {self.get_name()}: {e}')
#print('Reconnecting...')
#self.disconnect()
raise ConnectionError
def disconnect(self):
if self.conn:
self.conn.close()
print('Terminating serial socket..')
self._connected = False
if __name__ == '__main__':
ar = ArduinoConn(ProjectConfig(USB_PORT='COM3'))
ar.connect()
# Put job list into queue
def create_jobs(servers):
for s in servers:
thread_queue.put(s)
thread_queue.join() # blocks until task_done is called
# Do next job that is in the queue
def work():
s = thread_queue.get()
s.start()
if __name__ == '__main__':
server_list = []
config = ProjectConfig(default=False)
bt_server = ProducerConsumer(BluetoothConn(config))
usb_server = ProducerConsumer(ArduinoConn(config))
pc_server = ProducerConsumer(PcConn(config))
cam_server = ProducerConsumer(PiHttpStream(config))
pc_server.register([bt_server, usb_server])
bt_server.register([pc_server, usb_server])
usb_server.register([bt_server, pc_server])
cam_server.register([bt_server, pc_server, usb_server])
server_list.append(bt_server)
server_list.append(usb_server)
server_list.append(pc_server)
server_list.append(cam_server)
#raise ConnectionError
def write(self, message):
try:
json_str = json.dumps(message)
byte_msg = bytes(json_str, encoding='utf-8')
self.client.send(byte_msg)
print(f'Sent to Android device: {byte_msg}')
except Exception as e:
print(f'Error with writing {message} to {self.get_name()}: {e}')
#print('Reconnecting...')
#self.disconnect()
raise ConnectionError
def disconnect(self):
if self.conn:
self.conn.close()
print('Terminating server socket..')
if self.client:
self.client.close()
print('Terminating client socket..')
self._connected = False
if __name__ == '__main__':
server = BluetoothConn(ProjectConfig())
server.connect()
