def do_run(self, args, user_args):
if args.update:
_update(True, True)
for project in _projects(args, listed_must_be_cloned=False):
log.dbg('fetching:', project, level=log.VERBOSE_VERY)
_fetch(project)
def do_run(self, args, ignored):
self.args = args # Avoid having to pass them around
log.dbg('args:', args, level=log.VERBOSE_EXTREME)
self._sanity_precheck()
self._setup_build_dir()
if is_zephyr_build(self.build_dir):
self._update_cache()
if self.args.cmake or self.args.cmake_opts:
self.run_cmake = True
else:
self.run_cmake = True
self._setup_source_dir()
self._sanity_check()
log.inf('source directory: {}'.format(self.source_dir), colorize=True)
log.inf('build directory: {}{}'.format(
self.build_dir, (' (created)' if self.created_build_dir else '')),
colorize=True)
if self.cmake_cache:
board = self.cmake_cache.get('CACHED_BOARD')
elif self.args.board:
board = self.args.board
else:
board = 'UNKNOWN' # shouldn't happen
log.inf('BOARD:', board, colorize=True)
self._run_cmake(self.args.cmake_opts)
self._sanity_check()
self._update_cache()
extra_args = ['--target', args.target] if args.target else []
cmake.run_build(self.build_dir, extra_args=extra_args)
def do_run(self, command, **kwargs):
reset = False
if not self.find_device():
reset = True
log.dbg('Device not found, waiting for it',
level=log.VERBOSE_EXTREME)
# Use of print() here is advised. We don't want to lose
# this information in a separate log -- this is
# interactive and requires a terminal.
print('Please reset your board to switch to DFU mode...')
while not self.find_device():
time.sleep(0.1)
cmd = list(self.cmd)
if self.dfuse:
# http://dfu-util.sourceforge.net/dfuse.html
dcfg = self.dfuse_config
addr_opts = hex(dcfg.address) + ':' + dcfg.options
cmd.extend(['-s', addr_opts])
cmd.extend(['-a', self.alt, '-D', self.img])
self.check_call(cmd)
if self.dfuse and 'leave' in dcfg.options.split(':'):
# Normal DFU devices generally need to be reset to switch
# back to the flashed program.
#
# DfuSe targets do as well, except when 'leave' is given
# as an option.
reset = False
if reset:
print('Now reset your board again to switch back to runtime mode.')
def load_float(f, sep=","):
header = list(p.strip() for p in f.readline().split(sep))
values = list(list() for e in header)
for i,line in enumerate(f.xreadlines()):
if i%10000==0: logging.dbg("%i read..." % i)
for column,p in enumerate(line.split(sep)):
values[column].append( float(p) )
logging.info("%i read..." % (i+1))
return dict( itertools.izip(header, values) )
def accept(self):
try:
newsock = self.s.accept()
log.dbg("Received connection from %s" % newsock[1][0])
thread = Thread (target=self.fun, args=(newsock[0],) )
thread.daemon = True
thread.start()
except Exception, e:
log.dbg( "On Receive: %s" % e )
def load_float(f, sep=","):
header = list(p.strip() for p in f.readline().split(sep))
values = list(list() for e in header)
for i, line in enumerate(f.xreadlines()):
if i % 10000 == 0: logging.dbg("%i read..." % i)
for column, p in enumerate(line.split(sep)):
values[column].append(float(p))
logging.info("%i read..." % (i + 1))
return dict(itertools.izip(header, values))
def mgmt_connection( sock ):
sock.settimeout(.1)
notifications=[]
uid = ""
sessionActive = True
closeMgmt = False
central = []
while 1:
msg = {}
if (notifications):
if 'FASP' not in notifications[0]:
log.err("BAD notif: %s" % notifications[0])
notifications = notifications[1:]
continue
for i in range( 1, len(notifications) ):
try:
line = notifications[i].strip()
except:
continue
if line:
try:
k,v = line.split(": ", 1)
except:
log.dbg("Bad line: %s" % line)
continue
msg[k]=v
else:
fm2 = "%s\n" % "\n".join(notifications)
#print "****************\n%s************" % fm2
try:
central = centralConnect()
except Exception, e:
pass
for i in central:
try:
i.sendall(fm2)
except Exception, e:
pass
#notifications=notifications[i+1:]
log.dbg("Got message from ASCP %s" % pprint.pformat(msg))
try:
activeTransfers[msg['UserStr']][0].put(msg)
except:
try:
# Create two queues, one for messages received from fasp mgmt,
# the other for messages to send to fasp mgmt.
activeTransfers[msg['UserStr']] = ( Queue.Queue(), Queue.Queue() )
activeTransfers[msg['UserStr']][0].put(msg)
uid = msg['UserStr']
except Exception, e:
log.info("No UserId specified for transfer, closing FaspMgmt." )
sessionActive = False
continue # Parsing notifications until all notifications consumed.
def call(self, cmd):
'''Subclass subprocess.call() wrapper.
Subclasses should use this method to run command in a
subprocess and get its return code, rather than
using subprocess directly, to keep accurate debug logs.
'''
quoted = quote_sh_list(cmd)
if JUST_PRINT:
log.inf(quoted)
return 0
log.dbg(quoted)
return subprocess.call(cmd)
def cleanup(self):
log.dbg("Called cleanup on %s" % self.ident)
if self.dirty:
self.cancel()
try:
self.dirty = False
try:
if self.file_list:
pass
#os.remove(self.file_list)
except:
pass
self.ascp.kill()
except:
pass
def check_output(self, cmd):
'''Subclass subprocess.check_output() wrapper.
Subclasses should use this method to run command in a
subprocess and check that it executed correctly, rather than
using subprocess directly, to keep accurate debug logs.
'''
quoted = quote_sh_list(cmd)
if JUST_PRINT:
log.inf(quoted)
return b''
log.dbg(quoted)
try:
return subprocess.check_output(cmd)
except subprocess.CalledProcessError:
raise
def _setup_source_dir(self):
# Initialize source_dir attribute, either from command line argument,
# implicitly from the build directory's CMake cache, or using the
# default (current working directory).
log.dbg('setting up source directory', level=log.VERBOSE_EXTREME)
if self.args.source_dir:
source_dir = self.args.source_dir
elif self.cmake_cache:
source_dir = self.cmake_cache.get('APPLICATION_SOURCE_DIR')
if not source_dir:
# Maybe Zephyr changed the key? Give the user a way
# to retry, at least.
log.die("can't determine application from build directory "
"{}, please specify an application to build".format(
self.build_dir))
else:
source_dir = os.getcwd()
self.source_dir = os.path.abspath(source_dir)
def is_zephyr_build(path):
'''Return true if and only if `path` appears to be a valid Zephyr
build directory.
"Valid" means the given path is a directory which contains a CMake
cache with a 'ZEPHYR_TOOLCHAIN_VARIANT' key.
'''
try:
cache = cmake.CMakeCache.from_build_dir(path)
except FileNotFoundError:
cache = {}
if 'ZEPHYR_TOOLCHAIN_VARIANT' in cache:
log.dbg('{} is a zephyr build directory'.format(path),
level=log.VERBOSE_EXTREME)
return True
else:
log.dbg('{} is NOT a valid zephyr build directory'.format(path),
level=log.VERBOSE_EXTREME)
return False
def popen_ignore_int(self, cmd):
'''Spawn a child command, ensuring it ignores SIGINT.
The returned subprocess.Popen object must be manually terminated.'''
cflags = 0
preexec = None
system = platform.system()
quoted = quote_sh_list(cmd)
if system == 'Windows':
cflags |= subprocess.CREATE_NEW_PROCESS_GROUP
elif system in {'Linux', 'Darwin'}:
preexec = os.setsid
if JUST_PRINT:
log.inf(quoted)
return _DebugDummyPopen()
log.dbg(quoted)
return subprocess.Popen(cmd, creationflags=cflags, preexec_fn=preexec)
def status(self):
"Returns the current session state, ( INIT, SESSION, DONE, or ERROR )."
log.dbg("Doing self.status (%s)" % self.stat )
self.consume_msg()
try:
res = self.ascp.get_nowait()
self.ascp_isrunning = False
if res:
self.stat = "ERROR"
self.errs.append(res)
if res == None:
# Make sure we have consumed all messages.
self.consume_msg()
time.sleep(0.2)
self.consume_msg()
if self.stat in ("ERROR", "DONE"):
return self.stat
try:
if self.transfersAttempted == self.transfersPassed:
log.dbg("Setting state to DONE based on file success")
"This is a work around for ASCP4 which does not always print DONE"
self.stat = "DONE"
return self.stat
except:
log.dbg("Could not get completed file count")
log.dbg("Setting state to error based on inconclusive xfer results")
self.errs.append("DONE/ERROR status message not recieved")
self.stat = "ERROR"
else:
log.dbg("Unexpected Queue Value.")
except:
pass
return self.stat
def cancel(self):
if not self.isRunning():
return False
try:
# Request that ASCP kill itself.
msg = \
"""FASPMGR 2
Type: CANCEL
"""
activeTransfers[self.ident][1].put(msg)
depth = 0
while (depth < 10):
if not self.isRunning():
return True
time.sleep(0.5)
depth += 1
return False
except Exception, e:
log.dbg("Got an error trying to kill ASCP: %s" % e)
pass
def get_board_snr_from_user(self):
snrs = self.check_output(['nrfjprog', '--ids'])
snrs = snrs.decode(sys.getdefaultencoding()).strip().splitlines()
if len(snrs) == 0:
raise RuntimeError(
'"nrfjprog --ids" did not find a board; Is the board connected?'
)
elif len(snrs) == 1:
board_snr = snrs[0]
if board_snr == '0':
raise RuntimeError(
'"nrfjprog --ids" returned 0; is a debugger already connected?'
)
return board_snr
log.dbg("Refusing the temptation to guess a board",
level=log.VERBOSE_EXTREME)
# Use of print() here is advised. We don't want to lose
# this information in a separate log -- this is
# interactive and requires a terminal.
print('There are multiple boards connected.')
for i, snr in enumerate(snrs, 1):
print('{}. {}'.format(i, snr))
p = 'Please select one with desired serial number (1-{}): '.format(
len(snrs))
while True:
value = input(p)
try:
value = int(value)
except ValueError:
continue
if 1 <= value <= len(snrs):
break
return snrs[value - 1]
def _run_cmake(self, cmake_opts):
if not self.run_cmake:
log.dbg('not running cmake; build system is present')
return
# It's unfortunate to have to use the undocumented -B and -H
# options to set the source and binary directories.
#
# However, it's the only known way to set that directory and
# run CMake from the current working directory. This is
# important because users expect invocations like this to Just
# Work:
#
# west build -- -DOVERLAY_CONFIG=relative-path.conf
final_cmake_args = [
'-B{}'.format(self.build_dir), '-H{}'.format(self.source_dir),
'-G{}'.format(DEFAULT_CMAKE_GENERATOR)
]
if self.args.board:
final_cmake_args.append('-DBOARD={}'.format(self.args.board))
if cmake_opts:
final_cmake_args.extend(cmake_opts)
cmake.run_cmake(final_cmake_args)
def centralConnect():
global gAscpRoot
path = gAscpRoot.get_nowait()
port_paths = glob.glob("%s/var/run/aspera/asperacentral.port" % path) + \
glob.glob("%s/var/run/aspera/*.optport" % path)
central = []
for port_fn in port_paths:
try:
portnum = 0
log.dbg("Reading port %s" % port_fn)
with open(port_fn) as cPort:
portnum = int(cPort.read())
c= socket.socket()
c.connect( ('127.0.0.1', int(portnum) ) )
central.append(c)
log.dbg("Success connecting to central")
except Exception, e:
log.err("Connecting to port %s; %s" % ( port_fn, str(e)))
central = False
def create(cls, cfg, args):
daparg = os.environ.get('PYOCD_DAPARG')
if daparg:
log.wrn('Setting PYOCD_DAPARG in the environment is',
'deprecated; use the --daparg option instead.')
if args.daparg is None:
log.dbg('Missing --daparg set to {} from environment'.format(
daparg),
level=log.VERBOSE_VERY)
args.daparg = daparg
build_conf = BuildConfiguration(cfg.build_dir)
flash_addr = cls.get_flash_address(args, build_conf)
return PyOcdBinaryRunner(cfg,
args.target,
flashtool=args.flashtool,
flash_addr=flash_addr,
flashtool_opts=args.flashtool_opt,
gdbserver=args.gdbserver,
gdb_port=args.gdb_port,
tui=args.tui,
board_id=args.board_id,
daparg=args.daparg)
def _setup_build_dir(self):
# Initialize build_dir and created_build_dir attributes.
log.dbg('setting up build directory', level=log.VERBOSE_EXTREME)
if self.args.build_dir:
build_dir = self.args.build_dir
else:
cwd = os.getcwd()
if is_zephyr_build(cwd):
build_dir = cwd
else:
build_dir = DEFAULT_BUILD_DIR
build_dir = os.path.abspath(build_dir)
if os.path.exists(build_dir):
if not os.path.isdir(build_dir):
log.die(
'build directory {} exists and is not a directory'.format(
build_dir))
else:
os.makedirs(build_dir, exist_ok=False)
self.created_build_dir = True
self.run_cmake = True
self.build_dir = build_dir
activeTransfers[msg['UserStr']][0].put(msg)
uid = msg['UserStr']
except Exception, e:
log.info("No UserId specified for transfer, closing FaspMgmt." )
sessionActive = False
continue # Parsing notifications until all notifications consumed.
if uid:
msg = None
try:
msg = activeTransfers[uid][1].get_nowait()
except:
pass
if msg:
log.dbg("Sending msg to ascp '%s'" % msg)
sock.sendall(msg)
continue
if closeMgmt:
log.dbg("Closing mgmt connection; socket closed")
return
gotData = False
gotTimeout = False
notifications = []
h = ""
try:
h = sock.recv(16384)
notifications += h.splitlines()
def _sanity_check(self):
# Sanity check the build configuration.
# Side effect: may update cmake_cache attribute.
log.dbg('sanity checking the build', level=log.VERBOSE_EXTREME)
if self.source_dir == self.build_dir:
# There's no forcing this.
log.die('source and build directory {} cannot be the same; '
'use --build-dir {} to specify a build directory'.format(
self.source_dir, self.build_dir))
srcrel = os.path.relpath(self.source_dir)
if is_zephyr_build(self.source_dir):
self._check_force(
'it looks like {srcrel} is a build directory: '
'did you mean -build-dir {srcrel} instead?'.format(
srcrel=srcrel))
elif 'CMakeLists.txt' not in os.listdir(self.source_dir):
self._check_force(
'source directory "{srcrel}" does not contain '
'a CMakeLists.txt; is that really what you '
'want to build? (Use -s SOURCE_DIR to specify '
'the application source directory)'.format(srcrel=srcrel))
if not is_zephyr_build(self.build_dir) and not self.args.board:
self._check_force('this looks like a new or clean build, '
'please provide --board')
if not self.cmake_cache:
return # That's all we can check without a cache.
cached_app = self.cmake_cache.get('APPLICATION_SOURCE_DIR')
log.dbg('APPLICATION_SOURCE_DIR:',
cached_app,
level=log.VERBOSE_EXTREME)
source_abs = (os.path.abspath(self.args.source_dir)
if self.args.source_dir else None)
cached_abs = os.path.abspath(cached_app) if cached_app else None
if cached_abs and source_abs and source_abs != cached_abs:
self._check_force('build directory "{}" is for application "{}", '
'but source directory "{}" was specified; '
'please clean it or use --build-dir to set '
'another build directory'.format(
self.build_dir, cached_abs, source_abs))
self.run_cmake = True # If they insist, we need to re-run cmake.
cached_board = self.cmake_cache.get('CACHED_BOARD')
log.dbg('CACHED_BOARD:', cached_board, level=log.VERBOSE_EXTREME)
if not cached_board and not self.args.board:
if self.created_build_dir:
self._check_force(
'Building for the first time: you must provide --board')
else:
self._check_force(
'Board is missing or unknown, please provide --board')
if self.args.board and cached_board and \
self.args.board != cached_board:
self._check_force('Build directory {} targets board {}, '
'but board {} was specified. (Clean that '
'directory or use --build-dir to specify '
'a different one.)'.format(
self.build_dir, cached_board,
self.args.board))
from csv_writer import CSVWriter
from ppjjww_wmuvm_process import *
if __name__=="__main__":
try: inpath = sys.argv[1]
except: logging.err("Input LHE file path expected!"); sys.exit(-1)
try: outpath = sys.argv[2]
except: logging.err("Output CSV file path expected!"); sys.exit(-1)
lhe = LHELoader(open(inpath))
csv = CSVWriter(open(outpath, "w"))
logging.info("Loading and parsing events...")
for i,eventlines in enumerate(lhe.yield_events()):
if i%10000==0: logging.dbg("%i events read..." % i)
particles = eventlines_to_particlesdict(eventlines, \
particles_naming_function = name_particles)
variables = calculate_dependent_variables(particles)
csv.write_dict(variables)
logging.info("events_counter = %i" % lhe.events_counter)
from ppjjww_wmuvm_process import *
if __name__ == "__main__":
try:
inpath = sys.argv[1]
except:
logging.err("Input LHE file path expected!")
sys.exit(-1)
try:
outpath = sys.argv[2]
except:
logging.err("Output CSV file path expected!")
sys.exit(-1)
lhe = LHELoader(open(inpath))
csv = CSVWriter(open(outpath, "w"))
logging.info("Loading and parsing events...")
for i, eventlines in enumerate(lhe.yield_events()):
if i % 10000 == 0: logging.dbg("%i events read..." % i)
particles = eventlines_to_particlesdict(eventlines, \
particles_naming_function = name_particles)
variables = calculate_dependent_variables(particles)
if len(variables) != 0:
csv.write_dict(variables)
logging.info("events_counter = %i" % lhe.events_counter)
def handle(result):
log.dbg('project',
project.name,
'is {}cloned'.format('' if result else 'not '),
level=log.VERBOSE_EXTREME)
return result
def _git_helper(project, cmd, extra_args, cwd, capture_stdout, check):
# Runs a git command.
#
# project:
# The Project instance for the project, derived from the manifest file.
#
# cmd:
# String with git arguments. Supports some "(foo)" shorthands. See below.
#
# extra_args:
# List of additional arguments to pass to the git command (e.g. from the
# user).
#
# cwd:
# Directory to switch to first (None = current directory)
#
# capture_stdout:
# True if stdout should be captured into the returned
# subprocess.CompletedProcess instance instead of being printed.
#
# We never capture stderr, to prevent error messages from being eaten.
#
# check:
# True if an error should be raised if the git command finishes with a
# non-zero return code.
#
# Returns a subprocess.CompletedProcess instance.
# TODO: Run once somewhere?
if shutil.which('git') is None:
log.die('Git is not installed or cannot be found')
args = (('git', ) +
tuple(_expand_shorthands(project, arg)
for arg in cmd.split()) + tuple(extra_args))
cmd_str = util.quote_sh_list(args)
log.dbg("running '{}'".format(cmd_str), 'in', cwd, level=log.VERBOSE_VERY)
popen = subprocess.Popen(
args, stdout=subprocess.PIPE if capture_stdout else None, cwd=cwd)
stdout, _ = popen.communicate()
dbg_msg = "'{}' in {} finished with exit status {}" \
.format(cmd_str, cwd, popen.returncode)
if capture_stdout:
dbg_msg += " and wrote {} to stdout".format(stdout)
log.dbg(dbg_msg, level=log.VERBOSE_VERY)
if check and popen.returncode:
_die(project,
"Command '{}' failed for (name-and-path)".format(cmd_str))
if capture_stdout:
# Manual UTF-8 decoding and universal newlines. Before Python 3.6,
# Popen doesn't seem to allow using universal newlines mode (which
# enables decoding) with a specific encoding (because the encoding=
# parameter is missing).
#
# Also strip all trailing newlines as convenience. The splitlines()
# already means we lose a final '\n' anyway.
stdout = "\n".join(stdout.decode('utf-8').splitlines()).rstrip("\n")
return CompletedProcess(popen.args, popen.returncode, stdout)
def _dbg(project, msg, level):
# Like _wrn(), for debug messages
log.dbg(_expand_shorthands(project, msg), level=level)
