def do_run(self, args, user_args): if args.update: _update(True, True) for project in _projects(args, listed_must_be_cloned=False): log.dbg('fetching:', project, level=log.VERBOSE_VERY) _fetch(project)
def do_run(self, args, ignored): self.args = args # Avoid having to pass them around log.dbg('args:', args, level=log.VERBOSE_EXTREME) self._sanity_precheck() self._setup_build_dir() if is_zephyr_build(self.build_dir): self._update_cache() if self.args.cmake or self.args.cmake_opts: self.run_cmake = True else: self.run_cmake = True self._setup_source_dir() self._sanity_check() log.inf('source directory: {}'.format(self.source_dir), colorize=True) log.inf('build directory: {}{}'.format( self.build_dir, (' (created)' if self.created_build_dir else '')), colorize=True) if self.cmake_cache: board = self.cmake_cache.get('CACHED_BOARD') elif self.args.board: board = self.args.board else: board = 'UNKNOWN' # shouldn't happen log.inf('BOARD:', board, colorize=True) self._run_cmake(self.args.cmake_opts) self._sanity_check() self._update_cache() extra_args = ['--target', args.target] if args.target else [] cmake.run_build(self.build_dir, extra_args=extra_args)
def do_run(self, command, **kwargs): reset = False if not self.find_device(): reset = True log.dbg('Device not found, waiting for it', level=log.VERBOSE_EXTREME) # Use of print() here is advised. We don't want to lose # this information in a separate log -- this is # interactive and requires a terminal. print('Please reset your board to switch to DFU mode...') while not self.find_device(): time.sleep(0.1) cmd = list(self.cmd) if self.dfuse: # http://dfu-util.sourceforge.net/dfuse.html dcfg = self.dfuse_config addr_opts = hex(dcfg.address) + ':' + dcfg.options cmd.extend(['-s', addr_opts]) cmd.extend(['-a', self.alt, '-D', self.img]) self.check_call(cmd) if self.dfuse and 'leave' in dcfg.options.split(':'): # Normal DFU devices generally need to be reset to switch # back to the flashed program. # # DfuSe targets do as well, except when 'leave' is given # as an option. reset = False if reset: print('Now reset your board again to switch back to runtime mode.')
def load_float(f, sep=","): header = list(p.strip() for p in f.readline().split(sep)) values = list(list() for e in header) for i,line in enumerate(f.xreadlines()): if i%10000==0: logging.dbg("%i read..." % i) for column,p in enumerate(line.split(sep)): values[column].append( float(p) ) logging.info("%i read..." % (i+1)) return dict( itertools.izip(header, values) )
def accept(self): try: newsock = self.s.accept() log.dbg("Received connection from %s" % newsock[1][0]) thread = Thread (target=self.fun, args=(newsock[0],) ) thread.daemon = True thread.start() except Exception, e: log.dbg( "On Receive: %s" % e )
def load_float(f, sep=","): header = list(p.strip() for p in f.readline().split(sep)) values = list(list() for e in header) for i, line in enumerate(f.xreadlines()): if i % 10000 == 0: logging.dbg("%i read..." % i) for column, p in enumerate(line.split(sep)): values[column].append(float(p)) logging.info("%i read..." % (i + 1)) return dict(itertools.izip(header, values))
def mgmt_connection( sock ): sock.settimeout(.1) notifications=[] uid = "" sessionActive = True closeMgmt = False central = [] while 1: msg = {} if (notifications): if 'FASP' not in notifications[0]: log.err("BAD notif: %s" % notifications[0]) notifications = notifications[1:] continue for i in range( 1, len(notifications) ): try: line = notifications[i].strip() except: continue if line: try: k,v = line.split(": ", 1) except: log.dbg("Bad line: %s" % line) continue msg[k]=v else: fm2 = "%s\n" % "\n".join(notifications) #print "****************\n%s************" % fm2 try: central = centralConnect() except Exception, e: pass for i in central: try: i.sendall(fm2) except Exception, e: pass #notifications=notifications[i+1:] log.dbg("Got message from ASCP %s" % pprint.pformat(msg)) try: activeTransfers[msg['UserStr']][0].put(msg) except: try: # Create two queues, one for messages received from fasp mgmt, # the other for messages to send to fasp mgmt. activeTransfers[msg['UserStr']] = ( Queue.Queue(), Queue.Queue() ) activeTransfers[msg['UserStr']][0].put(msg) uid = msg['UserStr'] except Exception, e: log.info("No UserId specified for transfer, closing FaspMgmt." ) sessionActive = False continue # Parsing notifications until all notifications consumed.
def call(self, cmd): '''Subclass subprocess.call() wrapper. Subclasses should use this method to run command in a subprocess and get its return code, rather than using subprocess directly, to keep accurate debug logs. ''' quoted = quote_sh_list(cmd) if JUST_PRINT: log.inf(quoted) return 0 log.dbg(quoted) return subprocess.call(cmd)
def cleanup(self): log.dbg("Called cleanup on %s" % self.ident) if self.dirty: self.cancel() try: self.dirty = False try: if self.file_list: pass #os.remove(self.file_list) except: pass self.ascp.kill() except: pass
def check_output(self, cmd): '''Subclass subprocess.check_output() wrapper. Subclasses should use this method to run command in a subprocess and check that it executed correctly, rather than using subprocess directly, to keep accurate debug logs. ''' quoted = quote_sh_list(cmd) if JUST_PRINT: log.inf(quoted) return b'' log.dbg(quoted) try: return subprocess.check_output(cmd) except subprocess.CalledProcessError: raise
def _setup_source_dir(self): # Initialize source_dir attribute, either from command line argument, # implicitly from the build directory's CMake cache, or using the # default (current working directory). log.dbg('setting up source directory', level=log.VERBOSE_EXTREME) if self.args.source_dir: source_dir = self.args.source_dir elif self.cmake_cache: source_dir = self.cmake_cache.get('APPLICATION_SOURCE_DIR') if not source_dir: # Maybe Zephyr changed the key? Give the user a way # to retry, at least. log.die("can't determine application from build directory " "{}, please specify an application to build".format( self.build_dir)) else: source_dir = os.getcwd() self.source_dir = os.path.abspath(source_dir)
def is_zephyr_build(path): '''Return true if and only if `path` appears to be a valid Zephyr build directory. "Valid" means the given path is a directory which contains a CMake cache with a 'ZEPHYR_TOOLCHAIN_VARIANT' key. ''' try: cache = cmake.CMakeCache.from_build_dir(path) except FileNotFoundError: cache = {} if 'ZEPHYR_TOOLCHAIN_VARIANT' in cache: log.dbg('{} is a zephyr build directory'.format(path), level=log.VERBOSE_EXTREME) return True else: log.dbg('{} is NOT a valid zephyr build directory'.format(path), level=log.VERBOSE_EXTREME) return False
def popen_ignore_int(self, cmd): '''Spawn a child command, ensuring it ignores SIGINT. The returned subprocess.Popen object must be manually terminated.''' cflags = 0 preexec = None system = platform.system() quoted = quote_sh_list(cmd) if system == 'Windows': cflags |= subprocess.CREATE_NEW_PROCESS_GROUP elif system in {'Linux', 'Darwin'}: preexec = os.setsid if JUST_PRINT: log.inf(quoted) return _DebugDummyPopen() log.dbg(quoted) return subprocess.Popen(cmd, creationflags=cflags, preexec_fn=preexec)
def status(self): "Returns the current session state, ( INIT, SESSION, DONE, or ERROR )." log.dbg("Doing self.status (%s)" % self.stat ) self.consume_msg() try: res = self.ascp.get_nowait() self.ascp_isrunning = False if res: self.stat = "ERROR" self.errs.append(res) if res == None: # Make sure we have consumed all messages. self.consume_msg() time.sleep(0.2) self.consume_msg() if self.stat in ("ERROR", "DONE"): return self.stat try: if self.transfersAttempted == self.transfersPassed: log.dbg("Setting state to DONE based on file success") "This is a work around for ASCP4 which does not always print DONE" self.stat = "DONE" return self.stat except: log.dbg("Could not get completed file count") log.dbg("Setting state to error based on inconclusive xfer results") self.errs.append("DONE/ERROR status message not recieved") self.stat = "ERROR" else: log.dbg("Unexpected Queue Value.") except: pass return self.stat
def cancel(self): if not self.isRunning(): return False try: # Request that ASCP kill itself. msg = \ """FASPMGR 2 Type: CANCEL """ activeTransfers[self.ident][1].put(msg) depth = 0 while (depth < 10): if not self.isRunning(): return True time.sleep(0.5) depth += 1 return False except Exception, e: log.dbg("Got an error trying to kill ASCP: %s" % e) pass
def get_board_snr_from_user(self): snrs = self.check_output(['nrfjprog', '--ids']) snrs = snrs.decode(sys.getdefaultencoding()).strip().splitlines() if len(snrs) == 0: raise RuntimeError( '"nrfjprog --ids" did not find a board; Is the board connected?' ) elif len(snrs) == 1: board_snr = snrs[0] if board_snr == '0': raise RuntimeError( '"nrfjprog --ids" returned 0; is a debugger already connected?' ) return board_snr log.dbg("Refusing the temptation to guess a board", level=log.VERBOSE_EXTREME) # Use of print() here is advised. We don't want to lose # this information in a separate log -- this is # interactive and requires a terminal. print('There are multiple boards connected.') for i, snr in enumerate(snrs, 1): print('{}. {}'.format(i, snr)) p = 'Please select one with desired serial number (1-{}): '.format( len(snrs)) while True: value = input(p) try: value = int(value) except ValueError: continue if 1 <= value <= len(snrs): break return snrs[value - 1]
def _run_cmake(self, cmake_opts): if not self.run_cmake: log.dbg('not running cmake; build system is present') return # It's unfortunate to have to use the undocumented -B and -H # options to set the source and binary directories. # # However, it's the only known way to set that directory and # run CMake from the current working directory. This is # important because users expect invocations like this to Just # Work: # # west build -- -DOVERLAY_CONFIG=relative-path.conf final_cmake_args = [ '-B{}'.format(self.build_dir), '-H{}'.format(self.source_dir), '-G{}'.format(DEFAULT_CMAKE_GENERATOR) ] if self.args.board: final_cmake_args.append('-DBOARD={}'.format(self.args.board)) if cmake_opts: final_cmake_args.extend(cmake_opts) cmake.run_cmake(final_cmake_args)
def centralConnect(): global gAscpRoot path = gAscpRoot.get_nowait() port_paths = glob.glob("%s/var/run/aspera/asperacentral.port" % path) + \ glob.glob("%s/var/run/aspera/*.optport" % path) central = [] for port_fn in port_paths: try: portnum = 0 log.dbg("Reading port %s" % port_fn) with open(port_fn) as cPort: portnum = int(cPort.read()) c= socket.socket() c.connect( ('127.0.0.1', int(portnum) ) ) central.append(c) log.dbg("Success connecting to central") except Exception, e: log.err("Connecting to port %s; %s" % ( port_fn, str(e))) central = False
def create(cls, cfg, args): daparg = os.environ.get('PYOCD_DAPARG') if daparg: log.wrn('Setting PYOCD_DAPARG in the environment is', 'deprecated; use the --daparg option instead.') if args.daparg is None: log.dbg('Missing --daparg set to {} from environment'.format( daparg), level=log.VERBOSE_VERY) args.daparg = daparg build_conf = BuildConfiguration(cfg.build_dir) flash_addr = cls.get_flash_address(args, build_conf) return PyOcdBinaryRunner(cfg, args.target, flashtool=args.flashtool, flash_addr=flash_addr, flashtool_opts=args.flashtool_opt, gdbserver=args.gdbserver, gdb_port=args.gdb_port, tui=args.tui, board_id=args.board_id, daparg=args.daparg)
def _setup_build_dir(self): # Initialize build_dir and created_build_dir attributes. log.dbg('setting up build directory', level=log.VERBOSE_EXTREME) if self.args.build_dir: build_dir = self.args.build_dir else: cwd = os.getcwd() if is_zephyr_build(cwd): build_dir = cwd else: build_dir = DEFAULT_BUILD_DIR build_dir = os.path.abspath(build_dir) if os.path.exists(build_dir): if not os.path.isdir(build_dir): log.die( 'build directory {} exists and is not a directory'.format( build_dir)) else: os.makedirs(build_dir, exist_ok=False) self.created_build_dir = True self.run_cmake = True self.build_dir = build_dir
activeTransfers[msg['UserStr']][0].put(msg) uid = msg['UserStr'] except Exception, e: log.info("No UserId specified for transfer, closing FaspMgmt." ) sessionActive = False continue # Parsing notifications until all notifications consumed. if uid: msg = None try: msg = activeTransfers[uid][1].get_nowait() except: pass if msg: log.dbg("Sending msg to ascp '%s'" % msg) sock.sendall(msg) continue if closeMgmt: log.dbg("Closing mgmt connection; socket closed") return gotData = False gotTimeout = False notifications = [] h = "" try: h = sock.recv(16384) notifications += h.splitlines()
def _sanity_check(self): # Sanity check the build configuration. # Side effect: may update cmake_cache attribute. log.dbg('sanity checking the build', level=log.VERBOSE_EXTREME) if self.source_dir == self.build_dir: # There's no forcing this. log.die('source and build directory {} cannot be the same; ' 'use --build-dir {} to specify a build directory'.format( self.source_dir, self.build_dir)) srcrel = os.path.relpath(self.source_dir) if is_zephyr_build(self.source_dir): self._check_force( 'it looks like {srcrel} is a build directory: ' 'did you mean -build-dir {srcrel} instead?'.format( srcrel=srcrel)) elif 'CMakeLists.txt' not in os.listdir(self.source_dir): self._check_force( 'source directory "{srcrel}" does not contain ' 'a CMakeLists.txt; is that really what you ' 'want to build? (Use -s SOURCE_DIR to specify ' 'the application source directory)'.format(srcrel=srcrel)) if not is_zephyr_build(self.build_dir) and not self.args.board: self._check_force('this looks like a new or clean build, ' 'please provide --board') if not self.cmake_cache: return # That's all we can check without a cache. cached_app = self.cmake_cache.get('APPLICATION_SOURCE_DIR') log.dbg('APPLICATION_SOURCE_DIR:', cached_app, level=log.VERBOSE_EXTREME) source_abs = (os.path.abspath(self.args.source_dir) if self.args.source_dir else None) cached_abs = os.path.abspath(cached_app) if cached_app else None if cached_abs and source_abs and source_abs != cached_abs: self._check_force('build directory "{}" is for application "{}", ' 'but source directory "{}" was specified; ' 'please clean it or use --build-dir to set ' 'another build directory'.format( self.build_dir, cached_abs, source_abs)) self.run_cmake = True # If they insist, we need to re-run cmake. cached_board = self.cmake_cache.get('CACHED_BOARD') log.dbg('CACHED_BOARD:', cached_board, level=log.VERBOSE_EXTREME) if not cached_board and not self.args.board: if self.created_build_dir: self._check_force( 'Building for the first time: you must provide --board') else: self._check_force( 'Board is missing or unknown, please provide --board') if self.args.board and cached_board and \ self.args.board != cached_board: self._check_force('Build directory {} targets board {}, ' 'but board {} was specified. (Clean that ' 'directory or use --build-dir to specify ' 'a different one.)'.format( self.build_dir, cached_board, self.args.board))
from csv_writer import CSVWriter from ppjjww_wmuvm_process import * if __name__=="__main__": try: inpath = sys.argv[1] except: logging.err("Input LHE file path expected!"); sys.exit(-1) try: outpath = sys.argv[2] except: logging.err("Output CSV file path expected!"); sys.exit(-1) lhe = LHELoader(open(inpath)) csv = CSVWriter(open(outpath, "w")) logging.info("Loading and parsing events...") for i,eventlines in enumerate(lhe.yield_events()): if i%10000==0: logging.dbg("%i events read..." % i) particles = eventlines_to_particlesdict(eventlines, \ particles_naming_function = name_particles) variables = calculate_dependent_variables(particles) csv.write_dict(variables) logging.info("events_counter = %i" % lhe.events_counter)
from ppjjww_wmuvm_process import * if __name__ == "__main__": try: inpath = sys.argv[1] except: logging.err("Input LHE file path expected!") sys.exit(-1) try: outpath = sys.argv[2] except: logging.err("Output CSV file path expected!") sys.exit(-1) lhe = LHELoader(open(inpath)) csv = CSVWriter(open(outpath, "w")) logging.info("Loading and parsing events...") for i, eventlines in enumerate(lhe.yield_events()): if i % 10000 == 0: logging.dbg("%i events read..." % i) particles = eventlines_to_particlesdict(eventlines, \ particles_naming_function = name_particles) variables = calculate_dependent_variables(particles) if len(variables) != 0: csv.write_dict(variables) logging.info("events_counter = %i" % lhe.events_counter)
def handle(result): log.dbg('project', project.name, 'is {}cloned'.format('' if result else 'not '), level=log.VERBOSE_EXTREME) return result
def _git_helper(project, cmd, extra_args, cwd, capture_stdout, check): # Runs a git command. # # project: # The Project instance for the project, derived from the manifest file. # # cmd: # String with git arguments. Supports some "(foo)" shorthands. See below. # # extra_args: # List of additional arguments to pass to the git command (e.g. from the # user). # # cwd: # Directory to switch to first (None = current directory) # # capture_stdout: # True if stdout should be captured into the returned # subprocess.CompletedProcess instance instead of being printed. # # We never capture stderr, to prevent error messages from being eaten. # # check: # True if an error should be raised if the git command finishes with a # non-zero return code. # # Returns a subprocess.CompletedProcess instance. # TODO: Run once somewhere? if shutil.which('git') is None: log.die('Git is not installed or cannot be found') args = (('git', ) + tuple(_expand_shorthands(project, arg) for arg in cmd.split()) + tuple(extra_args)) cmd_str = util.quote_sh_list(args) log.dbg("running '{}'".format(cmd_str), 'in', cwd, level=log.VERBOSE_VERY) popen = subprocess.Popen( args, stdout=subprocess.PIPE if capture_stdout else None, cwd=cwd) stdout, _ = popen.communicate() dbg_msg = "'{}' in {} finished with exit status {}" \ .format(cmd_str, cwd, popen.returncode) if capture_stdout: dbg_msg += " and wrote {} to stdout".format(stdout) log.dbg(dbg_msg, level=log.VERBOSE_VERY) if check and popen.returncode: _die(project, "Command '{}' failed for (name-and-path)".format(cmd_str)) if capture_stdout: # Manual UTF-8 decoding and universal newlines. Before Python 3.6, # Popen doesn't seem to allow using universal newlines mode (which # enables decoding) with a specific encoding (because the encoding= # parameter is missing). # # Also strip all trailing newlines as convenience. The splitlines() # already means we lose a final '\n' anyway. stdout = "\n".join(stdout.decode('utf-8').splitlines()).rstrip("\n") return CompletedProcess(popen.args, popen.returncode, stdout)
def _dbg(project, msg, level): # Like _wrn(), for debug messages log.dbg(_expand_shorthands(project, msg), level=level)