def test_f2py(): # test that we can run f2py script if sys.platform == 'win32': exe_dir = dirname(sys.executable) if exe_dir.endswith('Scripts'): # virtualenv f2py_cmd = r"%s\f2py.py" % exe_dir else: f2py_cmd = r"%s\Scripts\f2py.py" % exe_dir code, stdout, stderr = run_command([sys.executable, f2py_cmd, '-v']) success = stdout.strip() == asbytes('2') assert_(success, "Warning: f2py not found in path") else: version = sys.version_info major = str(version.major) minor = str(version.minor) f2py_cmds = ('f2py', 'f2py' + major, 'f2py' + major + '.' + minor) success = False for f2py_cmd in f2py_cmds: try: code, stdout, stderr = run_command([f2py_cmd, '-v']) assert_equal(stdout.strip(), asbytes('2')) success = True break except: pass msg = "Warning: neither %s nor %s nor %s found in path" % f2py_cmds assert_(success, msg)
def test_f2py(): # test that we can run f2py script if sys.platform == 'win32': exe_dir = dirname(sys.executable) if exe_dir.endswith('Scripts'): # virtualenv f2py_cmd = r"%s\f2py.py" % exe_dir else: f2py_cmd = r"%s\Scripts\f2py.py" % exe_dir code, stdout, stderr = run_command([sys.executable, f2py_cmd, '-v']) success = stdout.strip() == asbytes('2') assert_(success, "Warning: f2py not found in path") else: # unclear what f2py cmd was installed as, check plain (f2py) and # current python version specific one (f2py3.4) f2py_cmds = ('f2py', 'f2py' + basename(sys.executable)[6:]) success = False for f2py_cmd in f2py_cmds: try: code, stdout, stderr = run_command([f2py_cmd, '-v']) assert_equal(stdout.strip(), asbytes('2')) success = True break except: pass msg = "Warning: neither %s nor %s found in path" % f2py_cmds assert_(success, msg)
def _write_header(fileobj, header): """ Write TCK header to file-like object. Parameters ---------- fileobj : file-like object An open file-like object in binary mode pointing to TCK file (and ready to read from the beginning of the TCK header). """ # Fields to exclude exclude = [ Field.MAGIC_NUMBER, # Handled separately. Field.NB_STREAMLINES, # Handled separately. Field.ENDIANNESS, # Handled separately. Field.VOXEL_TO_RASMM, # Streamlines are always in RAS+ mm. "count", "datatype", "file" ] # Fields being replaced. lines = [] lines.append(asstr(header[Field.MAGIC_NUMBER])) lines.append("count: {0:010}".format(header[Field.NB_STREAMLINES])) lines.append("datatype: Float32LE") # Always Float32LE. lines.extend([ "{0}: {1}".format(k, v) for k, v in header.items() if k not in exclude and not k.startswith("_") ]) lines.append("file: . ") # Manually add this last field. out = "\n".join(lines) # Check the header is well formatted. if out.count("\n") > len(lines) - 1: # \n only allowed between lines. msg = "Key-value pairs cannot contain '\\n':\n{}".format(out) raise HeaderError(msg) if out.count(":") > len(lines) - 1: # : only one per line (except the last one which contains END). msg = "Key-value pairs cannot contain ':':\n{}".format(out) raise HeaderError(msg) # Write header to file. fileobj.write(asbytes(out)) hdr_len_no_offset = len(out) + 5 # Need to add number of bytes to store offset as decimal string. We # start with estimate without string, then update if the # offset-as-decimal-string got longer after adding length of the # offset string. new_offset = -1 old_offset = hdr_len_no_offset while new_offset != old_offset: old_offset = new_offset new_offset = hdr_len_no_offset + len(str(old_offset)) fileobj.write(asbytes(str(new_offset) + "\n")) fileobj.write(asbytes("END\n"))
def test_is_supported_detect_format(): # Test is_supported and detect_format functions # Empty file/string f = BytesIO() assert not nib.streamlines.is_supported(f) assert not nib.streamlines.is_supported("") assert nib.streamlines.detect_format(f) is None assert nib.streamlines.detect_format("") is None # Valid file without extension for tfile_cls in FORMATS.values(): f = BytesIO() f.write(asbytes(tfile_cls.MAGIC_NUMBER)) f.seek(0, os.SEEK_SET) assert nib.streamlines.is_supported(f) assert nib.streamlines.detect_format(f) is tfile_cls # Wrong extension but right magic number for tfile_cls in FORMATS.values(): with tempfile.TemporaryFile(mode="w+b", suffix=".txt") as f: f.write(asbytes(tfile_cls.MAGIC_NUMBER)) f.seek(0, os.SEEK_SET) assert nib.streamlines.is_supported(f) assert nib.streamlines.detect_format(f) is tfile_cls # Good extension but wrong magic number for ext, tfile_cls in FORMATS.items(): with tempfile.TemporaryFile(mode="w+b", suffix=ext) as f: f.write(b"pass") f.seek(0, os.SEEK_SET) assert not nib.streamlines.is_supported(f) assert nib.streamlines.detect_format(f) is None # Wrong extension, string only f = "my_tractogram.asd" assert not nib.streamlines.is_supported(f) assert nib.streamlines.detect_format(f) is None # Good extension, string only for ext, tfile_cls in FORMATS.items(): f = "my_tractogram" + ext assert nib.streamlines.is_supported(f) assert nib.streamlines.detect_format(f) == tfile_cls # Extension should not be case-sensitive. for ext, tfile_cls in FORMATS.items(): f = "my_tractogram" + ext.upper() assert nib.streamlines.detect_format(f) is tfile_cls
def test_iter(): # Check we can iterate over lines, if the underlying file object allows it lines = \ """On the blue ridged mountains of virginia """.split('\n') with InTemporaryDirectory(): sobj = BytesIO() files_to_test = [('test.txt', True), ('test.txt.gz', False), ('test.txt.bz2', False), (sobj, True)] if HAVE_ZSTD: files_to_test += [('test.txt.zst', False)] for input, does_t in files_to_test: with Opener(input, 'wb') as fobj: for line in lines: fobj.write(asbytes(line + os.linesep)) with Opener(input, 'rb') as fobj: for back_line, line in zip(fobj, lines): assert asstr(back_line).rstrip() == line if not does_t: continue with Opener(input, 'rt') as fobj: for back_line, line in zip(fobj, lines): assert back_line.rstrip() == line lobj = Opener(Lunk('')) with pytest.raises(TypeError): list(lobj)
def test_iter(): # Check we can iterate over lines, if the underlying file object allows it lines = \ """On the blue ridged mountains of virginia """.split('\n') with InTemporaryDirectory(): sobj = BytesIO() for input, does_t in (('test.txt', True), ('test.txt.gz', False), ('test.txt.bz2', False), (sobj, True)): with Opener(input, 'wb') as fobj: for line in lines: fobj.write(asbytes(line + os.linesep)) with Opener(input, 'rb') as fobj: for back_line, line in zip(fobj, lines): assert_equal(asstr(back_line).rstrip(), line) if not does_t: continue with Opener(input, 'rt') as fobj: for back_line, line in zip(fobj, lines): assert_equal(back_line.rstrip(), line) lobj = Opener(Lunk('')) assert_raises(TypeError, list, lobj)
def _write_var_metadata(self, name): var = self.variables[name] self._pack_string(name) self._pack_int(len(var.dimensions)) for dimname in var.dimensions: dimid = self._dims.index(dimname) self._pack_int(dimid) self._write_att_array(var._attributes) nc_type = REVERSE[var.typecode(), var.itemsize()] self.fp.write(asbytes(nc_type)) if not var.isrec: vsize = var.data.size * var.data.itemsize vsize += -vsize % 4 else: # record variable try: vsize = var.data[0].size * var.data.itemsize except IndexError: vsize = 0 rec_vars = len( [var for var in self.variables.values() if var.isrec]) if rec_vars > 1: vsize += -vsize % 4 self.variables[name].__dict__['_vsize'] = vsize self._pack_int(vsize) # Pack a bogus begin, and set the real value later. self.variables[name].__dict__['_begin'] = self.fp.tell() self._pack_begin(0)
def puts(self, addr, s): """Put string of bytes at given address. Will overwrite any previous entries. """ a = array('B', asbytes(s)) for i in range(len(a)): self._buf[addr + i] = a[i]
def test_f2py(): # test that we can run f2py script if sys.platform == 'win32': f2py_cmd = r"%s\Scripts\f2py.py" % dirname(sys.executable) code, stdout, stderr = run_command([sys.executable, f2py_cmd, '-v']) else: f2py_cmd = 'f2py' + basename(sys.executable)[6:] code, stdout, stderr = run_command([f2py_cmd, '-v']) assert_equal(stdout.strip(), asbytes('2'))
def test_f2py(): # test that we can run f2py script if sys.platform == 'win32': f2py_cmd = r"%s\Scripts\f2py.py" % dirname(sys.executable) code, stdout, stderr = run_command([sys.executable, f2py_cmd, '-v']) assert_equal(stdout.strip(), asbytes('2')) else: # unclear what f2py cmd was installed as, check plain (f2py) and # current python version specific one (f2py3.4) f2py_cmds = ['f2py', 'f2py' + basename(sys.executable)[6:]] success = False for f2py_cmd in f2py_cmds: try: code, stdout, stderr = run_command([f2py_cmd, '-v']) assert_equal(stdout.strip(), asbytes('2')) success = True break except FileNotFoundError: pass assert_(success, "wasn't able to find f2py or %s on commandline" % f2py_cmds[1])
def gets(self, addr, length): """Get string of bytes from given address. If any entries are blank from addr through addr+length, a NotEnoughDataError exception will be raised. Padding is not used.""" a = array('B', asbytes('\0' * length)) try: for i in range(length): a[i] = self._buf[addr + i] except KeyError: raise NotEnoughDataError(address=addr, length=length) return asstr(a.tostring())
def test_f2py(): # test that we can run f2py script if sys.platform == 'win32': f2py_cmd = r"%s\Scripts\f2py.py" % dirname(sys.executable) code, stdout, stderr = run_command([sys.executable, f2py_cmd, '-v']) success = stdout.strip() == asbytes('2') assert_(success, "Warning: f2py not found in path") else: # unclear what f2py cmd was installed as, check plain (f2py) and # current python version specific one (f2py3.4) f2py_cmds = ('f2py', 'f2py' + basename(sys.executable)[6:]) success = False for f2py_cmd in f2py_cmds: try: code, stdout, stderr = run_command([f2py_cmd, '-v']) assert_equal(stdout.strip(), asbytes('2')) success = True break except: pass assert_(success, "Warning: neither %s nor %s found in path" % f2py_cmds)
def test_f2py(): # test that we can run f2py script if sys.platform == 'win32': exe_dir = dirname(sys.executable) if exe_dir.endswith('Scripts'): # virtualenv f2py_cmd = r"%s\f2py.py" % exe_dir else: f2py_cmd = r"%s\Scripts\f2py.py" % exe_dir code, stdout, stderr = run_command([sys.executable, f2py_cmd, '-v']) success = stdout.strip() == asbytes('2') assert_(success, "Warning: f2py not found in path") else: version = sys.version_info # Python 2.6 'sys.version_info' # is just a tuple, but this changes # in Python 2.7 to have a more user- # friendly interface with version[0] # being the 'major' version and # version[1] being the minor version major = str(version[0]) minor = str(version[1]) f2py_cmds = ('f2py', 'f2py' + major, 'f2py' + major + '.' + minor) success = False for f2py_cmd in f2py_cmds: try: code, stdout, stderr = run_command([f2py_cmd, '-v']) assert_equal(stdout.strip(), asbytes('2')) success = True break except: pass msg = "Warning: neither %s nor %s nor %s found in path" % f2py_cmds assert_(success, msg)
def test_f2py(): # test that we can run f2py script if sys.platform == 'win32': exe_dir = dirname(sys.executable) if exe_dir.endswith('Scripts'): # virtualenv f2py_cmd = r"%s\f2py.py" % exe_dir else: f2py_cmd = r"%s\Scripts\f2py.py" % exe_dir code, stdout, stderr = run_command([sys.executable, f2py_cmd, '-v']) success = stdout.strip() == asbytes('2') assert_(success, "Warning: f2py not found in path") else: # unclear what f2py cmd was installed as, check plain (f2py), # with major version (f2py3), or major/minor version (f2py3.4) code, stdout, stderr = run_command([sys.executable, '-V']) # for some reason, 'python -V' returns version in 'stderr' for # Python 2.x but in 'stdout' for Python 3.x version = (stdout or stderr)[7:].strip() major, minor, revision = version.decode('utf-8').split('.') f2py_cmds = ('f2py', 'f2py' + major, 'f2py' + major + '.' + minor) success = False for f2py_cmd in f2py_cmds: try: code, stdout, stderr = run_command([f2py_cmd, '-v']) assert_equal(stdout.strip(), asbytes('2')) success = True break except: pass msg = "Warning: neither %s nor %s nor %s found in path" % f2py_cmds assert_(success, msg)
def loadbin(self, fobj, offset=0): """Load bin file into internal buffer. Not needed if source set in constructor. This will overwrite addresses without warning if object was already initialized. @param fobj file name or file-like object @param offset starting address offset """ fread = getattr(fobj, "read", None) if fread is None: f = open(fobj, "rb") fread = f.read fclose = f.close else: fclose = None try: self.frombytes(array('B', asbytes(fread())), offset=offset) finally: if fclose: fclose()
def _write_values(self, values): if hasattr(values, 'dtype'): nc_type = REVERSE[values.dtype.char, values.dtype.itemsize] else: types = [ (int, NC_INT), (float, NC_FLOAT), (str, NC_CHAR), ] try: sample = values[0] except TypeError: sample = values for class_, nc_type in types: if isinstance(sample, class_): break typecode, size = TYPEMAP[nc_type] dtype_ = '>%s' % typecode values = asarray(values, dtype=dtype_) self.fp.write(asbytes(nc_type)) if values.dtype.char == 'S': nelems = values.itemsize else: nelems = values.size self._pack_int(nelems) if not values.shape and (values.dtype.byteorder == '<' or (values.dtype.byteorder == '=' and LITTLE_ENDIAN)): values = values.byteswap() self.fp.write(values.tostring()) count = values.size * values.itemsize self.fp.write(b'0' * (-count % 4)) # pad
def save(self, fileobj): """ Save tractogram to a filename or file-like object using TCK format. Parameters ---------- fileobj : string or file-like object If string, a filename; otherwise an open file-like object in binary mode pointing to TCK file (and ready to write from the beginning of the TCK header data). """ # Enforce float32 in little-endian byte order for data. dtype = np.dtype('<f4') header = self.create_empty_header() # Override hdr's fields by those contained in `header`. header.update(self.header) # Keep counts for correcting incoherent fields or warn. nb_streamlines = 0 with Opener(fileobj, mode="wb") as f: # Keep track of the beginning of the header. beginning = f.tell() # Write temporary header that we will update at the end self._write_header(f, header) # Make sure streamlines are in rasmm. tractogram = self.tractogram.to_world(lazy=True) # Assume looping over the streamlines can be done only once. tractogram = iter(tractogram) try: # Use the first element to check # 1) the tractogram is not empty; # 2) quantity of information saved along each streamline. first_item, tractogram = peek_next(tractogram) except StopIteration: # Empty tractogram header[Field.NB_STREAMLINES] = 0 self._finalize_header(f, header, offset=beginning) # Add the EOF_DELIMITER. f.write(self.EOF_DELIMITER.tobytes()) return data_for_streamline = first_item.data_for_streamline if len(data_for_streamline) > 0: keys = ", ".join(data_for_streamline.keys()) msg = ("TCK format does not support saving additional data" " alongside streamlines. Dropping: {}".format(keys)) warnings.warn(msg, DataWarning) data_for_points = first_item.data_for_points if len(data_for_points) > 0: keys = ", ".join(data_for_points.keys()) msg = ("TCK format does not support saving additional data" " alongside points. Dropping: {}".format(keys)) warnings.warn(msg, DataWarning) for t in tractogram: data = np.r_[t.streamline, self.FIBER_DELIMITER] f.write(data.astype(dtype).tobytes()) nb_streamlines += 1 header[Field.NB_STREAMLINES] = nb_streamlines # Add the EOF_DELIMITER. f.write(asbytes(self.EOF_DELIMITER.tobytes())) self._finalize_header(f, header, offset=beginning)
def test_f2py(): # test that we can run f2py script f2py_cmd = 'f2py' + basename(sys.executable)[6:] code, stdout, stderr = run_command([f2py_cmd, '-v']) assert_equal(stdout.strip(), asbytes('2'))
def _decode_record(self, s, line=0): '''Decode one record of HEX file. @param s line with HEX record. @param line line number (for error messages). @raise EndOfFile if EOF record encountered. ''' s = s.rstrip('\r\n') if not s: return # empty line if s[0] == ':': try: bin = array('B', unhexlify(asbytes(s[1:]))) except (TypeError, ValueError): # this might be raised by unhexlify when odd hexascii digits raise HexRecordError(line=line) length = len(bin) if length < 5: raise HexRecordError(line=line) else: raise HexRecordError(line=line) record_length = bin[0] if length != (5 + record_length): raise RecordLengthError(line=line) addr = bin[1] * 256 + bin[2] record_type = bin[3] if not (0 <= record_type <= 5): raise RecordTypeError(line=line) crc = sum(bin) crc &= 0x0FF if crc != 0: raise RecordChecksumError(line=line) if record_type == 0: # data record addr += self._offset for i in range(4, 4 + record_length): if not self._buf.get(addr, None) is None: raise AddressOverlapError(address=addr, line=line) self._buf[addr] = bin[i] addr += 1 # FIXME: addr should be wrapped # BUT after 02 record (at 64K boundary) # and after 04 record (at 4G boundary) elif record_type == 1: # end of file record if record_length != 0: raise EOFRecordError(line=line) raise _EndOfFile elif record_type == 2: # Extended 8086 Segment Record if record_length != 2 or addr != 0: raise ExtendedSegmentAddressRecordError(line=line) self._offset = (bin[4] * 256 + bin[5]) * 16 elif record_type == 4: # Extended Linear Address Record if record_length != 2 or addr != 0: raise ExtendedLinearAddressRecordError(line=line) self._offset = (bin[4] * 256 + bin[5]) * 65536 elif record_type == 3: # Start Segment Address Record if record_length != 4 or addr != 0: raise StartSegmentAddressRecordError(line=line) if self.start_addr: raise DuplicateStartAddressRecordError(line=line) self.start_addr = { 'CS': bin[4] * 256 + bin[5], 'IP': bin[6] * 256 + bin[7], } elif record_type == 5: # Start Linear Address Record if record_length != 4 or addr != 0: raise StartLinearAddressRecordError(line=line) if self.start_addr: raise DuplicateStartAddressRecordError(line=line) self.start_addr = { 'EIP': (bin[4] * 16777216 + bin[5] * 65536 + bin[6] * 256 + bin[7]), }
def _pack_string(self, s): count = len(s) self._pack_int(count) self.fp.write(asbytes(s)) self.fp.write(b'0' * (-count % 4)) # pad
def write_hex_file(self, f, write_start_addr=True): """Write data to file f in HEX format. @param f filename or file-like object for writing @param write_start_addr enable or disable writing start address record to file (enabled by default). If there is no start address in obj, nothing will be written regardless of this setting. """ fwrite = getattr(f, "write", None) if fwrite: fobj = f fclose = None else: fobj = open(f, 'w') fwrite = fobj.write fclose = fobj.close # Translation table for uppercasing hex ascii string. # timeit shows that using hexstr.translate(table) # is faster than hexstr.upper(): # 0.452ms vs. 0.652ms (translate vs. upper) if sys.version_info[0] >= 3: table = bytes(range(256)).upper() else: table = ''.join(chr(i).upper() for i in range(256)) # start address record if any if self.start_addr and write_start_addr: keys = self.start_addr.keys() keys.sort() bin = array('B', asbytes('\0' * 9)) if keys == ['CS', 'IP']: # Start Segment Address Record bin[0] = 4 # reclen bin[1] = 0 # offset msb bin[2] = 0 # offset lsb bin[3] = 3 # rectyp cs = self.start_addr['CS'] bin[4] = (cs >> 8) & 0x0FF bin[5] = cs & 0x0FF ip = self.start_addr['IP'] bin[6] = (ip >> 8) & 0x0FF bin[7] = ip & 0x0FF bin[8] = (-sum(bin)) & 0x0FF # chksum fwrite(':' + asstr(hexlify(bin.tostring()).translate(table)) + '\n') elif keys == ['EIP']: # Start Linear Address Record bin[0] = 4 # reclen bin[1] = 0 # offset msb bin[2] = 0 # offset lsb bin[3] = 5 # rectyp eip = self.start_addr['EIP'] bin[4] = (eip >> 24) & 0x0FF bin[5] = (eip >> 16) & 0x0FF bin[6] = (eip >> 8) & 0x0FF bin[7] = eip & 0x0FF bin[8] = (-sum(bin)) & 0x0FF # chksum fwrite(':' + asstr(hexlify(bin.tostring()).translate(table)) + '\n') else: if fclose: fclose() raise InvalidStartAddressValueError(start_addr=self.start_addr) # data addresses = self._buf.keys() addresses.sort() addr_len = len(addresses) if addr_len: minaddr = addresses[0] maxaddr = addresses[-1] if maxaddr > 65535: need_offset_record = True else: need_offset_record = False high_ofs = 0 cur_addr = minaddr cur_ix = 0 while cur_addr <= maxaddr: if need_offset_record: bin = array('B', asbytes('\0' * 7)) bin[0] = 2 # reclen bin[1] = 0 # offset msb bin[2] = 0 # offset lsb bin[3] = 4 # rectyp high_ofs = int(cur_addr >> 16) b = divmod(high_ofs, 256) bin[4] = b[0] # msb of high_ofs bin[5] = b[1] # lsb of high_ofs bin[6] = (-sum(bin)) & 0x0FF # chksum fwrite(':' + asstr(hexlify(bin.tostring()).translate(table)) + '\n') while True: # produce one record low_addr = cur_addr & 0x0FFFF # chain_len off by 1 chain_len = min(15, 65535 - low_addr, maxaddr - cur_addr) # search continuous chain stop_addr = cur_addr + chain_len if chain_len: ix = bisect_right( addresses, stop_addr, cur_ix, min(cur_ix + chain_len + 1, addr_len)) chain_len = ix - cur_ix # real chain_len # there could be small holes in the chain # but we will catch them by try-except later # so for big continuous files we will work # at maximum possible speed else: chain_len = 1 # real chain_len bin = array('B', asbytes('\0' * (5 + chain_len))) b = divmod(low_addr, 256) bin[1] = b[0] # msb of low_addr bin[2] = b[1] # lsb of low_addr bin[3] = 0 # rectype try: # if there is small holes we'll catch them for i in range(chain_len): bin[4 + i] = self._buf[cur_addr + i] except KeyError: # we catch a hole so we should shrink the chain chain_len = i bin = bin[:5 + i] bin[0] = chain_len bin[4 + chain_len] = (-sum(bin)) & 0x0FF # chksum fwrite(':' + asstr(hexlify(bin.tostring()).translate(table)) + '\n') # adjust cur_addr/cur_ix cur_ix += chain_len if cur_ix < addr_len: cur_addr = addresses[cur_ix] else: cur_addr = maxaddr + 1 break high_addr = int(cur_addr >> 16) if high_addr > high_ofs: break # end-of-file record fwrite(":00000001FF\n") if fclose: fclose()
def save_dfu(ih): """ Save as STMicroelectronics DfuSe file. see UM0391 - DfuSe File Format Specification :param ih: intelhex object """ if args.verbose: print("Saving %s..." % args.target) print(" Device ID: 0x%04x:0x%04x" % (args.vid, args.pid)) print(" Target name: %s" % args.target_name) # Image element # image_data = asbytes(ih.tobinstr()) data = struct.pack( "<II", ih.minaddr(), # dwElementAddress len(image_data), # dwElementSize ) + image_data # Data # Target prefix # szTargetName = args.target_name.encode("ascii") data = struct.pack( "<6sBI255sII", b"Target", # szSignature 0, # bAlternateSetting 1, # bTargetNamed szTargetName, # szTargetName len(data), # dwTargetSize 1 # dwNbElements ) + data # Prefix # data = struct.pack( "<5sBIB", b"DfuSe", # szSignature 0x01, # bVersion, len(data) + 11, # DFUImageSize, 1 # bTargets ) + data # Suffix # data += struct.pack( "<HHHH3sB", 0xFFFF, # bcdDevice args.pid, # idProduct args.vid, # idVendor 0x011a, # bdcDFU b"UFD", # ucDfuSignature 16 # bLength ) dwCRC = ~crc32(data) & 0xFFFFFFFF data += struct.pack( "<I", dwCRC # dwCRC ) try: open(args.target, "wb").write(data) except Exception as e: print(e) exit(1)