def default_cb_fct (self, evt, usrdata): import os from os.path import realpath as _realpath from os.path import basename as _basename libname = evt.fname step = evt.step if libname is None: return 0 n = _basename (_realpath (libname)) pid = os.getpid() def vmem(): from os import sysconf PAGE_SIZE = sysconf ('SC_PAGE_SIZE') # in bytes return PAGE_SIZE if step == 0: self._data[pid][n] = [vmem(), None] else: data = self._data[pid][n] data[1] = vmem() vmem_start = data[0]/1024. vmem_stop = data[1]/1024. dvmem = vmem_stop - vmem_start self.msg.info ( "[%d] loading lib: vmem=(%10.1f + %10.1f) kb [%s]", pid, vmem_start, dvmem, n ) self.out.writerow ([pid, n, vmem_start, vmem_stop, dvmem]) #del self._data[pid][n] return 0
def default_cb_fct(self, evt, usrdata): import os from os.path import realpath as _realpath from os.path import basename as _basename libname = evt.fname step = evt.step if libname is None: return 0 n = _basename(_realpath(libname)) pid = os.getpid() def vmem(): from os import sysconf PAGE_SIZE = sysconf('SC_PAGE_SIZE') # in bytes return PAGE_SIZE if step == 0: self._data[pid][n] = [vmem(), None] else: data = self._data[pid][n] data[1] = vmem() vmem_start = data[0] / 1024. vmem_stop = data[1] / 1024. dvmem = vmem_stop - vmem_start self.msg.info( "[%d] loading lib: vmem=(%10.1f + %10.1f) kb [%s]", pid, vmem_start, dvmem, n) self.out.writerow([pid, n, vmem_start, vmem_stop, dvmem]) #del self._data[pid][n] return 0
def default_cb_fct (self, evt, usrdata): libname = evt.contents.fname step = evt.contents.step if libname is None: return 0 n = _basename (_realpath (libname)) pid = os.getpid() def vmem(): from os import sysconf PAGE_SIZE = sysconf ('SC_PAGE_SIZE') # in bytes from string import split as ssplit m = 0 with open('/proc/self/statm') as f: m = int(ssplit (f.readlines()[0])[0]) return m * PAGE_SIZE # in bytes if step == 0: self._data[pid][n] = [vmem(), None] else: data = self._data[pid][n] data[1] = vmem() vmem_start = data[0]/1024. vmem_stop = data[1]/1024. dvmem = vmem_stop - vmem_start self.msg.info ( "[%d] loading lib: vmem=(%10.1f + %10.1f) kb [%s]", pid, vmem_start, dvmem, n ) self.out.writerow ([pid, n, vmem_start, vmem_stop, dvmem]) #del self._data[pid][n] return 0
def importFromPath(path): """ import all modules in dir except __init__ and places them in modules dict """ spec = _importlib.spec_from_file_location(_basename(path)[:-3], path) mod = _importlib.module_from_spec(spec) spec.loader.exec_module(mod) return mod
def get_python_package_entry_point(package, entry_point): """ Find an CLI entry point from a Python package. Args: package (str): Package name. entry_point (str): Entry point name. Returns: str or None: Path to entry point, or None if nothing found. """ site_packages_path = _dirname(_import_module(package).__path__[0]) # Find package info # Can be a directory ending by ".dist-info" or ".egg-info" with _scandir(site_packages_path) as entries: for entry in entries: if (entry.name.startswith(f'{package}-') and _splitext(entry.name)[1] in ('.dist-info', '.egg-info')): package_info_path = entry.path break else: # Package is not installed or do not have package info return None # Find manifest file # Can be a "RECORD" or a "installed-files.txt" file in package info folder for name in ('RECORD', 'installed-files.txt'): manifest_path = _join(package_info_path, name) if _isfile(manifest_path): break else: # Package do not have manifest file return None # Find entry point relative path in manifest file # Possibles manifest file lines formats: "path\n" or "path,checksum\n" with open(manifest_path, 'rt') as manifest: for line in manifest: entry_point_rel_path = line.strip().split(',', 1)[0] if _basename(entry_point_rel_path) == entry_point: break else: # Entry point is not present in manifest return None # Convert to absolute path # Paths in manifest are relative to site-packages or package info for prefix in (site_packages_path, package_info_path): entry_point_path = _realpath(_join(prefix, entry_point_rel_path)) if _isfile(entry_point_path): return entry_point_path
def _caller3(up): # in .named '''(INTERNAL) Get 3-tuple C{(caller name, file name, line number)}. ''' # sys._getframe(1) ... 'importlib._bootstrap' line 1032, # may throw a ValueError('call stack not deep enough') f = _sys._getframe(up + 1) return ( f.f_code.co_name, # caller name _basename(f.f_code.co_filename), # file name f.f_lineno) # line number
def update_io_registry(wkdir, mpid, iocomp_types=None): """helper method to correctly update the IoRegistry instances """ import os from os.path import join as _join from os.path import basename as _basename from os.path import isabs as _isabs from PyUtils.PoolFile import PoolFileCatalog # ioreg is a dict: # {'iocomp-name' : { 'old-fname' : ['iomode', 'new-fname'] }, ... } ioreg = IoRegistry.instances msg.debug("ioreg::: %s" % ioreg) pfc = PoolFileCatalog() ioreg_items = IoRegistry.instances.iteritems() for iocomp, iodata in ioreg_items: #print "--iocomp,len(iodata)",iocomp, len(iodata) io_items = iodata.iteritems() for ioname, ioval in io_items: # handle logical filenames... #ioname=pfc(ioname) pfc_name = pfc(ioname) if (pfc_name != ioname): ioreg[iocomp][ioname][1] = pfc_name ##print " --iocomp,ioname,ioval",iocomp,ioname,ioval iomode, newname = ioval[0], ioval[1] or ioname if iomode == '<output>': newname = _join( wkdir, "mpid_%s__%s" % (str(mpid).zfill(3), _basename(ioname))) msg.debug("update_io_registry:<output>: newname=%s" % newname) elif iomode == '<input>': if not _isabs(ioname) and not ioname.startswith( "root:") and not ioname.startswith("rfio"): # FIXME: handle URLs/URIs... src = os.path.abspath(_join(os.curdir, ioname)) dst = _join(wkdir, ioname) os.symlink(src, dst) msg.debug( "update_io_registry:<input> created symlink %s for" % dst) else: raise ValueError, "unexpected iomode value: %r" % iomode ioreg[iocomp][ioname][1] = newname pass pass msg.debug("IoRegistry.instances=%s" % IoRegistry.instances) return # update_io_registry
def put(sources, host=None, expiry="24h", archive=False, compression=None, name=None): """ Args: sources (iterable of str): The files or directory to share. archive (bool): If True, create an archive before putting on the share. Automatic if "sources" is a directory or multiples files. expiry (str or int): The download expiry time. In hours, or with a specific time unit by appending: "s" for seconds, "m" for minutes, "h" for hours or "d" for days (For instance: "7d" for 7 days, "30m" for 30 minutes). compression (str): The compression method to use when archiving. Possible values: "gz", "bz2", "xz". Default to value saved in configuration or "gz". host (str): The remote storage host. name (str): Rename the file being put. Returns: str: The shareable URL to the shared object. """ config = _get_host_config(host) expiry = _parse_expiry(expiry) max_expiry = config["max_expiry_sec"] if max_expiry and expiry > max_expiry: raise ValueError(f"Expiry can't be more than {config['max_expiry']}.") if archive or len(sources) > 1 or any(_isdir(path) for path in sources): compression = compression or config["compression"] dst = _set_dst_path(name or f"archive.tar.{compression}", config) with _open(dst, "wb") as fileobj: with _tarfile_open(fileobj=fileobj, mode=f"w:{compression}") as dst_archive: for src in sources: dst_archive.add(src, arcname=_basename(src)) else: dst = _set_dst_path(name or _basename(sources[0]), config) _copy(sources[0], dst) return _shareable_url(dst, expiry)
def update_io_registry(wkdir, mpid, iocomp_types=None): """helper method to correctly update the IoRegistry instances """ import os from os.path import join as _join from os.path import basename as _basename from os.path import isabs as _isabs from PyUtils.PoolFile import PoolFileCatalog # ioreg is a dict: # {'iocomp-name' : { 'old-fname' : ['iomode', 'new-fname'] }, ... } ioreg = IoRegistry.instances msg.debug("ioreg::: %s" % ioreg) pfc = PoolFileCatalog() ioreg_items = IoRegistry.instances.iteritems() for iocomp,iodata in ioreg_items: #print "--iocomp,len(iodata)",iocomp, len(iodata) io_items = iodata.iteritems() for ioname,ioval in io_items: # handle logical filenames... #ioname=pfc(ioname) pfc_name = pfc(ioname) if (pfc_name != ioname): ioreg[iocomp][ioname][1]=pfc_name ##print " --iocomp,ioname,ioval",iocomp,ioname,ioval iomode,newname = ioval[0], ioval[1] or ioname if iomode == '<output>': newname = _join (wkdir, "mpid_%s__%s"%(str(mpid).zfill(3), _basename(ioname))) msg.debug ("update_io_registry:<output>: newname=%s" % newname) elif iomode == '<input>': if not _isabs(ioname) and not ioname.startswith("root:") and not ioname.startswith("rfio"): # FIXME: handle URLs/URIs... src = os.path.abspath(_join(os.curdir, ioname)) dst = _join(wkdir, ioname) os.symlink(src, dst) msg.debug( "update_io_registry:<input> created symlink %s for" % dst) else: raise ValueError, "unexpected iomode value: %r"%iomode ioreg[iocomp][ioname][1] = newname pass pass msg.debug( "IoRegistry.instances=%s" % IoRegistry.instances ) return # update_io_registry
def _set_dst_path(src, config): """ Set the storage destination path. Args: src (str): Source path of the file to share. config (dict): The remote storage host configuration. Returns: str: Destination path of the file on the storage. """ dst = [config["host"], config["files_directory"]] if config["random_prefix"]: dst.append(_token_hex()) dst.append(_basename(src)) return _join(*dst)
def _build_bitmap_data(): ''' Build an SFrame from 10 saved drawings. ''' from os.path import join as _join, realpath as _realpath from os.path import splitext as _splitext, basename as _basename from os.path import dirname as _dirname drawings_dir = _join(_dirname(_realpath(__file__)), "drawings") sf = _tc.image_analysis.load_images(drawings_dir, with_path=True) sf = sf.rename({"image": "drawing", "path": "label"}) sf["label"] = sf["label"].apply( lambda filepath: _splitext(_basename(filepath))[0][:-1] # Extract the class name from the filename, "check1.png" -> "check" # [:-1] is to get "check" out of "check1" ) return sf
def get(url, output=".", extract=False): """ Get a file or archive from an URL. Args: output (str): Output file or directory path. url (str): Input URL. extract (bool): If True, extract archive. """ response = _request("GET", url, stream=True) response.raise_for_status() if extract: with _tarfile_open(fileobj=response.raw) as archive: archive.extractall(output) else: if not _isfile(output): output = _join(output, _basename(_urlparse(url).path)) with open(output, "wb") as dest: for chunk in response.iter_content(): dest.write(chunk)
def _interesting_parts_via_path(path): # Split long path up in to dirname (discarded) and basename from os.path import basename as _basename, splitext as _splitext basename = _basename(path) # Split basename up into basename head and (discarded) extension basename_head, ext = _splitext(basename) assert '.md' == ext # (in some crazy future this might change and that's okay) # Split basename head up into entity identifier and the rest md = _re.match(r'(?:(\d+(?:\.(?:\d+|[A-Z]))*)[.-])?(.+)', basename_head) if not md: xx(f"regex oops: {basename_head!r}") eid, rest = md.groups() # Split the rest up into "title pieces" title_pieces = tuple(md[0] for md in _re.finditer(r'\w+', rest)) return eid, title_pieces
def render_template(src, dst, show=True, **kwargs): """ Render a file from a template using Jinja2. Args: src (str): Source template. dst (str): Destination file. show (bool): If true, print result. kwargs: Template arguments. """ from jinja2 import Environment, FileSystemLoader env = Environment(loader=FileSystemLoader(_dirname(src))) template = env.get_template(_basename(src)) rendered = template.render(**kwargs) if show: print("\033[34m== START RENDERED ==\033[30m\n" f"{rendered}" "\n\033[34m== END RENDERED ==\033[30m") with open(dst, "wt") as file: file.write(rendered)
def default_cb_fct (self, evt, usrdata): import os from os.path import realpath as _realpath from os.path import basename as _basename libname = evt.contents.fname step = evt.contents.step if libname is None: return 0 n = _basename (_realpath (libname)) pid = os.getpid() def vmem(): from os import sysconf PAGE_SIZE = sysconf ('SC_PAGE_SIZE') # in bytes from string import split as ssplit m = 0 from sys import platform with open('/proc/self/statm') as f: m = int(ssplit (f.readlines()[0])[0]) return m * PAGE_SIZE # in bytes if step == 0: self._data[pid][n] = [vmem(), None] else: data = self._data[pid][n] data[1] = vmem() vmem_start = data[0]/1024. vmem_stop = data[1]/1024. dvmem = vmem_stop - vmem_start self.msg.info ( "[%d] loading lib: vmem=(%10.1f + %10.1f) kb [%s]", pid, vmem_start, dvmem, n ) self.out.writerow ([pid, n, vmem_start, vmem_stop, dvmem]) #del self._data[pid][n] return 0
def zip_files(name_of_zip, files_to_zip): """Zip files. Examples -------- .. code-block:: robotframework ZipFiles my_zip_file rabbit.txt ZipFiles my_zip_file_2 dog.txt ZipFiles my_zip_file_3 rabbit.txt, dog.txt ZipFiles my_zip_file_4 C:/Users/pace/secrets/cat.txt ZipFiles my_zip_file_5 C:/Users/pace/secrets/cat.txt, C:/automation/kangaroo.txt Parameters ---------- name_of_zip : str Name of the zip file created. files_to_zip : str Files to be zipped, separated by "," in case of multiple files. """ if not name_of_zip.endswith('.zip'): name_of_zip += '.zip' files = files_to_zip.split(',') try: with ZipFile(name_of_zip, 'w') as zipped: for file in files: file = download.get_path(file.strip()) if os.path.isdir(file): for root, _, files2 in os.walk(file): for file2 in files2: zipped.write(os.path.join(root, file2)) else: zipped.write(file, _basename(file)) except OSError as e: raise QWebValueError('\nFile name "{}" contained illegal characters.' '\nError message: {}'.format(name_of_zip, str(e))) from e logger.info('Zipped files {} into the file {}' .format(str(files), name_of_zip), also_console=True)
def basename(value): return _basename(value)
class FpgaDriver(_FpgaDriverBase): """ Generates functions to use XRT with accelize_drm.DrmManager. Args: fpga_slot_id (int): FPGA slot ID. fpga_image (str): Path to ".xclbin" binary to use to program FPGA. drm_ctrl_base_addr (int): DRM Controller base address. log_dir (path-like object): directory where XRT will output log file. """ _name = _match(r'_(.+)\.py', _basename(__file__)).group(1) _reglock = _Lock() @staticmethod def _get_xrt_lib(): """ Detect XRT installation path: """ for prefix in (_environ.get("XILINX_XRT", "/opt/xilinx/xrt"), '/usr', '/usr/local'): if _isfile(_join(prefix, 'bin', 'xbutil')): return prefix raise RuntimeError('Unable to find Xilinx XRT') @staticmethod def _get_driver(): """ Get FPGA driver Returns: ctypes.CDLL: FPGA driver. """ xrt_path = FpgaDriver._get_xrt_lib() if _isfile(_join(xrt_path, 'lib', 'libxrt_aws.so')): print('Loading XRT API library for AWS targets') fpga_library = _cdll.LoadLibrary( _join(xrt_path, 'lib', 'libxrt_aws.so')) elif _isfile(_join(xrt_path, 'lib', 'libxrt_core.so')): print('Loading XRT API library for Xilinx targets') fpga_library = _cdll.LoadLibrary( _join(xrt_path, 'lib', 'libxrt_core.so')) else: raise RuntimeError('Unable to find Xilinx XRT Library') return fpga_library @staticmethod def _get_xbutil(): xrt_path = FpgaDriver._get_xrt_lib() _xbutil_path = _join(xrt_path, 'bin', 'awssak') if not _isfile(_xbutil_path): _xbutil_path = _join(xrt_path, 'bin', 'xbutil') if not _isfile(_xbutil_path): raise RuntimeError('Unable to find Xilinx XRT Board Utility') return _xbutil_path @property def _xbutil(self): return self._get_xbutil() def _get_lock(self): """ Get a lock on the FPGA driver """ def create_lock(): return XrtLock(self) return create_lock def _clear_fpga(self): """ Clear FPGA """ clear_fpga = _run( ['fpga-clear-local-image', '-S', str(self._fpga_slot_id)], stderr=_STDOUT, stdout=_PIPE, universal_newlines=True, check=False) if clear_fpga.returncode: raise RuntimeError(clear_fpga.stdout) print('FPGA cleared') def _program_fpga(self, fpga_image): """ Program the FPGA with the specified image. Args: fpga_image (str): FPGA image. """ # Vitis does not reprogram a FPGA that has already the bitstream. # So to force it we write another bitstream first. clear_image = _join(SCRIPT_DIR, 'clear.awsxclbin') load_image = _run([ self._xbutil, 'program', '-d', str(self._fpga_slot_id), '-p', clear_image ], stderr=_STDOUT, stdout=_PIPE, universal_newlines=True, check=False) if load_image.returncode: raise RuntimeError(load_image.stdout) print('Cleared AWS XRT slot #%d' % self._fpga_slot_id) # Now load the real image fpga_image = _realpath(_fsdecode(fpga_image)) load_image = _run([ self._xbutil, 'program', '-d', str(self._fpga_slot_id), '-p', fpga_image ], stderr=_STDOUT, stdout=_PIPE, universal_newlines=True, check=False) if load_image.returncode: raise RuntimeError(load_image.stdout) print('Programmed AWS XRT slot #%d with FPGA image %s' % (self._fpga_slot_id, fpga_image)) def _reset_fpga(self): """ Reset FPGA including FPGA image. """ reset_image = _run( [self._xbutil, 'reset', '-d', str(self._fpga_slot_id)], stderr=_STDOUT, stdout=_PIPE, universal_newlines=True, check=False) if reset_image.returncode: raise RuntimeError(reset_image.stdout) def _init_fpga(self): """ Initialize FPGA handle with driver library. """ # Find all devices xcl_probe = self._fpga_library.xclProbe xcl_probe.restype = _c_uint # Devices count if xcl_probe() < 1: raise RuntimeError("xclProbe does not found devices") # Open device xcl_open = self._fpga_library.xclOpen xcl_open.restype = _POINTER(_c_void_p) # Device handle xcl_open.argtypes = ( _c_uint, # deviceIndex _c_char_p, # logFileName _c_int, # level ) log_file = _join(self._log_dir, 'slot_%d_xrt.log' % self._fpga_slot_id) device_handle = xcl_open( self._fpga_slot_id, log_file.encode(), 3 # XCL_ERROR ) if not device_handle: raise RuntimeError("xclOpen failed to open device") self._fpga_handle = device_handle def _get_read_register_callback(self): """ Read register callback. Returns: function: Read register callback """ xcl_read = self._fpga_library.xclRead xcl_read.restype = _c_size_t # read size or error code xcl_read.argtypes = ( _c_void_p, # handle _c_int, # space _c_uint64, # offset _c_void_p, # hostBuf _c_size_t # size ) self._fpga_read_register = xcl_read def read_register(register_offset, returned_data, driver=self): """ Read register. Args: register_offset (int): Offset returned_data (int pointer): Return data. driver (accelize_drm.fpga_drivers._aws_xrt.FpgaDriver): Keep a reference to driver. """ with driver._fpga_read_register_lock(): size_or_error = driver._fpga_read_register( driver._fpga_handle, 2, # XCL_ADDR_KERNEL_CTRL driver._drm_ctrl_base_addr + register_offset, returned_data, 4 # 4 bytes ) # Return 0 return code if read size else return error code return size_or_error if size_or_error != 4 else 0 return read_register def _get_write_register_callback(self): """ Write register callback. Returns: function: Write register callback """ xcl_write = self._fpga_library.xclWrite xcl_write.restype = _c_size_t # written size or error code xcl_write.argtypes = ( _c_void_p, # handle _c_int, # space _c_uint64, # offset _c_char_p, # hostBuf _c_size_t # size ) self._fpga_write_register = xcl_write def write_register(register_offset, data_to_write, driver=self): """ Write register. Args: register_offset (int): Offset data_to_write (int): Data to write. driver (accelize_drm.fpga_drivers._aws_xrt.FpgaDriver): Keep a reference to driver. """ with driver._fpga_write_register_lock(): size_or_error = driver._fpga_write_register( driver._fpga_handle, 2, # XCL_ADDR_KERNEL_CTRL driver._drm_ctrl_base_addr + register_offset, data_to_write.to_bytes(4, byteorder="little"), 4 # 4 bytes ) # Return 0 return code if written size else return error code return size_or_error if size_or_error != 4 else 0 return write_register
def to_basename(*paths): return _basename(to_abspath(*paths))
# Importing the dependencies from os.path import dirname as _dirname, basename as _basename,\ isfile as _isfile import glob as _glob exec('\n'.join( map(lambda name: "from ." + name + " import *", [ _basename(f)[:-3] for f in _glob.glob(_dirname(__file__) + "/*.py") if _isfile(f) and not _basename(f).startswith('_') ])))
class FpgaDriver(_FpgaDriverBase): """ Generates functions to use AWS FPGA F1 with accelize_drm.DrmManager. Args: fpga_slot_id (int): FPGA slot ID. fpga_image (str): AGFI or AFI to use to program FPGA. drm_ctrl_base_addr (int): DRM Controller base address. log_dir (path-like object): Unused with this driver. """ _name = _match(r'_(.+)\.py', _basename(__file__)).group(1) @staticmethod def _get_driver(): """ Get FPGA driver Returns: ctypes.CDLL: FPGA driver. """ # Load AWS FPGA library fpga_library = _cdll.LoadLibrary("libfpga_mgmt.so") fpga_pci_init = fpga_library.fpga_pci_init fpga_pci_init.restype = _c_int # return code if fpga_pci_init(): raise RuntimeError('Unable to initialize the "fpga_pci" library') return fpga_library @staticmethod def _get_lock(): """ Get a lock on the FPGA driver """ return _Lock def _clear_fpga(self): """ Clear FPGA """ clear_fpga = _run( ['fpga-clear-local-image', '-S', str(self._fpga_slot_id)], stderr=_STDOUT, stdout=_PIPE, universal_newlines=True, check=False) if clear_fpga.returncode: raise RuntimeError(clear_fpga.stdout) def _program_fpga(self, fpga_image): """ Program the FPGA with the specified image. Args: fpga_image (str): FPGA image. """ retries = 3 while True: load_image = _run([ 'fpga-load-local-image', '-S', str(self._fpga_slot_id), '-I', fpga_image ], stderr=_STDOUT, stdout=_PIPE, universal_newlines=True, check=False) if load_image.returncode == 0: break elif ("-110" in load_image.stdout) and (retries != 0): retries -= 1 print('Retry programming') else: raise RuntimeError(load_image.stdout) def _reset_fpga(self): """ Reset FPGA including FPGA image. """ reset_image = _run( ['fpga-clear-local-image', '-S', str(self._fpga_slot_id), '-H'], stderr=_STDOUT, stdout=_PIPE, universal_newlines=True, check=False) if reset_image.returncode: raise RuntimeError(reset_image.stdout) def _init_fpga(self): """ Initialize FPGA handle with driver library. """ # Set FPGA handle to default value self._fpga_handle = _c_int(-1) # PCI_BAR_HANDLE_INIT # Attach FPGA fpga_pci_attach = self._fpga_library.fpga_pci_attach fpga_pci_attach.restype = _c_int # return code fpga_pci_attach.argtypes = ( _c_int, # slot_id _c_int, # pf_id _c_int, # bar_id _c_uint32, # flags _POINTER(_c_int) # handle ) if fpga_pci_attach( self._fpga_slot_id, 0, # FPGA_APP_PF 0, # APP_PF_BAR0 0, _byref(self._fpga_handle)): raise RuntimeError("Unable to attach to the AFI on slot ID %s" % self._fpga_slot_id) def _get_read_register_callback(self): """ Read register callback. Returns: function: Read register callback """ fpga_pci_peek = self._fpga_library.fpga_pci_peek fpga_pci_peek.restype = _c_int # return code fpga_pci_peek.argtypes = ( _c_int, # handle _c_uint64, # offset _POINTER(_c_uint32) # value ) self._fpga_read_register = fpga_pci_peek def read_register(register_offset, returned_data, driver=self): """ Read register. Args: register_offset (int): Offset returned_data (int pointer): Return data. driver (accelize_drm.fpga_drivers._aws_f1.FpgaDriver): Keep a reference to driver. """ with driver._fpga_read_register_lock(): return driver._fpga_read_register( driver._fpga_handle, driver._drm_ctrl_base_addr + register_offset, returned_data) return read_register def _get_write_register_callback(self): """ Write register callback. Returns: function: Write register callback """ fpga_pci_poke = self._fpga_library.fpga_pci_poke fpga_pci_poke.restype = _c_int # return code fpga_pci_poke.argtypes = ( _c_int, # handle _c_uint64, # offset _c_uint32 # value ) self._fpga_write_register = fpga_pci_poke def write_register(register_offset, data_to_write, driver=self): """ Write register. Args: register_offset (int): Offset data_to_write (int): Data to write. driver (accelize_drm.fpga_drivers._aws_f1.FpgaDriver): Keep a reference to driver. """ with driver._fpga_write_register_lock(): return driver._fpga_write_register( driver._fpga_handle, driver._drm_ctrl_base_addr + register_offset, data_to_write) return write_register
from os.path import dirname as _dirname, basename as _basename, isfile as _isfile import glob as _glob exec("\n".join( map( lambda name: "from ." + name + " import *", [ _basename(f)[:-3] for f in _glob.glob(_dirname(__file__) + "/*.py") if _isfile(f) and not _basename(f).startswith("_") ], )))
from os.path import basename as _basename from os import remove as _remove form_name = _basename(__file__).split('.')[0][4:] + ':id' is_form = {'chembl:id': form_name, 'ChEMBL:id': form_name} info = ["", ""] with_topology = True with_trajectory = False def to_UniProt_id(form_id): pass def extract(item, atom_indices='all', structure_indices='all'): if (atom_indices is 'all') and (structure_indices is 'all'): return item else: raise NotImplementedError def copy(item): raise NotImplementedError ###### Get
def Compile(self, Path, **Kwargs): #self.SetCompilerOptions(Arguments) ccntr = _util.Dmfc_Container() # Load and parse DefleMask module dmf = dmfm.Dmf() dmf.Load(Path) ccntr.DmfObj = dmf # Set PCM info pcminfo = _util.PcmInfo() pcminfo.Analyse(dmf) if ('UseExpcm' in Kwargs): ccntr.USES_EXPCM = Kwargs['UseExpcm'] else: ccntr.USES_EXPCM = pcminfo.Uses_Expcm if (ccntr.USES_EXPCM): ccntr.AMOUNT_CHANNELS = 13 else: ccntr.AMOUNT_CHANNELS = 9 # Create mdx object mdx = pymdx.Mdx(ccntr.USES_EXPCM) ccntr.MdxObj = mdx # Set header data mdx.Header.Title = dmf.Header.SongName + " - " + dmf.Header.SongAuthor mdx.Header.PdxFilename = "outre.pdx" # Set tones for c, ins in enumerate(dmf.Instruments): tone = _util.Set_MDX_Tone(ins) tone.ToneId = c mdx.Tones.append(tone) # Set speed related variables ccntr.TIME_BASE = dmf.Module.TimeBase + 1 if (dmf.Module.UseCustomHz): ccntr.REFRESH_RATE = dmf.Module.CustomHz else: if (dmf.Module.Framemode == 0): ccntr.REFRESH_RATE = 50 elif (dmf.Module.Framemode == 1): ccntr.REFRESH_RATE = 60 else: raise Exception # Prepare the container ccntr.InitList() routine1.Dmfc_Parser().Compiler(ccntr) uxobj = uxc_util.UX_CompiledObj() errors = False for channel in ccntr.Channels: if (channel.Errors != []): uxobj.Errors.append(channel.Errors) errors = True if not (errors): ccntr.Export() uxobj.MdxObj = ccntr.MdxObj #mdx.DataTracks[0].Add.Repeat_End() pcminfo.Uses_Pcm = False ###### # PCM compiling if (pcminfo.Uses_Pcm): ccntr.MdxObj.Header.PdxFilename = _basename(Path).split('.')[0] pdx = pymdx.Pdx() for i, sample in enumerate(dmf.Samples): pdx.Banks[0].Samples[i].Rate = sample.Rate if sample.Bits == 8: pdx.Banks[0].Samples[ i].Encoding = pymdx._misc._encoding.SAMPLE_ENCODING.LPCM_8 elif sample.Bits == 16: pdx.Banks[0].Samples[ i].Encoding = pymdx._misc._encoding.SAMPLE_ENCODING.LPCM_16 else: raise Exception pdx.Banks[0].Samples[ i].EncodeTo = pymdx._misc._encoding.SAMPLE_ENCODING.ADPCM_OKI pdx.Banks[0].Samples[i].Data = sample.Data # errors / warnings uxobj.PdxObj = pdx return uxobj
from os.path import dirname as _dirname, basename as _basename, isfile as _isfile import glob as _glob exec('\n'.join(map(lambda name: "from ." + name + " import *", [_basename(f)[:-3] for f in _glob.glob(_dirname(__file__) + "/*.py") \ if _isfile(f) and not _basename(f).startswith('_')])))
from reskit import util from reskit import weather from reskit import economic # Add useful paths for testing and stuff from collections import OrderedDict as _OrderedDict from glob import glob as _glob from os.path import join as _join, dirname as _dirname, basename as _basename _TEST_DATA_ = _OrderedDict() for f in _glob(_join(_dirname(__file__), "test", "data", "*")): _TEST_DATA_[_basename(f)] = f
def basename(path): return _basename(path)
from os.path import basename as _basename form_name = _basename(__file__).split('.')[0].split('_')[-1] is_form = {'gpickle': form_name, 'gpickle.gz': form_name} def to_native_Network(item): from ..classes.api_networkx_Graph import to_native_Network as _networkx_Graph_to_native_Network tmp_form_aux = to_networkx_Graph(item) tmp_form = _networkx_Graph_to_native_Network(tmp_form_aux) del (_networkx_Graph_to_native_Network, tmp_form_aux) return tmp_form def to_native_KineticNetwork(item): pass def to_native_PotentialEnergyNetwork(item): from ..classes.api_networkx_Graph import to_native_PotentialEnergyNetwork as \ _networkx_Graph_to_native_PotentialEnergyNetwork tmp_form_aux = to_networkx_Graph(item) tmp_form = _networkx_Graph_to_native_PotentialEnergyNetwork(tmp_form_aux) del (_networkx_Graph_to_native_PotentialEnergyNetwork, tmp_form_aux) return tmp_form def to_networkx(item): return to_networkx_Graph(item)
class FpgaDriver(_FpgaDriverBase): """ Generates functions to use XRT with accelize_drm.DrmManager. Args: fpga_slot_id (int): FPGA slot ID. fpga_image (str): Path to ".xclbin" binary to use to program FPGA. drm_ctrl_base_addr (int): DRM Controller base address. log_dir (path-like object): directory where XRT will output log file. """ _name = _match(r'_(.+)\.py', _basename(__file__)).group(1) _reglock = _Lock() @staticmethod def _get_xrt_lib(): """ Detect XRT installation path: """ for prefix in (_environ.get("XILINX_XRT", "/opt/xilinx/xrt"), '/usr', '/usr/local'): if _isfile(_join(prefix, 'bin', 'xbutil')): return prefix raise RuntimeError('Unable to find Xilinx XRT') @staticmethod def _get_driver(): """ Get FPGA driver Returns: ctypes.CDLL: FPGA driver. """ xrt_path = FpgaDriver._get_xrt_lib() if _isfile(_join(xrt_path, 'lib', 'libxrt_aws.so')): print('Loading XRT API library for AWS targets') fpga_library = _cdll.LoadLibrary( _join(xrt_path, 'lib', 'libxrt_aws.so')) elif _isfile(_join(xrt_path, 'lib', 'libxrt_core.so')): print('Loading XRT API library for Xilinx targets') fpga_library = _cdll.LoadLibrary( _join(xrt_path, 'lib', 'libxrt_core.so')) else: raise RuntimeError('Unable to find Xilinx XRT Library') return fpga_library @staticmethod def _get_xbutil(): xrt_path = FpgaDriver._get_xrt_lib() _xbutil_path = _join(xrt_path, 'bin', 'awssak') if not _isfile(_xbutil_path): _xbutil_path = _join(xrt_path, 'bin', 'xbutil') if not _isfile(_xbutil_path): raise RuntimeError('Unable to find Xilinx XRT Board Utility') return _xbutil_path @property def _xbutil(self): return self._get_xbutil() def _get_lock(self): """ Get a lock on the FPGA driver """ return _Lock def _clear_fpga(self): """ Clear FPGA """ ''' clear_fpga = _run( ['fpga-clear-local-image', '-S', str(self._fpga_slot_id)], stderr=_STDOUT, stdout=_PIPE, universal_newlines=True, check=False) if clear_fpga.returncode: raise RuntimeError(clear_fpga.stdout) ''' print('FPGA cleared') def _program_fpga(self, fpga_image): """ Program the FPGA with the specified image. Args: fpga_image (str): FPGA image. """ ''' # Vitis does not reprogram a FPGA that has already the bitstream. # So to force it we write another bitstream first. clear_image = _join(SCRIPT_DIR, 'clear.awsxclbin') load_image = _run( [self._xbutil, 'program', '-d', str(self._fpga_slot_id), '-p', clear_image], stderr=_STDOUT, stdout=_PIPE, universal_newlines=True, check=False) if load_image.returncode: raise RuntimeError(load_image.stdout) # Now load the real image fpga_image = _realpath(_fsdecode(fpga_image)) load_image = _run( [self._xbutil, 'program', '-d', str(self._fpga_slot_id), '-p', fpga_image], stderr=_STDOUT, stdout=_PIPE, universal_newlines=True, check=False) if load_image.returncode: raise RuntimeError(load_image.stdout) ''' # Init global specific variables self.shm_pages = list() self.ctrl_sw_exec = None # Start Controller SW fpga_image = '' if not _isfile(fpga_image): pass # raise RuntimeError('Controller SW executable path is invald: ', fpga_image) # self.ctrl_sw_exec = Popen([self.ctrl_sw_exec, self._fpga_image], shell=False, stdout=_PIPE, stderr=_STDOUT) print('Programmed AWS SoM with', self.ctrl_sw_exec) def _reset_fpga(self): """ Reset FPGA including FPGA image. """ ''' reset_image = _run( [self._xbutil, 'reset', '-d', str(self._fpga_slot_id)], stderr=_STDOUT, stdout=_PIPE, universal_newlines=True, check=False) if reset_image.returncode: raise RuntimeError(reset_image.stdout) ''' pass def _init_fpga(self): """ Initialize FPGA handle with driver library. """ ''' # Find all devices xcl_probe = self._fpga_library.xclProbe xcl_probe.restype = _c_uint # Devices count if xcl_probe() < 1: raise RuntimeError("xclProbe does not found devices") # Open device xcl_open = self._fpga_library.xclOpen xcl_open.restype = _POINTER(_c_void_p) # Device handle xcl_open.argtypes = ( _c_uint, # deviceIndex _c_char_p, # logFileName _c_int, # level ) log_file = _join(self._log_dir, 'slot_%d_xrt.log' % self._fpga_slot_id) device_handle = xcl_open( self._fpga_slot_id, log_file.encode(), 3 # XCL_ERROR ) if not device_handle: raise RuntimeError("xclOpen failed to open device") self._fpga_handle = device_handle ''' # Connect to Shared Memory self.shm_pages = list() for i in range(1, 7): key = ipc.ftok(SHM_PATH, i) shm = ipc.SharedMemory(key, 0, 0) shm.attach(0, 0) self.shm_pages.append(shm) print('Connected to Shared Memory') def _uninit_fpga(self): import signal # Release access to shared memory for shm in self.shm_pages: shm.detach() print('Disconnected from Shared Memory') if self.ctrl_sw_exec: self.ctrl_sw_exec.send_signal(signal.SIGINT) self.ctrl_sw_exec.wait() print('Terminate DRM') def int_to_bytes(x: int) -> bytes: return x.to_bytes((x.bit_length() + 7) // 8, 'little', signed=False) def int_from_bytes(xbytes: bytes) -> int: return int.from_bytes(xbytes, 'big') def _get_read_register_callback(self): """ Read register callback. Returns: function: Read register callback """ ''' xcl_read = self._fpga_library.xclRead xcl_read.restype = _c_size_t # read size or error code xcl_read.argtypes = ( _c_void_p, # handle _c_int, # space _c_uint64, # offset _c_void_p, # hostBuf _c_size_t # size ) self._fpga_read_register = xcl_read ''' def read_register(register_offset, returned_data, driver=self): """ Read register. Args: register_offset (int): Offset returned_data (int pointer): Return data. driver (accelize_drm.fpga_drivers._aws_xrt.FpgaDriver): Keep a reference to driver. """ with driver._fpga_read_register_lock(): if register_offset >= 0x10000: ''' size_or_error = driver._fpga_read_register( driver._fpga_handle, 2, # XCL_ADDR_KERNEL_CTRL driver._drm_ctrl_base_addr + register_offset, returned_data, 4 # 4 bytes ) ret = size_or_error if size_or_error != 4 else 0 ''' return 0 raise RuntimeError( 'Reading from Activator is not supported in Emu-HW') else: page_index = bytes_to_int(driver.shm_pages[0].read(4, 0)) if register_offset == 0: reg_value = page_index else: shm = driver.shm_pages[page_index] reg_value = bytes_to_int(shm.read(4, register_offset)) # print('Read @%08X: 0x%08X' % (register_offset, reg_value)) returned_data.contents.value = reg_value ret = 0 return ret return read_register def _get_write_register_callback(self): """ Write register callback. Returns: function: Write register callback """ xcl_write = self._fpga_library.xclWrite xcl_write.restype = _c_size_t # written size or error code xcl_write.argtypes = ( _c_void_p, # handle _c_int, # space _c_uint64, # offset _c_char_p, # hostBuf _c_size_t # size ) self._fpga_write_register = xcl_write def write_register(register_offset, data_to_write, driver=self): """ Write register. Args: register_offset (int): Offset data_to_write (int): Data to write. driver (accelize_drm.fpga_drivers._aws_xrt.FpgaDriver): Keep a reference to driver. """ with driver._fpga_read_register_lock(): if register_offset >= 0x10000: ''' size_or_error = driver._fpga_write_register( driver._fpga_handle, 2, # XCL_ADDR_KERNEL_CTRL driver._drm_ctrl_base_addr + register_offset, data_to_write.to_bytes(4, byteorder="little"), 4 # 4 bytes ) ret = size_or_error if size_or_error != 4 else 0 ''' return 0 raise RuntimeError( 'Writing to Activator is not supported in Emu-HW') else: page_index = bytes_to_int(driver.shm_pages[0].read(4, 0)) if register_offset == 0: driver.shm_pages[0].write(int_to_bytes(data_to_write), 0) else: shm = driver.shm_pages[page_index] shm.write(int_to_bytes(data_to_write), register_offset) # print('Wrote @%08X: 0x%08X' % (register_offset, data_to_write)) ret = 0 # Return 0 return code if written size else return error code return ret return write_register
) # import the utilities import geokit.util import geokit.srs import geokit.geom import geokit.raster import geokit.vector # import the main objects from geokit.core.location import Location, LocationSet from geokit.core.extent import Extent from geokit.core.regionmask import RegionMask # import vidualizing functions to top level since they are from geokit.core.util import drawImage from geokit.core.geom import drawGeoms from geokit.core.raster import drawRaster, drawSmopyMap # import the special algorithms import geokit.algorithms # Add useful paths for testing and stuff from collections import OrderedDict as _OrderedDict from glob import glob as _glob _test_data_ = _OrderedDict() for f in _glob(_join(_dirname(__file__), "test", "data", "*")): _test_data_[_basename(f)] = f
from os.path import basename as _basename from openmm.app.modeller import Modeller as _openmm_Modeller form_name = _basename(__file__).split('.')[0].replace('api_', '').replace('_', '.') is_form = {_openmm_Modeller: form_name, 'openmm.Modeller': form_name} def to_openmm(item): return item def to_mdtraj_Trajectory(item): # mdtraj uses nanometers: see http://mdtraj.org/development/api/generated/mdtraj.Trajectory.html from openmm.unit import nanometers as _nanometers from numpy import array as _nparray tmp_attribute = item.value_in_units(_nanometers) tmp_attribute = _nparray(tmp_attribute) return tmp_item
class FpgaDriver(_FpgaDriverBase): """ Generates functions to use XRT with accelize_drm.DrmManager. Args: fpga_slot_id (int): FPGA slot ID. fpga_image (str): Path to ".xclbin" binary to use to program FPGA. drm_ctrl_base_addr (int): DRM Controller base address. log_dir (path-like object): directory where XRT will output log file. """ _name = _match(r'_(.+)\.py', _basename(__file__)).group(1) _reglock = _Lock() @staticmethod def _get_xrt_lib(): """ Detect XRT installation path: """ prefix = '/usr' if _isfile(_join(prefix, 'bin', 'xbutil')): return prefix raise RuntimeError('Unable to find Xilinx XRT') @staticmethod def _get_driver(): """ Get FPGA driver Returns: ctypes.CDLL: FPGA driver. """ xrt_path = FpgaDriver._get_xrt_lib() if _isfile(_join(xrt_path, 'lib', 'libxrt_core.so')): print('Loading XRT API library for SoM targets') fpga_library = _cdll.LoadLibrary( _join(xrt_path, 'lib', 'libxrt_core.so')) else: raise RuntimeError('Unable to find Xilinx XRT Library') return fpga_library @staticmethod def _get_xbutil(): xrt_path = FpgaDriver._get_xrt_lib() _xbutil_path = _join(xrt_path, 'bin', 'xmutil') if not _isfile(_xbutil_path): raise RuntimeError('Unable to find Xilinx XRT Board Utility') return _xbutil_path @property def _xmutil(self): return self._get_xbutil() def _get_lock(self): """ Get a lock on the FPGA driver """ def create_lock(): return XrtLock(self) return create_lock def _clear_fpga(self): """ Clear FPGA """ clear_fpga = _run([self._xmutil, 'unloadapp'], stderr=_STDOUT, stdout=_PIPE, universal_newlines=True, check=False) if clear_fpga.returncode: raise RuntimeError(clear_fpga.stdout) print('FPGA cleared') def _program_fpga(self, fpga_image): """ Program the FPGA with the specified image. Args: fpga_image (str): FPGA image. """ # Must clear the FPGA first #self._clear_fpga() # Now load the bitstream image_name = splitext(fpga_image)[0] load_image = _run([self._xmutil, 'loadapp', image_name], stderr=_STDOUT, stdout=_PIPE, universal_newlines=True, check=False) if load_image.returncode or search(r'error', load_image.stdout): raise RuntimeError(load_image.stdout) print('Programmed SoM XRT with FPGA image %s' % fpga_image) def _reset_fpga(self): """ Reset FPGA including FPGA image. """ list_apps = _run([self._xmutil, 'listapps'], stderr=_STDOUT, stdout=_PIPE, universal_newlines=True, check=False) if list_apps.returncode or search(r'error', list_apps.stdout): raise RuntimeError(list_apps.stdout) image_name = '' for line in list_apps.stdout.splitlines(): m = _match(r'\s*(.+?)\s+', line) if m: image_name = m.group(1) if not image_name: print('No loaded bitstream to reset') return self._program_fpga(image_name + ".som") def _init_fpga(self): """ Initialize FPGA handle with XRT and OpenCL libraries. """ image_name = splitext(self._fpga_image)[0] dev = cl.get_platforms()[0].get_devices() binary = open( _join('/lib', 'firmware', 'xilinx', image_name, image_name + '.xclbin'), 'rb').read() ctx = cl.Context(dev_type=cl.device_type.ALL) prg = cl.Program(ctx, dev, [binary]) prg.build() # Find all devices xcl_probe = self._fpga_library.xclProbe xcl_probe.restype = _c_uint # Devices count if xcl_probe() < 1: raise RuntimeError("xclProbe does not found devices") # Open device xcl_open = self._fpga_library.xclOpen xcl_open.restype = _POINTER(_c_void_p) # Device handle xcl_open.argtypes = ( _c_uint, # deviceIndex _c_char_p, # logFileName _c_int, # level ) log_file = _join(self._log_dir, 'slot_%d_xrt.log' % self._fpga_slot_id) device_handle = xcl_open( self._fpga_slot_id, log_file.encode(), 3 # XCL_ERROR ) if not device_handle: raise RuntimeError("xclOpen failed to open device") self._fpga_handle = device_handle def _uninit_fpga(self): pass def _get_read_register_callback(self): """ Read register callback. Returns: function: Read register callback """ xcl_read = self._fpga_library.xclRead xcl_read.restype = _c_size_t # read size or error code xcl_read.argtypes = ( _c_void_p, # handle _c_int, # space _c_uint64, # offset _c_void_p, # hostBuf _c_size_t # size ) self._fpga_read_register = xcl_read def read_register(register_offset, returned_data, driver=self): """ Read register. Args: register_offset (int): Offset returned_data (int pointer): Return data. driver (accelize_drm.fpga_drivers._aws_xrt.FpgaDriver): Keep a reference to driver. """ with driver._fpga_read_register_lock(): size_or_error = driver._fpga_read_register( driver._fpga_handle, 2, # XCL_ADDR_KERNEL_CTRL driver._drm_ctrl_base_addr + register_offset, returned_data, 4 # 4 bytes ) # Return 0 return code if read size else return error code return size_or_error if size_or_error != 4 else 0 return read_register def _get_write_register_callback(self): """ Write register callback. Returns: function: Write register callback """ xcl_write = self._fpga_library.xclWrite xcl_write.restype = _c_size_t # written size or error code xcl_write.argtypes = ( _c_void_p, # handle _c_int, # space _c_uint64, # offset _c_char_p, # hostBuf _c_size_t # size ) self._fpga_write_register = xcl_write def write_register(register_offset, data_to_write, driver=self): """ Write register. Args: register_offset (int): Offset data_to_write (int): Data to write. driver (accelize_drm.fpga_drivers._aws_xrt.FpgaDriver): Keep a reference to driver. """ with driver._fpga_write_register_lock(): size_or_error = driver._fpga_write_register( driver._fpga_handle, 2, # XCL_ADDR_KERNEL_CTRL driver._drm_ctrl_base_addr + register_offset, data_to_write.to_bytes(4, byteorder="little"), 4 # 4 bytes ) # Return 0 return code if written size else return error code return size_or_error if size_or_error != 4 else 0 return write_register