Beispiel #1
0
def walk(path):
    for (dirpath, dirnames, filenames) in _walk(path):
        yield dirpath, dirnames, filenames
        for i in dirnames:
            _path = join(dirpath,i)
            if islink(_path):
                _link_path = link_path(_path)
                for _ in _walk(_link_path):
                    _ = list(_)
                    _[0] = _[0].replace(_link_path, _path) 
                    yield tuple(_)
Beispiel #2
0
def walk(path):
    for (dirpath, dirnames, filenames) in _walk(path):
        yield dirpath, dirnames, filenames
        for i in dirnames:
            _path = join(dirpath, i)
            if islink(_path):
                _link_path = link_path(_path)
                for _ in _walk(_link_path):
                    _ = list(_)
                    _[0] = _[0].replace(_link_path, _path)
                    yield tuple(_)
Beispiel #3
0
def walk(root_dir_path, abspaths=False, include_hidden=True):
    for root_dir_path, dir_names, file_names in _walk(root_dir_path):
        if include_hidden or not ishidden(root_dir_path):
            if abspaths:
                dir_paths = []
                for dir_name in dir_names:
                    dir_path = join(root_dir_path, dir_name)
                    if include_hidden or not ishidden(dir_path):
                        dir_paths.append(dir_path)

                file_paths = []
                for file_name in file_names:
                    file_path = join(root_dir_path, file_name)
                    if include_hidden or not ishidden(file_path):
                        file_paths.append(file_path)

                yield root_dir_path, dir_paths, file_paths
            elif not include_hidden:
                dir_names = [dir_name for dir_name in dir_names
                             if not ishidden(join(root_dir_path, dir_name))]
                file_names = [file_name for file_name in file_names
                              if not ishidden(join(root_dir_path, file_name))]

                yield root_dir_path, dir_names, file_names
            else:
                yield root_dir_path, dir_names, file_names
Beispiel #4
0
def cleanup_files(root_path: str, ttl: int) -> tuple:
    """Remove obsolete files and empty directories
    """
    now = _current_time()
    success = []
    failed = []

    for root, d_names, f_names in _walk(root_path):
        for f_name in f_names:
            f_path = _path.join(root, f_name)
            m_time = _path.getmtime(f_path)
            if m_time + ttl < now:
                try:
                    _unlink(f_path)
                    success.append(f_path)

                except Exception as e:
                    failed.append((f_path, e))

        # Remove empty directories
        for d_name in d_names:
            d_path = _path.join(root, d_name)
            if not _listdir(d_path):
                _rmdir(d_path)

    return success, failed
Beispiel #5
0
 def keys(self, pool: str) -> _Generator[str, None, None]:
     """Get all keys of the pool
     """
     for root, dirs, files in _walk(_path.join(self._path, self._server_name, pool), topdown=False):
         for name in files:
             f_path = _path.join(root, name)
             with open(f_path, 'rb') as f:
                 yield _pickle_load(f.read())['k']
Beispiel #6
0
def _list_files():
    sondecsvs=[[],[],[]]
    # for each location
    for dind, path in enumerate(_sondesdirs):
        # look in each year folder and pull out the csvs
        for yearpath, years, blahfiles in _walk(path):
            if 'lowres' in yearpath: # ignore low res data
                continue
            for fpath, blahdirs, fnames in _walk(yearpath):
                for fname in fnames:
                    if 'lowres' in fpath: continue
                    if (fname[0] != '.') and (fname[-4:] == '.csv') :
                        sondecsvs[dind].append(fpath+'/'+fname)
        uniqs=set(sondecsvs[dind])
        sondecsvs[dind] = list(uniqs)
        sondecsvs[dind].sort()
    return(sondecsvs)
Beispiel #7
0
def enum_files(root,*allow_ext):
    allow_ext=list(map(lambda s:s.lower(),allow_ext))
    if len(allow_ext)==0:
        allow_ext=['']
    if not _isdir(root):
        return
    for di,fd,fl in _walk(root):
        for f in fl:
            for ext in allow_ext:
                if f.lower().endswith(ext):
                    yield _join(di,f)
Beispiel #8
0
def init(dllpath=None, root="C:\\", bypass_check=False):
    """ Initialize the underlying tos-databridge DLL

  dllpath: string of the exact path of the DLL
  root: string of the directory to start walking/searching to find the DLL
  """
    global _dll
    rel = set()
    if not bypass_check and dllpath is None and root == "C:\\":
        if abort_init_after_warn():
            return False

    def _remove_older_versions():
        nonlocal rel
        getver = lambda x: _search(_REGEX_VER_SFFX, x).group().strip('-')
        vers = tuple(zip(map(getver, rel), rel))
        vers_max = max(vers)[0].split('.')[0]
        mtup = tuple(( x[0].split('.')[1],x[1]) \
                       for x in vers if x[0].split('.')[0] == vers_max )
        mtup_max = max(mtup)[0]
        rel = set(x[1] for x in mtup if x[0] == mtup_max)

    try:
        if dllpath is None:
            matcher = _partial(_match, _REGEX_DLL_NAME)  # regex match function
            for nfile in map(matcher, _listdir(_curdir)):
                if nfile:  # try the current dir first
                    rel.add(_curdir + _sep + nfile.string)
            if not rel:  # no luck, walk the dir tree
                for root, dirs, files in _walk(root):
                    for file in map(matcher, files):
                        if file:
                            rel.add(root + _sep + file.string)
                if not rel:  # if still nothing throw
                    raise TOSDB_InitError(" could not locate DLL")
            if len(rel) > 1:  # only use the most recent version(s)
                _remove_older_versions()
            # most recently updated
            d = dict(zip(map(lambda x: _stat(x).st_mtime, rel), rel))
            rec = max(d)
            dllpath = d[rec]
        _dll = _WinDLL(dllpath)
        print("+ Using Module ", dllpath)
        print("+ Last Update ", _asctime(_localtime(_stat(dllpath).st_mtime)))
        if connect():
            print("+ Succesfully Connected to Service \ Engine")
        else:
            print("- Failed to Connect to Service \ Engine")
        return True  # indicate the lib was loaded (but not if connect succeeded)
    except TOSDB_Error:
        raise
    except Exception as e:
        raise TOSDB_InitError("unable to initialize library", e)
Beispiel #9
0
    def cleanup(self, pool: str):
        """Cleanup outdated items from the cache
        """
        for root, dirs, files in _walk(_path.join(self._path, self._server_name, pool), topdown=False):
            for name in files:
                f_path = _path.join(root, name)
                with open(f_path, 'rb') as f:
                    expires = _pickle_load(f.read())['e']

                if expires and expires <= _time():
                    try:
                        _unlink(f_path)
                    except FileNotFoundError:
                        pass
Beispiel #10
0
def mirror_subdirectories(source_dir_path, target_dir_path):
    if exists(source_dir_path):
        if not exists(target_dir_path):
            makedirs(target_dir_path)

        for root_dir_path, dir_names, _ in _walk(source_dir_path):
            if not ishidden(root_dir_path):
                for dir_name in dir_names:
                    source_subdir_path = join(root_dir_path, dir_name)
                    if not ishidden(source_subdir_path):
                        source_subdir_path = \
                            relpath(source_subdir_path, source_dir_path)
                        target_subdir_path = \
                            join(target_dir_path, source_subdir_path)
                        if not exists(target_subdir_path):
                            makedirs(target_subdir_path)
Beispiel #11
0
def init(dllpath = None, root = "C:\\", bypass_check=False):
    """ Initialize the underlying tos-databridge DLL

    dllpath: string of the exact path of the DLL
    root: string of the directory to start walking/searching to find the DLL
    """  
    global _dll
    rel = set()
    if not bypass_check and dllpath is None and root == "C:\\":
        if abort_init_after_warn():
            return
    try:
        if dllpath is None:
            matcher = _partial( _match, _REGEX_DLL_NAME)  # regex match function
            for nfile in map( matcher, _listdir( _curdir )):
                if nfile: # try the current dir first             
                    rel.add( _curdir+ _sep + nfile.string )                    
            if not rel:                
                for root,dirs, files in _walk(root): # no luck, walk the dir tree 
                    for file in map( matcher, files):  
                        if file:                           
                            rel.add( root + _sep + file.string )                           
                if not rel: # if still nothing throw
                    raise TOSDB_Error(" could not locate DLL")    
            if len(rel) > 1:  # only use the most recent version(s)
                ver = _compile('-[\d]{1,2}.[\d]{1,2}-')
                vers = tuple( zip( map( 
                    lambda x: _search(ver,x).group().strip('-'), rel), rel) )
                vers_max = max(vers)[0].split('.')[0]
                mtup = tuple( (x[0].split('.')[1],x[1]) 
                              for x in vers if x[0].split('.')[0] == vers_max)         
                mtup_max = max(mtup)[0]
                rel = set( x[1] for x in mtup if x[0] == mtup_max )                      
            # find the most recently updated
            d = dict( zip(map( lambda x : _stat(x).st_mtime, rel), rel ) )
            rec = max(d)
            dllpath = d[ rec ]
        _dll = _WinDLL( dllpath )
        print( "+ Using Module ", dllpath )
        print( "+ Last Update ", _asctime(_localtime(_stat(dllpath).st_mtime)))
        if connect():
            print("+ Succesfully Connected to Service \ Engine")       
        else:
            print("- Failed to Connect to Service \ Engine")
        return True # indicate the lib was loaded
    except Exception as e:
        raise TOSDB_CLibError( "unable to initialize library", e )        
Beispiel #12
0
def init(dllpath=None, root="C:\\", bypass_check=False):
    """ Initialize the underlying tos-databridge DLL

    dllpath: string of the exact path of the DLL
    root: string of the directory to start walking/searching to find the DLL
    """  
    global _dll, _dll_depend1
    rel = set()
    if not bypass_check and dllpath is None and root == "C:\\":
        if abort_init_after_warn():
            return False

    def _remove_older_versions():
        nonlocal rel  
        getver = lambda x: _search(_REGEX_VER_SFFX,x).group().strip('-')
        vers = tuple(zip(map(getver, rel), rel))
        vers_max = max(vers)[0].split('.')[0]
        mtup = tuple(( x[0].split('.')[1],x[1]) \
                       for x in vers if x[0].split('.')[0] == vers_max )         
        mtup_max = max(mtup)[0]
        rel = set(x[1] for x in mtup if x[0] == mtup_max)

    def _get_depends1_dll_path(dllpath):
        d = _path.dirname(dllpath)
        return d + "/" + DLL_DEPENDS1_NAME + "-" + SYS_ARCH_TYPE + ".dll"
    
    try:   
        if dllpath is None:
            matcher = _partial(_match, _REGEX_DLL_NAME)  
            for nfile in map(matcher, _listdir(_curdir)):
                if nfile: # try the current dir first             
                    rel.add(_curdir+ _sep + nfile.string)                    
            if not rel: # no luck, walk the dir tree              
                for root, dirs, files in _walk(root):  
                    for file in map(matcher, files):  
                        if file:                           
                            rel.add(root + _sep + file.string)                           
                if not rel: # if still nothing throw
                    raise TOSDB_InitError(" could not locate DLL")          
            if len(rel) > 1:  # only use the most recent version(s)
                _remove_older_versions()              
            # most recently updated
            d = dict(zip(map(lambda x: _stat(x).st_mtime, rel), rel)) 
            rec = max(d)
            dllpath = d[rec]

        dllpath_depends1 = _get_depends1_dll_path(dllpath)
        _dll_depend1 = _CDLL(dllpath_depends1)
        _dll = _CDLL(dllpath)
        
        print("+ Using Module(s) ", dllpath)
        print("                  ", dllpath_depends1)
        print("+ Last Update ", _asctime(_localtime(_stat(dllpath).st_mtime)))
        if connect():
            print("+ Succesfully Connected to Service \ Engine")       
        else:
            print("- Failed to Connect to Service \ Engine")
        return True # indicate the lib was loaded (but not if connect succeeded)
    except TOSDB_Error:
        raise
    except Exception as e:
        raise TOSDB_InitError("unable to initialize library", e)        
Beispiel #13
0
 def os_walk(top, topdown=True, onerror=None, followlinks=False):
     return os._walk(longpathify(uni(top)), topdown, onerror, followlinks)
def list_files(*args):
    return str(list(_walk('../Frontend/uploads'))[0][2]).replace('\'', '"')
Beispiel #15
0
 def walk(path, topdown=topdown, followlinks=followlinks):
     yield next(_walk(path, topdown=topdown, followlinks=followlinks))
Beispiel #16
0
def init(dllpath=None, root="C:\\", bypass_check=False):
    """ Initialize the underlying tos-databridge DLL and try to connect.

    Returns True if library was successfully loaded, not necessarily that
    it was also able to connect. Details are sent to stdout.
    
    init(dllpath=None, root="C:\\", bypass_check=False)
    
    dllpath      :: str  :: exact path of the DLL -or-
    root         :: str  :: directory to start walking/searching to find the DLL
    bypass_check :: bool :: used by virtual layer implemenation (DO NOT SET)

    returns -> bool 

    throws TOSDB_InitError TOSDB_Error
    """
    global _dll, _dll_depend1
    rel = set()
    if not bypass_check and dllpath is None and root == "C:\\":
        if abort_init_after_warn():
            return False

    def _remove_older_versions():
        nonlocal rel
        getver = lambda x: _search(_REGEX_VER_SFFX, x).group().strip('-')
        vers = tuple(zip(map(getver, rel), rel))
        vers_max = max(vers)[0].split('.')[0]
        mtup = tuple(( x[0].split('.')[1],x[1]) \
                       for x in vers if x[0].split('.')[0] == vers_max )
        mtup_max = max(mtup)[0]
        rel = set(x[1] for x in mtup if x[0] == mtup_max)

    def _get_depends1_dll_path(dllpath):
        d = _path.dirname(dllpath)
        dbg = _match(_REGEX_DBG_DLL_PATH, dllpath)
        base = d + "/" + DLL_DEPENDS1_NAME + "-" + SYS_ARCH_TYPE
        path = base + ("_d.dll" if dbg else ".dll")
        return path

    try:
        if dllpath is None:
            matcher = _partial(_match, _REGEX_DLL_NAME)
            for nfile in map(matcher, _listdir(_curdir)):
                if nfile:  # try the current dir first
                    rel.add(_curdir + _sep + nfile.string)
            if not rel:  # no luck, walk the dir tree
                for root, dirs, files in _walk(root):
                    for file in map(matcher, files):
                        if file:
                            rel.add(root + _sep + file.string)
                if not rel:  # if still nothing throw
                    raise TOSDB_InitError(" could not locate DLL")
            if len(rel) > 1:  # only use the most recent version(s)
                _remove_older_versions()
            # most recently updated
            d = dict(zip(map(lambda x: _stat(x).st_mtime, rel), rel))
            rec = max(d)
            dllpath = d[rec]

        dllpath_depends1 = _get_depends1_dll_path(dllpath)
        _dll_depend1 = _CDLL(dllpath_depends1)
        _dll = _CDLL(dllpath)
        print("+ Using Module(s) ", dllpath)
        print("                  ", dllpath_depends1)
        print("+ Last Update:", _asctime(_localtime(_stat(dllpath).st_mtime)))
        print("+ Process ID:", str(_getpid()))
        if connect():
            print("+ Succesfully Connected to Service\Engine")
            if connected():
                print("+ Succesfully Connected to TOS")
            else:
                print("- Failed to Connect to TOS")
        else:
            print("- Failed to Connect to Service\Engine")
            print("- Failed to Connect to TOS")
        return True  # indicate the lib was loaded (but not if connect succeeded)
    except Exception as e:
        raise TOSDB_InitError("unable to initialize library:", e)
Beispiel #17
0
#!/usr/bin/env python

"""Deepest: determine the maximum depth of a directory tree."""

import os
try:
    from os.path import walk
except ImportError:  # pragma: no cover
    from os import walk as _walk  # Python >= 3.x
    walk = lambda x, y, z: [y(z, path, files) for path, _, files in _walk(x)]

import sys

from . import globals  # Needed for ShedSkin

from .constants import DESCRIPTION
from .printer import print_header, print_footer
from .traverser import traversal_callback


def get_depth(dirname):
    """
    Function for obtaining the deepest directory below `dirname`.

    @param dirname: The name of the directory to examine.
    @type dirname: str
    @return: The deepest directory, and its depth.
    @rtype: tuple
    """
    walk(dirname, traversal_callback, '')
    return (globals.deepest_path, globals.max_depth)
Beispiel #18
0
#!/usr/bin/env python
"""Deepest: determine the maximum depth of a directory tree."""

import os
try:
    from os.path import walk
except ImportError:  # pragma: no cover
    from os import walk as _walk  # Python >= 3.x
    walk = lambda x, y, z: [y(z, path, files) for path, _, files in _walk(x)]

import sys

from . import globals  # Needed for ShedSkin

from .constants import DESCRIPTION
from .printer import print_header, print_footer
from .traverser import traversal_callback


def get_depth(dirname):
    """
    Function for obtaining the deepest directory below `dirname`.

    @param dirname: The name of the directory to examine.
    @type dirname: str
    @return: The deepest directory, and its depth.
    @rtype: tuple
    """
    walk(dirname, traversal_callback, '')
    return (globals.deepest_path, globals.max_depth)
Beispiel #19
0
from os import walk as _walk
from os.path import join as _join, dirname as _dirname
from glob import glob as _glob

NAME = 'simpleorderbook'
SOURCE_DIR = _join(_dirname(__file__),'src')

setup_dict = {
    "name":NAME,
    "version":'0.6',
    "description": "financial-market orderbook and matching engine",
    "author":"Jonathon Ogden",
    "author_email":"*****@*****.**"
} 

cpp_sources = [f for d,_,files in _walk(SOURCE_DIR) \
               for f in _glob(_join(d, "*.cpp")) + _glob(_join(d, "*.c"))]

cpp_include_dirs = ["../include", "./include"]
cpp_compile_flags = ["-std=c++11", "-Wno-invalid-offsetof"]

cpp_ext = Extension(
    NAME,
    sources = cpp_sources,
    include_dirs = cpp_include_dirs,
    libraries=['SimpleOrderbook'],
    library_dirs=["../bin/release"],
    extra_compile_args=cpp_compile_flags,    
    undef_macros=["NDEBUG"], #internal DEBUG/NDEBUG checks should handle this
)
Beispiel #20
0
 def walker(path, topdown=topdown, followlinks=followlinks):
   """Alternative walker."""
   yield next(_walk(path, topdown=topdown, followlinks=followlinks))
Beispiel #21
0
def walk(directory,
         onerror=None,
         filename='.gitignore',
         overrides=None,
         ignore_completely=None):
    """
    Generate the file names in a directory tree by walking the tree
    top-down, while obeying the rules of .gitignore. Links will not
    be followed.
    """
    starting_directory = Path(os.path.abspath(directory))

    if not overrides:
        overrides = []
    overrides = [
        rule_from_pattern(p,
                          str(starting_directory),
                          source=('manual override', None)) for p in overrides
    ]

    # Git is incapable of adding .git files to stage -- by default walk()
    # will skip it entirely in a *non-overrideable* manner.
    if ignore_completely is None:
        ignore_completely = ['.git']
    elif not ignore_completely:
        ignore_completely = []
    ignore_completely = [
        rule_from_pattern(p, source=('application-level override', None))
        for p in ignore_completely
    ]
    if [rule for rule in ignore_completely if rule.negation]:
        raise ValueError('negation rules are not allowed in the '
                         'ignore completely rules')

    # Rule list will be a dict, keyed by directory with each value a
    # possibly-empty list of IgnoreRules
    rule_list = {}
    while True:
        for root, dirs, files in _walk(directory, onerror=onerror):
            rules = []
            if filename in files:
                rules.extend(rules_from_file(filename, os.path.abspath(root)))
            current_dir = Path(os.path.abspath(root))
            rel_path = str(current_dir.relative_to(starting_directory))
            rule_list[rel_path] = rules
            # Now, make a list of rules, working our way back to the
            # base directory.
            applicable_rules = [rule_list[rel_path]]
            if root != directory:
                for p in Path(root).parents:
                    rel_parent = str(p.relative_to(starting_directory))
                    applicable_rules.append(rule_list[rel_parent])
                    if p not in starting_directory.parents:
                        break
            applicable_rules.append(rule_list['.'])
            # Our rules are actually ordered from the base down
            applicable_rules = applicable_rules[::-1]
            flat_list = list(itertools.chain.from_iterable(applicable_rules))
            # overrides and ignore-completely patterns are always applicable
            flat_list.extend(overrides)
            flat_list.extend(ignore_completely)
            ignore = []
            for d in dirs:
                included = True
                path = os.path.abspath(os.path.join(root, d))
                for rule in flat_list:
                    if included != rule.negation:
                        if rule.match(path):
                            included = not included
                if not included:
                    ignore.append(d)
            dirs[:] = [d for d in dirs if d not in ignore]
            ignore = []
            for f in files:
                included = True
                path = os.path.join(root, f)
                for rule in flat_list:
                    if rule.directory_only:
                        continue
                    if included != rule.negation:
                        if rule.match(os.path.abspath(path)):
                            included = not included
                if not included:
                    ignore.append(f)
            files[:] = [f for f in files if f not in ignore]
            yield root, dirs, files
        return
 def os_walk(top, topdown=True, onerror=None, followlinks=False):
   return os._walk(longpathify(uni(top)), topdown, onerror, followlinks)
Beispiel #23
0
def unicode_walk(root):
    for dirname, dirnames, files in _walk(encode_path(root)):
        yield (decode_path(dirname), 
               map(decode_path, dirnames),
               map(decode_path, files))
Beispiel #24
0
def walk(directory, onerror=None, filename='.gitignore',
         overrides=None, ignore_completely=None):
    """
    Generate the file names in a directory tree by walking the tree
    top-down, while obeying the rules of .gitignore. Links will not
    be followed.
    """
    starting_directory = Path(os.path.abspath(directory))

    if not overrides:
        overrides = []
    overrides = [rule_from_pattern(
        p, str(starting_directory), source=('manual override', None)
    ) for p in overrides]

    # Git is incapable of adding .git files to stage -- by default walk()
    # will skip it entirely in a *non-overrideable* manner.
    if ignore_completely is None:
        ignore_completely = ['.git']
    elif not ignore_completely:
        ignore_completely = []
    ignore_completely = [
        rule_from_pattern(p, source=('application-level override', None))
        for p in ignore_completely
    ]
    if [rule for rule in ignore_completely if rule.negation]:
        raise ValueError('negation rules are not allowed in the '
                         'ignore completely rules')

    # Rule list will be a dict, keyed by directory with each value a
    # possibly-empty list of IgnoreRules
    rule_list = {}
    while True:
        for root, dirs, files in _walk(directory, onerror=onerror):
            rules = []
            if filename in files:
                rules.extend(rules_from_file(filename, os.path.abspath(root)))
            current_dir = Path(os.path.abspath(root))
            rel_path = str(current_dir.relative_to(starting_directory))
            rule_list[rel_path] = rules
            # Now, make a list of rules, working our way back to the
            # base directory.
            applicable_rules = [rule_list[rel_path]]
            if root != directory:
                for p in Path(root).parents:
                    rel_parent = str(p.relative_to(starting_directory))
                    applicable_rules.append(rule_list[rel_parent])
                    if p not in starting_directory.parents:
                        break
            applicable_rules.append(rule_list['.'])
            # Our rules are actually ordered from the base down
            applicable_rules = applicable_rules[::-1]
            flat_list = list(
                itertools.chain.from_iterable(applicable_rules)
            )
            # overrides and ignore-completely patterns are always applicable
            flat_list.extend(overrides)
            flat_list.extend(ignore_completely)
            ignore = []
            for d in dirs:
                included = True
                path = os.path.abspath(os.path.join(root, d))
                for rule in flat_list:
                    if included != rule.negation:
                        if rule.match(path):
                            included = not included
                if not included:
                    ignore.append(d)
            dirs[:] = [d for d in dirs if d not in ignore]
            ignore = []
            for f in files:
                included = True
                path = os.path.join(root, f)
                for rule in flat_list:
                    if rule.directory_only:
                        continue
                    if included != rule.negation:
                        if rule.match(os.path.abspath(path)):
                            included = not included
                if not included:
                    ignore.append(f)
            files[:] = [f for f in files if f not in ignore]
            yield root, dirs, files
        return