def walk(path):
for (dirpath, dirnames, filenames) in _walk(path):
yield dirpath, dirnames, filenames
for i in dirnames:
_path = join(dirpath,i)
if islink(_path):
_link_path = link_path(_path)
for _ in _walk(_link_path):
_ = list(_)
_[0] = _[0].replace(_link_path, _path)
yield tuple(_)
def walk(path):
for (dirpath, dirnames, filenames) in _walk(path):
yield dirpath, dirnames, filenames
for i in dirnames:
_path = join(dirpath, i)
if islink(_path):
_link_path = link_path(_path)
for _ in _walk(_link_path):
_ = list(_)
_[0] = _[0].replace(_link_path, _path)
yield tuple(_)
def walk(root_dir_path, abspaths=False, include_hidden=True):
for root_dir_path, dir_names, file_names in _walk(root_dir_path):
if include_hidden or not ishidden(root_dir_path):
if abspaths:
dir_paths = []
for dir_name in dir_names:
dir_path = join(root_dir_path, dir_name)
if include_hidden or not ishidden(dir_path):
dir_paths.append(dir_path)
file_paths = []
for file_name in file_names:
file_path = join(root_dir_path, file_name)
if include_hidden or not ishidden(file_path):
file_paths.append(file_path)
yield root_dir_path, dir_paths, file_paths
elif not include_hidden:
dir_names = [dir_name for dir_name in dir_names
if not ishidden(join(root_dir_path, dir_name))]
file_names = [file_name for file_name in file_names
if not ishidden(join(root_dir_path, file_name))]
yield root_dir_path, dir_names, file_names
else:
yield root_dir_path, dir_names, file_names
def cleanup_files(root_path: str, ttl: int) -> tuple:
"""Remove obsolete files and empty directories
"""
now = _current_time()
success = []
failed = []
for root, d_names, f_names in _walk(root_path):
for f_name in f_names:
f_path = _path.join(root, f_name)
m_time = _path.getmtime(f_path)
if m_time + ttl < now:
try:
_unlink(f_path)
success.append(f_path)
except Exception as e:
failed.append((f_path, e))
# Remove empty directories
for d_name in d_names:
d_path = _path.join(root, d_name)
if not _listdir(d_path):
_rmdir(d_path)
return success, failed
def keys(self, pool: str) -> _Generator[str, None, None]:
"""Get all keys of the pool
"""
for root, dirs, files in _walk(_path.join(self._path, self._server_name, pool), topdown=False):
for name in files:
f_path = _path.join(root, name)
with open(f_path, 'rb') as f:
yield _pickle_load(f.read())['k']
def _list_files():
sondecsvs=[[],[],[]]
# for each location
for dind, path in enumerate(_sondesdirs):
# look in each year folder and pull out the csvs
for yearpath, years, blahfiles in _walk(path):
if 'lowres' in yearpath: # ignore low res data
continue
for fpath, blahdirs, fnames in _walk(yearpath):
for fname in fnames:
if 'lowres' in fpath: continue
if (fname[0] != '.') and (fname[-4:] == '.csv') :
sondecsvs[dind].append(fpath+'/'+fname)
uniqs=set(sondecsvs[dind])
sondecsvs[dind] = list(uniqs)
sondecsvs[dind].sort()
return(sondecsvs)
def enum_files(root,*allow_ext):
allow_ext=list(map(lambda s:s.lower(),allow_ext))
if len(allow_ext)==0:
allow_ext=['']
if not _isdir(root):
return
for di,fd,fl in _walk(root):
for f in fl:
for ext in allow_ext:
if f.lower().endswith(ext):
yield _join(di,f)
def init(dllpath=None, root="C:\\", bypass_check=False):
""" Initialize the underlying tos-databridge DLL
dllpath: string of the exact path of the DLL
root: string of the directory to start walking/searching to find the DLL
"""
global _dll
rel = set()
if not bypass_check and dllpath is None and root == "C:\\":
if abort_init_after_warn():
return False
def _remove_older_versions():
nonlocal rel
getver = lambda x: _search(_REGEX_VER_SFFX, x).group().strip('-')
vers = tuple(zip(map(getver, rel), rel))
vers_max = max(vers)[0].split('.')[0]
mtup = tuple(( x[0].split('.')[1],x[1]) \
for x in vers if x[0].split('.')[0] == vers_max )
mtup_max = max(mtup)[0]
rel = set(x[1] for x in mtup if x[0] == mtup_max)
try:
if dllpath is None:
matcher = _partial(_match, _REGEX_DLL_NAME) # regex match function
for nfile in map(matcher, _listdir(_curdir)):
if nfile: # try the current dir first
rel.add(_curdir + _sep + nfile.string)
if not rel: # no luck, walk the dir tree
for root, dirs, files in _walk(root):
for file in map(matcher, files):
if file:
rel.add(root + _sep + file.string)
if not rel: # if still nothing throw
raise TOSDB_InitError(" could not locate DLL")
if len(rel) > 1: # only use the most recent version(s)
_remove_older_versions()
# most recently updated
d = dict(zip(map(lambda x: _stat(x).st_mtime, rel), rel))
rec = max(d)
dllpath = d[rec]
_dll = _WinDLL(dllpath)
print("+ Using Module ", dllpath)
print("+ Last Update ", _asctime(_localtime(_stat(dllpath).st_mtime)))
if connect():
print("+ Succesfully Connected to Service \ Engine")
else:
print("- Failed to Connect to Service \ Engine")
return True # indicate the lib was loaded (but not if connect succeeded)
except TOSDB_Error:
raise
except Exception as e:
raise TOSDB_InitError("unable to initialize library", e)
def cleanup(self, pool: str):
"""Cleanup outdated items from the cache
"""
for root, dirs, files in _walk(_path.join(self._path, self._server_name, pool), topdown=False):
for name in files:
f_path = _path.join(root, name)
with open(f_path, 'rb') as f:
expires = _pickle_load(f.read())['e']
if expires and expires <= _time():
try:
_unlink(f_path)
except FileNotFoundError:
pass
def mirror_subdirectories(source_dir_path, target_dir_path):
if exists(source_dir_path):
if not exists(target_dir_path):
makedirs(target_dir_path)
for root_dir_path, dir_names, _ in _walk(source_dir_path):
if not ishidden(root_dir_path):
for dir_name in dir_names:
source_subdir_path = join(root_dir_path, dir_name)
if not ishidden(source_subdir_path):
source_subdir_path = \
relpath(source_subdir_path, source_dir_path)
target_subdir_path = \
join(target_dir_path, source_subdir_path)
if not exists(target_subdir_path):
makedirs(target_subdir_path)
def init(dllpath = None, root = "C:\\", bypass_check=False):
""" Initialize the underlying tos-databridge DLL
dllpath: string of the exact path of the DLL
root: string of the directory to start walking/searching to find the DLL
"""
global _dll
rel = set()
if not bypass_check and dllpath is None and root == "C:\\":
if abort_init_after_warn():
return
try:
if dllpath is None:
matcher = _partial( _match, _REGEX_DLL_NAME) # regex match function
for nfile in map( matcher, _listdir( _curdir )):
if nfile: # try the current dir first
rel.add( _curdir+ _sep + nfile.string )
if not rel:
for root,dirs, files in _walk(root): # no luck, walk the dir tree
for file in map( matcher, files):
if file:
rel.add( root + _sep + file.string )
if not rel: # if still nothing throw
raise TOSDB_Error(" could not locate DLL")
if len(rel) > 1: # only use the most recent version(s)
ver = _compile('-[\d]{1,2}.[\d]{1,2}-')
vers = tuple( zip( map(
lambda x: _search(ver,x).group().strip('-'), rel), rel) )
vers_max = max(vers)[0].split('.')[0]
mtup = tuple( (x[0].split('.')[1],x[1])
for x in vers if x[0].split('.')[0] == vers_max)
mtup_max = max(mtup)[0]
rel = set( x[1] for x in mtup if x[0] == mtup_max )
# find the most recently updated
d = dict( zip(map( lambda x : _stat(x).st_mtime, rel), rel ) )
rec = max(d)
dllpath = d[ rec ]
_dll = _WinDLL( dllpath )
print( "+ Using Module ", dllpath )
print( "+ Last Update ", _asctime(_localtime(_stat(dllpath).st_mtime)))
if connect():
print("+ Succesfully Connected to Service \ Engine")
else:
print("- Failed to Connect to Service \ Engine")
return True # indicate the lib was loaded
except Exception as e:
raise TOSDB_CLibError( "unable to initialize library", e )
def init(dllpath=None, root="C:\\", bypass_check=False):
""" Initialize the underlying tos-databridge DLL
dllpath: string of the exact path of the DLL
root: string of the directory to start walking/searching to find the DLL
"""
global _dll, _dll_depend1
rel = set()
if not bypass_check and dllpath is None and root == "C:\\":
if abort_init_after_warn():
return False
def _remove_older_versions():
nonlocal rel
getver = lambda x: _search(_REGEX_VER_SFFX,x).group().strip('-')
vers = tuple(zip(map(getver, rel), rel))
vers_max = max(vers)[0].split('.')[0]
mtup = tuple(( x[0].split('.')[1],x[1]) \
for x in vers if x[0].split('.')[0] == vers_max )
mtup_max = max(mtup)[0]
rel = set(x[1] for x in mtup if x[0] == mtup_max)
def _get_depends1_dll_path(dllpath):
d = _path.dirname(dllpath)
return d + "/" + DLL_DEPENDS1_NAME + "-" + SYS_ARCH_TYPE + ".dll"
try:
if dllpath is None:
matcher = _partial(_match, _REGEX_DLL_NAME)
for nfile in map(matcher, _listdir(_curdir)):
if nfile: # try the current dir first
rel.add(_curdir+ _sep + nfile.string)
if not rel: # no luck, walk the dir tree
for root, dirs, files in _walk(root):
for file in map(matcher, files):
if file:
rel.add(root + _sep + file.string)
if not rel: # if still nothing throw
raise TOSDB_InitError(" could not locate DLL")
if len(rel) > 1: # only use the most recent version(s)
_remove_older_versions()
# most recently updated
d = dict(zip(map(lambda x: _stat(x).st_mtime, rel), rel))
rec = max(d)
dllpath = d[rec]
dllpath_depends1 = _get_depends1_dll_path(dllpath)
_dll_depend1 = _CDLL(dllpath_depends1)
_dll = _CDLL(dllpath)
print("+ Using Module(s) ", dllpath)
print(" ", dllpath_depends1)
print("+ Last Update ", _asctime(_localtime(_stat(dllpath).st_mtime)))
if connect():
print("+ Succesfully Connected to Service \ Engine")
else:
print("- Failed to Connect to Service \ Engine")
return True # indicate the lib was loaded (but not if connect succeeded)
except TOSDB_Error:
raise
except Exception as e:
raise TOSDB_InitError("unable to initialize library", e)
def os_walk(top, topdown=True, onerror=None, followlinks=False):
return os._walk(longpathify(uni(top)), topdown, onerror, followlinks)
def list_files(*args):
return str(list(_walk('../Frontend/uploads'))[0][2]).replace('\'', '"')
def walk(path, topdown=topdown, followlinks=followlinks):
yield next(_walk(path, topdown=topdown, followlinks=followlinks))
def init(dllpath=None, root="C:\\", bypass_check=False):
""" Initialize the underlying tos-databridge DLL and try to connect.
Returns True if library was successfully loaded, not necessarily that
it was also able to connect. Details are sent to stdout.
init(dllpath=None, root="C:\\", bypass_check=False)
dllpath :: str :: exact path of the DLL -or-
root :: str :: directory to start walking/searching to find the DLL
bypass_check :: bool :: used by virtual layer implemenation (DO NOT SET)
returns -> bool
throws TOSDB_InitError TOSDB_Error
"""
global _dll, _dll_depend1
rel = set()
if not bypass_check and dllpath is None and root == "C:\\":
if abort_init_after_warn():
return False
def _remove_older_versions():
nonlocal rel
getver = lambda x: _search(_REGEX_VER_SFFX, x).group().strip('-')
vers = tuple(zip(map(getver, rel), rel))
vers_max = max(vers)[0].split('.')[0]
mtup = tuple(( x[0].split('.')[1],x[1]) \
for x in vers if x[0].split('.')[0] == vers_max )
mtup_max = max(mtup)[0]
rel = set(x[1] for x in mtup if x[0] == mtup_max)
def _get_depends1_dll_path(dllpath):
d = _path.dirname(dllpath)
dbg = _match(_REGEX_DBG_DLL_PATH, dllpath)
base = d + "/" + DLL_DEPENDS1_NAME + "-" + SYS_ARCH_TYPE
path = base + ("_d.dll" if dbg else ".dll")
return path
try:
if dllpath is None:
matcher = _partial(_match, _REGEX_DLL_NAME)
for nfile in map(matcher, _listdir(_curdir)):
if nfile: # try the current dir first
rel.add(_curdir + _sep + nfile.string)
if not rel: # no luck, walk the dir tree
for root, dirs, files in _walk(root):
for file in map(matcher, files):
if file:
rel.add(root + _sep + file.string)
if not rel: # if still nothing throw
raise TOSDB_InitError(" could not locate DLL")
if len(rel) > 1: # only use the most recent version(s)
_remove_older_versions()
# most recently updated
d = dict(zip(map(lambda x: _stat(x).st_mtime, rel), rel))
rec = max(d)
dllpath = d[rec]
dllpath_depends1 = _get_depends1_dll_path(dllpath)
_dll_depend1 = _CDLL(dllpath_depends1)
_dll = _CDLL(dllpath)
print("+ Using Module(s) ", dllpath)
print(" ", dllpath_depends1)
print("+ Last Update:", _asctime(_localtime(_stat(dllpath).st_mtime)))
print("+ Process ID:", str(_getpid()))
if connect():
print("+ Succesfully Connected to Service\Engine")
if connected():
print("+ Succesfully Connected to TOS")
else:
print("- Failed to Connect to TOS")
else:
print("- Failed to Connect to Service\Engine")
print("- Failed to Connect to TOS")
return True # indicate the lib was loaded (but not if connect succeeded)
except Exception as e:
raise TOSDB_InitError("unable to initialize library:", e)
#!/usr/bin/env python
"""Deepest: determine the maximum depth of a directory tree."""
import os
try:
from os.path import walk
except ImportError: # pragma: no cover
from os import walk as _walk # Python >= 3.x
walk = lambda x, y, z: [y(z, path, files) for path, _, files in _walk(x)]
import sys
from . import globals # Needed for ShedSkin
from .constants import DESCRIPTION
from .printer import print_header, print_footer
from .traverser import traversal_callback
def get_depth(dirname):
"""
Function for obtaining the deepest directory below `dirname`.
@param dirname: The name of the directory to examine.
@type dirname: str
@return: The deepest directory, and its depth.
@rtype: tuple
"""
walk(dirname, traversal_callback, '')
return (globals.deepest_path, globals.max_depth)
#!/usr/bin/env python
"""Deepest: determine the maximum depth of a directory tree."""
import os
try:
from os.path import walk
except ImportError: # pragma: no cover
from os import walk as _walk # Python >= 3.x
walk = lambda x, y, z: [y(z, path, files) for path, _, files in _walk(x)]
import sys
from . import globals # Needed for ShedSkin
from .constants import DESCRIPTION
from .printer import print_header, print_footer
from .traverser import traversal_callback
def get_depth(dirname):
"""
Function for obtaining the deepest directory below `dirname`.
@param dirname: The name of the directory to examine.
@type dirname: str
@return: The deepest directory, and its depth.
@rtype: tuple
"""
walk(dirname, traversal_callback, '')
return (globals.deepest_path, globals.max_depth)
from os import walk as _walk
from os.path import join as _join, dirname as _dirname
from glob import glob as _glob
NAME = 'simpleorderbook'
SOURCE_DIR = _join(_dirname(__file__),'src')
setup_dict = {
"name":NAME,
"version":'0.6',
"description": "financial-market orderbook and matching engine",
"author":"Jonathon Ogden",
"author_email":"*****@*****.**"
}
cpp_sources = [f for d,_,files in _walk(SOURCE_DIR) \
for f in _glob(_join(d, "*.cpp")) + _glob(_join(d, "*.c"))]
cpp_include_dirs = ["../include", "./include"]
cpp_compile_flags = ["-std=c++11", "-Wno-invalid-offsetof"]
cpp_ext = Extension(
NAME,
sources = cpp_sources,
include_dirs = cpp_include_dirs,
libraries=['SimpleOrderbook'],
library_dirs=["../bin/release"],
extra_compile_args=cpp_compile_flags,
undef_macros=["NDEBUG"], #internal DEBUG/NDEBUG checks should handle this
)
def walker(path, topdown=topdown, followlinks=followlinks): """Alternative walker.""" yield next(_walk(path, topdown=topdown, followlinks=followlinks))
def walk(directory,
onerror=None,
filename='.gitignore',
overrides=None,
ignore_completely=None):
"""
Generate the file names in a directory tree by walking the tree
top-down, while obeying the rules of .gitignore. Links will not
be followed.
"""
starting_directory = Path(os.path.abspath(directory))
if not overrides:
overrides = []
overrides = [
rule_from_pattern(p,
str(starting_directory),
source=('manual override', None)) for p in overrides
]
# Git is incapable of adding .git files to stage -- by default walk()
# will skip it entirely in a *non-overrideable* manner.
if ignore_completely is None:
ignore_completely = ['.git']
elif not ignore_completely:
ignore_completely = []
ignore_completely = [
rule_from_pattern(p, source=('application-level override', None))
for p in ignore_completely
]
if [rule for rule in ignore_completely if rule.negation]:
raise ValueError('negation rules are not allowed in the '
'ignore completely rules')
# Rule list will be a dict, keyed by directory with each value a
# possibly-empty list of IgnoreRules
rule_list = {}
while True:
for root, dirs, files in _walk(directory, onerror=onerror):
rules = []
if filename in files:
rules.extend(rules_from_file(filename, os.path.abspath(root)))
current_dir = Path(os.path.abspath(root))
rel_path = str(current_dir.relative_to(starting_directory))
rule_list[rel_path] = rules
# Now, make a list of rules, working our way back to the
# base directory.
applicable_rules = [rule_list[rel_path]]
if root != directory:
for p in Path(root).parents:
rel_parent = str(p.relative_to(starting_directory))
applicable_rules.append(rule_list[rel_parent])
if p not in starting_directory.parents:
break
applicable_rules.append(rule_list['.'])
# Our rules are actually ordered from the base down
applicable_rules = applicable_rules[::-1]
flat_list = list(itertools.chain.from_iterable(applicable_rules))
# overrides and ignore-completely patterns are always applicable
flat_list.extend(overrides)
flat_list.extend(ignore_completely)
ignore = []
for d in dirs:
included = True
path = os.path.abspath(os.path.join(root, d))
for rule in flat_list:
if included != rule.negation:
if rule.match(path):
included = not included
if not included:
ignore.append(d)
dirs[:] = [d for d in dirs if d not in ignore]
ignore = []
for f in files:
included = True
path = os.path.join(root, f)
for rule in flat_list:
if rule.directory_only:
continue
if included != rule.negation:
if rule.match(os.path.abspath(path)):
included = not included
if not included:
ignore.append(f)
files[:] = [f for f in files if f not in ignore]
yield root, dirs, files
return
def os_walk(top, topdown=True, onerror=None, followlinks=False): return os._walk(longpathify(uni(top)), topdown, onerror, followlinks)
def unicode_walk(root):
for dirname, dirnames, files in _walk(encode_path(root)):
yield (decode_path(dirname),
map(decode_path, dirnames),
map(decode_path, files))
def walk(directory, onerror=None, filename='.gitignore',
overrides=None, ignore_completely=None):
"""
Generate the file names in a directory tree by walking the tree
top-down, while obeying the rules of .gitignore. Links will not
be followed.
"""
starting_directory = Path(os.path.abspath(directory))
if not overrides:
overrides = []
overrides = [rule_from_pattern(
p, str(starting_directory), source=('manual override', None)
) for p in overrides]
# Git is incapable of adding .git files to stage -- by default walk()
# will skip it entirely in a *non-overrideable* manner.
if ignore_completely is None:
ignore_completely = ['.git']
elif not ignore_completely:
ignore_completely = []
ignore_completely = [
rule_from_pattern(p, source=('application-level override', None))
for p in ignore_completely
]
if [rule for rule in ignore_completely if rule.negation]:
raise ValueError('negation rules are not allowed in the '
'ignore completely rules')
# Rule list will be a dict, keyed by directory with each value a
# possibly-empty list of IgnoreRules
rule_list = {}
while True:
for root, dirs, files in _walk(directory, onerror=onerror):
rules = []
if filename in files:
rules.extend(rules_from_file(filename, os.path.abspath(root)))
current_dir = Path(os.path.abspath(root))
rel_path = str(current_dir.relative_to(starting_directory))
rule_list[rel_path] = rules
# Now, make a list of rules, working our way back to the
# base directory.
applicable_rules = [rule_list[rel_path]]
if root != directory:
for p in Path(root).parents:
rel_parent = str(p.relative_to(starting_directory))
applicable_rules.append(rule_list[rel_parent])
if p not in starting_directory.parents:
break
applicable_rules.append(rule_list['.'])
# Our rules are actually ordered from the base down
applicable_rules = applicable_rules[::-1]
flat_list = list(
itertools.chain.from_iterable(applicable_rules)
)
# overrides and ignore-completely patterns are always applicable
flat_list.extend(overrides)
flat_list.extend(ignore_completely)
ignore = []
for d in dirs:
included = True
path = os.path.abspath(os.path.join(root, d))
for rule in flat_list:
if included != rule.negation:
if rule.match(path):
included = not included
if not included:
ignore.append(d)
dirs[:] = [d for d in dirs if d not in ignore]
ignore = []
for f in files:
included = True
path = os.path.join(root, f)
for rule in flat_list:
if rule.directory_only:
continue
if included != rule.negation:
if rule.match(os.path.abspath(path)):
included = not included
if not included:
ignore.append(f)
files[:] = [f for f in files if f not in ignore]
yield root, dirs, files
return
