def __init__(self, server, root, recursive_threshold, max_scanners, timeout, quiet, display_progress, max_files = None,
include_sizes=True, ignore_subdirs=[]):
PyThread.__init__(self)
self.RecursiveThreshold = recursive_threshold
self.Server = server
self.Root = canonic_path(root)
self.MaxScanners = max_scanners
self.Results = DEQueue()
self.ScannerQueue = TaskQueue(max_scanners)
self.Timeout = timeout
self.Done = False
self.Error = None
self.Failed = False
self.RootFailed = False
self.Directories = set()
self.RecursiveFailed = {} # parent path -> count
self.Errors = {} # location -> count
self.GaveUp = {}
self.LastReport = time.time()
self.EmptyDirs = set()
self.NScanned = 0
self.NToScan = 1
self.Quiet = quiet
self.DisplayProgress = display_progress and Use_tqdm and not quiet
if self.DisplayProgress:
self.TQ = tqdm.tqdm(total=self.NToScan, unit="dir")
self.LastV = 0
self.NFiles = self.NDirectories = 0
self.MaxFiles = max_files # will stop after number of files found exceeds this number. Used for debugging
self.IgnoreSubdirs = ignore_subdirs
self.IgnoredFiles = self.IgnoredDirs = 0
self.IncludeSizes = include_sizes
self.TotalSize = 0.0 if include_sizes else None # Megabytes
class WebsocketServer(PyThread):
def __init__(self,
port,
app,
max_connections=10,
max_queued=30,
**ws_args):
PyThread.__init__(self)
self.Port = port
self.HandlerQueue = TaskQueue(max_connections, capacity=max_queued)
self.App = app
self.WSArgs = ws_args
def run(self):
srv_sock = socket(AF_INET, SOCK_STREAM)
srv_sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
srv_sock.bind(("", self.Port))
srv_sock.listen(5)
while True:
sock, address = srv_sock.accept()
receiver = WebsocketClientConnection(self.App, sock, address,
self.WSArgs)
try:
self.HandlerQueue.addTask(receiver)
except:
# ws response here !
sock.close()
def __init__(self, session, files, max_matchers, stagger):
Primitive.__init__(self)
self.Matches = []
self.Unmatches = []
self.Queue = TaskQueue(
max_matchers,
stagger=stagger,
tasks=[Matcher(self, session, path) for path in files])
def configure(self, config=None):
config = config or self.Config
self.Config = config
reload_files = config.get("touch_reload", [])
if isinstance(reload_files, str):
reload_files = [reload_files]
self.ReloadFileTimestamps = {path: self.mtime(path) for path in reload_files}
self.Prefix = config.get("prefix", "/")
self.ReplacePrefix = config.get("replace_prefix")
self.Timeout = config.get("timeout", 10)
saved_path = sys.path[:]
saved_modules = set(sys.modules.keys())
saved_environ = os.environ.copy()
try:
args = None
fname = config["file"]
g = {}
extra_path = config.get("python_path")
if extra_path is not None:
if isinstance(extra_path, str):
extra_path = [extra_path]
sys.path = extra_path + sys.path
if "env" in config:
os.environ.update(config["env"])
exec(open(fname, "r").read(), g)
if "create" in config:
args = config.get("args")
app = g[config["create"]](args)
else:
app = g[config.get("application", "application")]
self.AppArgs = args
self.WSGIApp = app
max_workers = config.get("max_workers", 5)
queue_capacity = config.get("queue_capacity", 10)
self.RequestQueue = TaskQueue(max_workers, capacity = queue_capacity,
delegate=self)
self.log("(re)configured")
finally:
sys.path = saved_path
extra_modules = set(sys.modules.keys()) - set(saved_modules)
#print("loadApp: removing modules:", sorted(list(extra_modules)))
for m in extra_modules:
del sys.modules[m]
for n in set(os.environ.keys()) - set(saved_environ.keys()):
del os.environ[n]
os.environ.update(saved_environ)
def __init__(self,
port,
app,
max_connections=10,
max_queued=30,
**ws_args):
PyThread.__init__(self)
self.Port = port
self.HandlerQueue = TaskQueue(max_connections, capacity=max_queued)
self.App = app
self.WSArgs = ws_args
def __init__(self,
port,
ip='',
max_clients=None,
queue_capacity=None,
stagger=None,
enabled=True):
PyThread.__init__(self)
self.Sock = None
self.Clients = TaskQueue(max_clients,
capacity=queue_capacity,
stagger=stagger)
self.Port = port
self.IP = ip
self.Enabled = enabled
self.Shutdown = False
self.LastIdle = 0.0
def __init__(self,
port,
app=None,
services=[],
sock=None,
logger=None,
max_connections=100,
timeout=20.0,
enabled=True,
max_queued=100,
logging=False,
log_file="-",
debug=None,
certfile=None,
keyfile=None,
verify="none",
ca_file=None,
password=None):
PyThread.__init__(self)
self.Port = port
self.Sock = sock
assert self.Port is not None, "Port must be specified"
if logger is None and logging:
logger = Logger(log_file)
#print("logs sent to:", f)
Logged.__init__(self, f"[server {self.Port}]", logger, debug=True)
self.Logger = logger
self.Timeout = timeout
max_connections = max_connections
queue_capacity = max_queued
self.RequestReaderQueue = TaskQueue(max_connections,
capacity=queue_capacity,
delegate=self)
self.SocketWrapper = SSLSocketWrapper(
certfile, keyfile, verify, ca_file, password) if keyfile else None
if app is not None:
services = [Service(app, logger)]
self.Services = services
self.Stop = False
class MatchJob(Primitive):
def __init__(self, session, files, max_matchers, stagger):
Primitive.__init__(self)
self.Matches = []
self.Unmatches = []
self.Queue = TaskQueue(
max_matchers,
stagger=stagger,
tasks=[Matcher(self, session, path) for path in files])
def wait(self):
self.Queue.waitUntilEmpty()
@synchronized
def addData(self, matcher, n_observations, n_objects, matches, unmatches):
print "File: %s: %s observations comapred to %d objects: %d matches, %d unmatches" % (
matcher.InputPath, n_observations, n_objects, len(matches),
len(unmatches))
if len(matches):
self.Matches.append(matches)
if len(unmatches):
self.Unmatches.append(unmatches)
def __init__(self,
port,
app,
max_connections=100,
timeout=10.0,
enabled=True,
max_queued=100,
logging=False,
log_file=None,
debug=None):
PyThread.__init__(self)
#self.debug("Server started")
self.Port = port
self.Timeout = timeout
self.WSGIApp = app
self.Enabled = False
self.Logging = logging
self.LogFile = sys.stdout if log_file is None else log_file
self.Connections = TaskQueue(max_connections, capacity=max_queued)
if enabled:
self.enableServer()
self.Debug = debug
def __init__(self,
port,
app,
remove_prefix="",
url_pattern="*",
max_connections=100,
enabled=True,
max_queued=100,
logging=True,
log_file=None):
PyThread.__init__(self)
#self.debug("Server started")
self.Port = port
self.WSGIApp = app
self.Match = url_pattern
self.Enabled = False
self.Logging = logging
self.LogFile = sys.stdout if log_file is None else log_file
self.Connections = TaskQueue(max_connections, capacity=max_queued)
self.RemovePrefix = remove_prefix
if enabled:
self.enableServer()
class ReplicationManager(Primitive, Logged):
def __init__(self, cfg):
Primitive.__init__(self)
self.Replicators = TaskQueue(cfg.get('max_rep', 2), stagger=0.5)
self.DClient = DataClient((cfg["broadcast"], cfg["listen_port"]),
cfg["farm_name"])
def replicate(self, nfrep, lfn, lpath, info):
if nfrep > 2:
# make 2 replicators
n1 = nfrep // 2
n2 = nfrep - n1
r = Replicator(lfn, lpath, info, n1, self.DClient, self)
self.debug('replicator created: %s *%d' % (lpath, n1))
self.Replicators.addTask(r)
r = Replicator(lfn, lpath, info, n2, self.DClient, self)
self.debug('replicator created: %s *%d' % (lpath, n2))
self.Replicators.addTask(r)
elif nfrep > 0:
r = Replicator(lfn, lpath, info, nfrep, self.DClient, self)
self.Replicators.addTask(r)
self.debug('replicator created: %s *%d' % (lpath, nfrep))
def done(self, rep):
pass
def retry(self, rep):
rep.reinit()
t = threading.Timer(0.1 + random.random(),
self.Replicators.addTask,
args=(rep, ))
t.start()
def statTxns(self):
pending, active = self.Replocators.tasks()
stats = ["RP * %s" % (x.LogPath, ) for x in active
] + ["RP I %s" % (x.LogPath, ) for x in pending]
return '\n'.join(stats) + ".\n"
class QueuedApplication(Primitive, Logged):
def __init__(self, config, logger=None):
self.Instance = config["instance"]
Primitive.__init__(self, name=f"[app {self.Instance}]")
Logged.__init__(self, f"[app {self.Instance}]", logger, debug=True)
self.configure(config)
@synchronized
def configure(self, config=None):
config = config or self.Config
self.Config = config
reload_files = config.get("touch_reload", [])
if isinstance(reload_files, str):
reload_files = [reload_files]
self.ReloadFileTimestamps = {path: self.mtime(path) for path in reload_files}
self.Prefix = config.get("prefix", "/")
self.ReplacePrefix = config.get("replace_prefix")
self.Timeout = config.get("timeout", 10)
saved_path = sys.path[:]
saved_modules = set(sys.modules.keys())
saved_environ = os.environ.copy()
try:
args = None
fname = config["file"]
g = {}
extra_path = config.get("python_path")
if extra_path is not None:
if isinstance(extra_path, str):
extra_path = [extra_path]
sys.path = extra_path + sys.path
if "env" in config:
os.environ.update(config["env"])
exec(open(fname, "r").read(), g)
if "create" in config:
args = config.get("args")
app = g[config["create"]](args)
else:
app = g[config.get("application", "application")]
self.AppArgs = args
self.WSGIApp = app
max_workers = config.get("max_workers", 5)
queue_capacity = config.get("queue_capacity", 10)
self.RequestQueue = TaskQueue(max_workers, capacity = queue_capacity,
delegate=self)
self.log("(re)configured")
finally:
sys.path = saved_path
extra_modules = set(sys.modules.keys()) - set(saved_modules)
#print("loadApp: removing modules:", sorted(list(extra_modules)))
for m in extra_modules:
del sys.modules[m]
for n in set(os.environ.keys()) - set(saved_environ.keys()):
del os.environ[n]
os.environ.update(saved_environ)
def taskFailed(self, queue, task, exc_type, exc_value, tb):
self.log_error("request failed:", "\n".join(traceback.format_exception(exc_type, exc_value, tb)))
def accept(self, request):
header = request.HTTPHeader
uri = header.URI
self.debug("accept: uri:", uri, " prefix:", self.Prefix)
if uri.startswith(self.Prefix):
uri = uri[len(self.Prefix):]
if not uri.startswith("/"): uri = "/" + uri
if self.ReplacePrefix:
uri = self.ReplacePrefix + uri
header.replaceURI(uri)
request.AppName = self.Instance
self.RequestQueue.addTask(RequestTask(self.WSGIApp, request, self.Logger))
return True
else:
return False
def mtime(self, path):
try: return os.path.getmtime(path)
except: return None
def reloadIfNeeded(self):
for path, old_timestamp in self.ReloadFileTimestamps.items():
mt = self.mtime(path)
if mt is not None and mt != old_timestamp:
ct = time.ctime(mt)
self.log(f"file {path} was modified at {ct}")
break
else:
return False
self.configure()
from pythreader import TaskQueue, Task
from threading import Timer
class MyTask(Task):
def __init__(self, tid):
Task.__init__(self)
self.Id = tid
def run(self):
print(time.time(), self.Id, "started as instance")
time.sleep(random.random() * 3)
print(self.Id, "ended")
def failed(self, e):
print(e)
q = TaskQueue(5, stagger=1.0, tasks=[MyTask(x) for x in range(10)])
q << MyTask(30) << MyTask(31)
MyTask(32) >> q
q += MyTask(33)
Timer(3.5, lambda q: q.addTask(MyTask(40)), (q, )).start()
Timer(3.51, lambda q: q.addTask(MyTask(41)), (q, )).start()
Timer(3.52, lambda q: q.addTask(MyTask(42)), (q, )).start()
Timer(3.53, lambda q: q.addTask(MyTask(43)), (q, )).start()
Timer(3.9, lambda q: q.addTask(MyTask(44)), (q, )).start()
q.waitUntilEmpty()
def initialize(self, config=None):
config = config or self.Config
self.Config = config
reload_files = config.get("touch_reload", [])
if isinstance(reload_files, str):
reload_files = [reload_files]
self.ReloadFileTimestamps = {
path: self.mtime(path)
for path in reload_files
}
self.Prefix = config.get("prefix", "/")
self.ReplacePrefix = config.get("replace_prefix")
self.Timeout = config.get("timeout", 10)
saved_path = sys.path[:]
saved_modules = set(sys.modules.keys())
saved_environ = os.environ.copy()
try:
args = None
if "file" in config:
print(
'*** Use of "file" parameter is deprecated. Use "module" instead'
)
self.ScriptFileName = fname = config.get("module",
config.get("file"))
g = {}
extra_path = config.get("python_path")
if extra_path is not None:
if isinstance(extra_path, str):
extra_path = [extra_path]
sys.path = extra_path + sys.path
if "env" in config:
os.environ.update(config["env"])
try:
exec(open(fname, "r").read(), g)
except:
tb = traceback.format_exc()
self.log_error(f"Error importing module {fname}:\n{tb}")
return False
if "create" in config:
# deprecated
print(
'*** Use of "create" parameter is deprecated. Use "application: function()" instead'
)
application = config["create"] + "()"
else:
application = config.get("application", "application")
if application.endswith("()"):
args = config.get("args")
fcn_name = application[:-2]
fcn = g.get(fcn_name)
if fcn is None:
self.log_error(
f"Application creation function {fcn_name} not found in module {fname}"
)
return False
try:
if isinstance(args, dict):
app = fcn(**args)
elif isinstance(args, (list, tuple)):
app = fcn(*args)
elif args is None:
app = fcn()
else:
app = fcn(args)
except:
tb = traceback.format_exc()
self.log_error(
f"Error calling the application initialization function:\n{tb}"
)
return False
if app is None:
self.log_error(
f'Application creation function {fcn_name} returned None'
)
return False
else:
app = g.get(application)
if app is None:
self.log_error(
f'Application object "{application}" not found in {fname}'
)
return False
self.AppArgs = args
self.WSGIApp = app
max_workers = config.get("max_workers", 5)
queue_capacity = config.get("queue_capacity", 10)
self.RequestQueue = TaskQueue(max_workers,
capacity=queue_capacity,
delegate=self)
self.log("initiaized")
except:
tb = traceback.format_exc()
self.log_error(f"Error initializing application:\n{tb}")
return False
finally:
sys.path = saved_path
extra_modules = set(sys.modules.keys()) - set(saved_modules)
#print("loadApp: removing modules:", sorted(list(extra_modules)))
for m in extra_modules:
del sys.modules[m]
for n in set(os.environ.keys()) - set(saved_environ.keys()):
del os.environ[n]
os.environ.update(saved_environ)
return True
-s <stagger>, default = 10 (seconds)
"""
MaxWorkers = 5
BatchSize = 10
Stagger = 10
opts, args = getopt.getopt(sys.argv[1:], "m:n:s:")
for opt, val in opts:
if opt == "-m": MaxWorkers = int(val)
elif opt == "-n": BatchSize = int(val)
elif opt == "-s": Stagger = int(val)
if len(args) != 4:
print Usage
sys.exit(1)
SchemaFile, Directory, BucketName, DatasetName = args
files = sorted(glob.glob("%s/*.fits" % (Directory, )))
tq = TaskQueue(MaxWorkers)
i = 0
while files:
batch = files[:BatchSize]
files = files[BatchSize:]
t = FileLoaderTask(i, SchemaFile, batch, BucketName, DatasetName)
tq << t
time.sleep(Stagger)
i += 1
-N <n> - run all <n> tasks at once
"""
MaxPipes = 30
NRUN = 1000
stagger = 0.1
output = None #open("/dev/null", "w")
opts, args = getopt.getopt(sys.argv[1:], "m:n:s:N:")
for opt, val in opts:
if opt == '-m': MaxPipes = int(val)
if opt == '-n': NRUN = int(val)
if opt == '-s': stagger = float(val)
if opt == '-N':
NRUN = int(val)
MaxPipes = NRUN
if not args:
print Usage
sys.exit(1)
command = args
tq = TaskQueue(MaxPipes)
for i in xrange(NRUN):
tq.addTask(SubprocessTask(i, NRUN, command))
time.sleep(stagger)
tq.waitUntilEmpty()
class Service(Primitive, Logged):
def __init__(self, config, logger=None):
name = config["name"]
#print("Service(): config:", config)
self.ServiceName = name
Primitive.__init__(self, name=f"[service {name}]")
Logged.__init__(self, f"[app {name}]", logger, debug=True)
self.Config = None
self.Initialized = self.initialize(config)
@synchronized
def initialize(self, config=None):
config = config or self.Config
self.Config = config
reload_files = config.get("touch_reload", [])
if isinstance(reload_files, str):
reload_files = [reload_files]
self.ReloadFileTimestamps = {
path: self.mtime(path)
for path in reload_files
}
self.Prefix = config.get("prefix", "/")
self.ReplacePrefix = config.get("replace_prefix")
self.Timeout = config.get("timeout", 10)
saved_path = sys.path[:]
saved_modules = set(sys.modules.keys())
saved_environ = os.environ.copy()
try:
args = None
if "file" in config:
print(
'*** Use of "file" parameter is deprecated. Use "module" instead'
)
self.ScriptFileName = fname = config.get("module",
config.get("file"))
g = {}
extra_path = config.get("python_path")
if extra_path is not None:
if isinstance(extra_path, str):
extra_path = [extra_path]
sys.path = extra_path + sys.path
if "env" in config:
os.environ.update(config["env"])
try:
exec(open(fname, "r").read(), g)
except:
tb = traceback.format_exc()
self.log_error(f"Error importing module {fname}:\n{tb}")
return False
if "create" in config:
# deprecated
print(
'*** Use of "create" parameter is deprecated. Use "application: function()" instead'
)
application = config["create"] + "()"
else:
application = config.get("application", "application")
if application.endswith("()"):
args = config.get("args")
fcn_name = application[:-2]
fcn = g.get(fcn_name)
if fcn is None:
self.log_error(
f"Application creation function {fcn_name} not found in module {fname}"
)
return False
try:
if isinstance(args, dict):
app = fcn(**args)
elif isinstance(args, (list, tuple)):
app = fcn(*args)
elif args is None:
app = fcn()
else:
app = fcn(args)
except:
tb = traceback.format_exc()
self.log_error(
f"Error calling the application initialization function:\n{tb}"
)
return False
if app is None:
self.log_error(
f'Application creation function {fcn_name} returned None'
)
return False
else:
app = g.get(application)
if app is None:
self.log_error(
f'Application object "{application}" not found in {fname}'
)
return False
self.AppArgs = args
self.WSGIApp = app
max_workers = config.get("max_workers", 5)
queue_capacity = config.get("queue_capacity", 10)
self.RequestQueue = TaskQueue(max_workers,
capacity=queue_capacity,
delegate=self)
self.log("initiaized")
except:
tb = traceback.format_exc()
self.log_error(f"Error initializing application:\n{tb}")
return False
finally:
sys.path = saved_path
extra_modules = set(sys.modules.keys()) - set(saved_modules)
#print("loadApp: removing modules:", sorted(list(extra_modules)))
for m in extra_modules:
del sys.modules[m]
for n in set(os.environ.keys()) - set(saved_environ.keys()):
del os.environ[n]
os.environ.update(saved_environ)
return True
def taskFailed(self, queue, task, exc_type, exc_value, tb):
self.log_error(
"request failed:",
"".join(traceback.format_exception(exc_type, exc_value, tb)))
try:
task.Request.close()
except:
pass
def accept(self, request):
#print(f"Service {self}: accept()")
if not self.Initialized:
return False
header = request.HTTPHeader
uri = header.URI
self.debug("accept: uri:", uri, " prefix:", self.Prefix)
#print("Sevice", self," accept: uri:", uri, " prefix:", self.Prefix)
if uri.startswith(self.Prefix):
uri = uri[len(self.Prefix):]
if not uri.startswith("/"): uri = "/" + uri
if self.ReplacePrefix:
uri = self.ReplacePrefix + uri
header.replaceURI(uri)
request.AppName = self.ServiceName
script_path = self.Prefix
while script_path and script_path.endswith("/"):
script_path = script_path[:-1]
request.Environ["SCRIPT_NAME"] = script_path
request.Environ["SCRIPT_FILENAME"] = self.ScriptFileName
self.RequestQueue.addTask(
RequestTask(self.WSGIApp, request, self.Logger))
#print("Service", self, " accepted")
return True
else:
#print("Service", self, " rejected")
return False
def close(self):
self.RequestQueue.hold()
def join(self):
self.RequestQueue.join()
def mtime(self, path):
try:
return os.path.getmtime(path)
except:
return None
def reloadIfNeeded(self):
for path, old_timestamp in self.ReloadFileTimestamps.items():
mt = self.mtime(path)
if mt is not None and mt != old_timestamp:
ct = time.ctime(mt)
self.log(f"file {path} was modified at {ct}")
break
else:
return False
self.Initialized = self.initialize()
def purgeEmptyDirs(self):
if self.EmptyDirs:
queue = TaskQueue(self.MaxScanners)
for path in self.EmptyDirs:
queue.addTask(RMDir(self.Server, path))
queue.waitUntilEmpty()
def __init__(self, app, logger=None):
Logged.__init__(self, f"[app {app.__class__.__name__}]", logger)
self.Name = app.__class__.__name__
self.WPApp = app
self.ProcessorQueue = TaskQueue(5)
class ScannerMaster(PyThread):
MAX_RECURSION_FAILED_COUNT = 5
REPORT_INTERVAL = 10.0
def __init__(self, server, root, recursive_threshold, max_scanners, timeout, quiet, display_progress, max_files = None,
include_sizes=True, ignore_subdirs=[]):
PyThread.__init__(self)
self.RecursiveThreshold = recursive_threshold
self.Server = server
self.Root = canonic_path(root)
self.MaxScanners = max_scanners
self.Results = DEQueue()
self.ScannerQueue = TaskQueue(max_scanners)
self.Timeout = timeout
self.Done = False
self.Error = None
self.Failed = False
self.RootFailed = False
self.Directories = set()
self.RecursiveFailed = {} # parent path -> count
self.Errors = {} # location -> count
self.GaveUp = {}
self.LastReport = time.time()
self.EmptyDirs = set()
self.NScanned = 0
self.NToScan = 1
self.Quiet = quiet
self.DisplayProgress = display_progress and Use_tqdm and not quiet
if self.DisplayProgress:
self.TQ = tqdm.tqdm(total=self.NToScan, unit="dir")
self.LastV = 0
self.NFiles = self.NDirectories = 0
self.MaxFiles = max_files # will stop after number of files found exceeds this number. Used for debugging
self.IgnoreSubdirs = ignore_subdirs
self.IgnoredFiles = self.IgnoredDirs = 0
self.IncludeSizes = include_sizes
self.TotalSize = 0.0 if include_sizes else None # Megabytes
def run(self):
#
# scan Root non-recursovely first, if failed, return immediarely
#
#server, location, recursive, timeout
scanner_task = Scanner(self, self.Server, self.Root, self.RecursiveThreshold == 0, self.Timeout, include_sizes=self.IncludeSizes)
self.ScannerQueue.addTask(scanner_task)
self.ScannerQueue.waitUntilEmpty()
self.Results.close()
self.ScannerQueue.Delegate = None # detach for garbage collection
self.ScannerQueue = None
def dir_ignored(self, path):
# path is expected to be canonic here
relpath = relative_path(self.Root, path)
ignore = any((relpath == subdir or relpath.startswith(subdir+"/")) for subdir in self.IgnoreSubdirs)
return ignore
def file_ignored(self, path):
# path is expected to be canonic here
relpath = relative_path(self.Root, path)
return any(relpath.startswith(subdir+"/") for subdir in self.IgnoreSubdirs)
@synchronized
def addFiles(self, files):
if not self.Failed:
self.Results.append(('f', files))
self.NFiles += len(files)
def parent(self, path):
parts = path.rsplit("/", 1)
if len(parts) < 2:
return "/"
else:
return parts[0]
def addDirectory(self, path, scan, allow_recursive):
if scan and not self.Failed:
assert path.startswith(self.Root)
relpath = path[len(self.Root):]
while relpath and relpath[0] == '/':
relpath = relpath[1:]
while relpath and relpath[-1] == '/':
relpath = relpath[:-1]
reldepth = 0 if not relpath else len(relpath.split('/'))
parent = self.parent(path)
allow_recursive = allow_recursive and (self.RecursiveThreshold is not None
and reldepth >= self.RecursiveThreshold
)
if self.MaxFiles is None or self.NFiles < self.MaxFiles:
self.ScannerQueue.addTask(
Scanner(self, self.Server, path, allow_recursive, self.Timeout, include_sizes=self.IncludeSizes)
)
self.NToScan += 1
def addDirectories(self, dirs, scan=True, allow_recursive=True):
if not self.Failed:
self.Results.append(('d', dirs))
self.NDirectories += len(dirs)
for d in dirs:
d = canonic_path(d)
if self.dir_ignored(d):
if scan:
print(d, " - ignored")
self.IgnoredDirs += 1
else:
self.addDirectory(d, scan, allow_recursive)
self.show_progress()
self.report()
@synchronized
def report(self):
if time.time() > self.LastReport + self.REPORT_INTERVAL:
waiting, active = self.ScannerQueue.tasks()
#sys.stderr.write("--- Locations to scan: %d\n" % (len(active)+len(waiting),))
self.LastReport = time.time()
@synchronized
def scanner_failed(self, scanner, error):
path = scanner.Location
with self:
# update error counts
if scanner.WasRecursive:
parent = self.parent(path)
self.RecursiveFailed[parent] = self.RecursiveFailed.get(parent, 0) + 1
else:
self.Errors[path] = self.Errors.get(path, 0) + 1
retry = (scanner.RecAttempts > 0) or (scanner.FlatAttempts > 0)
if retry:
print("resubmitted:", scanner.Location, scanner.RecAttempts, scanner.FlatAttempts)
self.ScannerQueue.addTask(scanner)
else:
print("Gave up:", scanner.Location)
self.GaveUp[scanner.Location] = error
self.NScanned += 1
#sys.stderr.write("Gave up on: %s\n" % (path,))
self.show_progress() #"Error scanning %s: %s -- retrying" % (scanner.Location, error))
@synchronized
def scanner_succeeded(self, location, was_recursive, files, dirs):
with self:
if len(files) == 0 and (was_recursive or len(dirs) == 0):
self.EmptyDirs.add(location)
else:
if location in self.EmptyDirs:
self.EmptyDirs.remove(location)
if was_recursive:
parent = self.parent(location)
nfailed = self.RecursiveFailed.get(parent, 0)
self.RecursiveFailed[parent] = nfailed - 1
self.NScanned += 1
if files:
paths, sizes = zip(*files)
self.addFiles(paths)
#for path, size in files:
# print(f"path: {path}, size:{size}")
#print("total size:", sum(sizes), location)
if self.IncludeSizes:
self.TotalSize += sum(sizes)
if dirs:
paths, sizes = zip(*dirs)
scan = not was_recursive
allow_recursive = scan and len(dirs) > 1
self.addDirectories(paths, scan, allow_recursive)
#if self.IncludeSizes:
# self.TotalSize += sum(sizes)
self.show_progress()
def files(self):
yield from self.paths('f')
def paths(self, type=None):
for t, lst in self.Results:
if lst and (type is None or type == t):
for path in lst:
path = canonic_path(path)
if self.file_ignored(path):
self.IgnoredFiles += 1
else:
if type is None:
yield t, path
else:
yield path
@synchronized
def show_progress(self, message=None):
if self.DisplayProgress:
self.TQ.total = self.NToScan
delta = max(0, self.NScanned - self.LastV)
self.TQ.update(delta)
self.LastV = self.NScanned
enf = 0
if self.NScanned > 0:
enf = int(self.NFiles * self.NToScan/self.NScanned)
self.TQ.set_postfix(f=self.NFiles, ed=len(self.EmptyDirs), d=self.NDirectories, enf=enf)
if message:
self.TQ.write(message)
@synchronized
def message(self, message):
if not self.Quiet:
if self.DisplayProgress:
self.TQ.write(message)
else:
print(message)
sys.stdout.flush()
def close_progress(self):
if self.DisplayProgress:
self.TQ.close()
def purgeEmptyDirs(self):
if self.EmptyDirs:
queue = TaskQueue(self.MaxScanners)
for path in self.EmptyDirs:
queue.addTask(RMDir(self.Server, path))
queue.waitUntilEmpty()
class HTTPServer(PyThread, Logged):
def __init__(self,
port,
app=None,
services=[],
sock=None,
logger=None,
max_connections=100,
timeout=20.0,
enabled=True,
max_queued=100,
logging=False,
log_file="-",
debug=None,
certfile=None,
keyfile=None,
verify="none",
ca_file=None,
password=None):
PyThread.__init__(self)
self.Port = port
self.Sock = sock
assert self.Port is not None, "Port must be specified"
if logger is None and logging:
logger = Logger(log_file)
#print("logs sent to:", f)
Logged.__init__(self, f"[server {self.Port}]", logger, debug=True)
self.Logger = logger
self.Timeout = timeout
max_connections = max_connections
queue_capacity = max_queued
self.RequestReaderQueue = TaskQueue(max_connections,
capacity=queue_capacity,
delegate=self)
self.SocketWrapper = SSLSocketWrapper(
certfile, keyfile, verify, ca_file, password) if keyfile else None
if app is not None:
services = [Service(app, logger)]
self.Services = services
self.Stop = False
def close(self):
self.Stop = True
self.RequestReaderQueue.hold()
def join(self):
self.RequestReaderQueue.join()
@staticmethod
def from_config(config,
services,
logger=None,
logging=False,
log_file=None,
debug=None):
port = config["port"]
timeout = config.get("timeout", 20.0)
max_connections = config.get("max_connections", 100)
queue_capacity = config.get("queue_capacity", 100)
# TLS
certfile = config.get("cert")
keyfile = config.get("key")
verify = config.get("verify", "none")
ca_file = config.get("ca_file")
password = config.get("password")
#print("HTTPServer.from_config: services:", services)
return HTTPServer(port,
services=services,
logger=logger,
max_connections=max_connections,
timeout=timeout,
max_queued=queue_capacity,
logging=logging,
log_file=log_file,
debug=debug,
certfile=certfile,
keyfile=keyfile,
verify=verify,
ca_file=ca_file,
password=password)
def setServices(self, services):
self.Services = services
def connectionCount(self):
return len(self.Connections)
def run(self):
if self.Sock is None:
# therwise use the socket supplied to the constructior
self.Sock = socket(AF_INET, SOCK_STREAM)
self.Sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
self.Sock.bind(('', self.Port))
self.Sock.listen(10)
while not self.Stop:
self.debug("--- accept loop port=%d start" % (self.Port, ))
csock = None
caddr = ('-', '-')
try:
csock, caddr = self.Sock.accept()
self.connection_accepted(csock, caddr)
except Exception as exc:
#print(exc)
if not self.Stop:
self.debug("connection processing error: %s" %
(traceback.format_exc(), ))
self.log_error(caddr,
"Error processing connection: %s" % (exc, ))
if csock is not None:
try:
csock.close()
except:
pass
self.debug("--- accept loop port=%d end" % (self.Port, ))
if self.Stop: self.debug("stopped")
try:
self.Sock.close()
except:
pass
self.Sock = None
def connection_accepted(self, csock,
caddr): # called externally by multiserver
request = Request(self.Port, csock, caddr)
self.debug("connection %s accepted from %s:%s" %
(request.Id, caddr[0], caddr[1]))
reader = RequestReader(self, request, self.SocketWrapper, self.Timeout,
self.Logger)
self.RequestReaderQueue << reader
@synchronized
def stop(self):
self.Stop = True
try:
self.Sock.close()
except:
pass
@synchronized
def dispatch(self, request):
for service in self.Services:
if service.accept(request):
return service
else:
return None
def taskFailed(self, queue, task, exc_type, exc, tb):
traceback.print_exception(exc_type, exc, tb)
class TCPServerThread(PyThread):
def __init__(self,
port,
ip='',
max_clients=None,
queue_capacity=None,
stagger=None,
enabled=True):
PyThread.__init__(self)
self.Sock = None
self.Clients = TaskQueue(max_clients,
capacity=queue_capacity,
stagger=stagger)
self.Port = port
self.IP = ip
self.Enabled = enabled
self.Shutdown = False
self.LastIdle = 0.0
def run(self):
self.Sock = socket(AF_INET, SOCK_STREAM)
self.Sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
self.Sock.bind((self.IP, self.Port))
if self.Enabled:
self.enable()
while not self.Shutdown:
fd = self.Sock.fileno()
#print("TCPServerThread: select...")
r, w, e = select.select([fd], [fd], [fd], 1.0)
#print("TCPServerThread: r,w,e:", r, w, e)
if fd in r or fd in e or fd in w:
csock, caddr = self.Sock.accept()
task = self.createClientInterface(csock, caddr)
#print ("TCPServerThread: client task created:", task)
if task is not None:
self.Clients.addTask(task)
else:
t = time.time()
self.idle(t, self.LastIdle)
self.LastIdle = t
self.Sock.close()
@synchronized
def enable(self, backlog=5):
if self.Sock is not None:
self.Sock.listen(backlog)
self.Clients.release()
self.Enabled = True
@synchronized
def disable(self):
self.Sock.listen(0)
self.Clients.hold()
self.Enabled = False
def shutdown(self):
self.Shutdown = True
def waitForClients(self):
self.Clients.waitUntilEmpty()
# overridables
def idle(self, now, last_idle):
pass
def createClientInterface(self, sock, addr):
return None
pass # virtual
def __init__(self, cfg):
Primitive.__init__(self)
self.Replicators = TaskQueue(cfg.get('max_rep', 2), stagger=0.5)
self.DClient = DataClient((cfg["broadcast"], cfg["listen_port"]),
cfg["farm_name"])
class HTTPServer(PyThread):
def __init__(self,
port,
app,
max_connections=100,
timeout=10.0,
enabled=True,
max_queued=100,
logging=False,
log_file=None,
debug=None):
PyThread.__init__(self)
#self.debug("Server started")
self.Port = port
self.Timeout = timeout
self.WSGIApp = app
self.Enabled = False
self.Logging = logging
self.LogFile = sys.stdout if log_file is None else log_file
self.Connections = TaskQueue(max_connections, capacity=max_queued)
if enabled:
self.enableServer()
self.Debug = debug
@synchronized
def debug(self, msg):
#print("debug: %s %s" % (type(self.Debug), self.Debug))
if self.Debug:
self.Debug.write("%s: [debug] %s\n" % (time.ctime(), msg))
if self.Debug is sys.stdout:
self.Debug.flush()
@synchronized
def log(self, caddr, method, uri, status, bytes_sent):
if self.Logging:
self.LogFile.write("{}: {}:{} {} {} {} {}\n".format(
time.ctime(), caddr[0], caddr[1], method, uri, status,
bytes_sent))
if self.LogFile is sys.stdout:
self.LogFile.flush()
@synchronized
def log_error(self, caddr, message):
if self.Logging:
self.LogFile.write("{}: {}:{} {}\n".format(time.ctime(), caddr[0],
caddr[1], message))
if self.LogFile is sys.stdout:
self.LogFile.flush()
else:
print("{}: {}:{} {}\n".format(time.ctime(), caddr[0], caddr[1],
message))
def wsgi_app(self, env, start_response):
#print("server.wsgi_app")
return self.WSGIApp(env, start_response)
@synchronized
def enableServer(self, backlog=5):
self.Enabled = True
@synchronized
def disableServer(self):
self.Enabled = False
def connectionClosed(self, conn):
pass
@synchronized
def connectionCount(self):
return len(self.Connections)
def run(self):
self.Sock = socket(AF_INET, SOCK_STREAM)
self.Sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
self.Sock.bind(('', self.Port))
self.Sock.listen(10)
while True:
self.debug("--- accept loop port=%d start" % (self.Port, ))
csock = None
caddr = ('-', '-')
try:
csock, caddr = self.Sock.accept()
cid = uid()
self.debug("connection %s accepted from %s:%s" %
(cid, caddr[0], caddr[1]))
conn = self.createConnection(cid, csock, caddr)
if conn is not None:
self.Connections << conn
self.debug(
"%s from %s queued. Active/queued connections: %d/%d" %
(conn, caddr, len(self.Connections.activeTasks()),
len(self.Connections.waitingTasks())))
except Exception as exc:
self.debug("connection processing error: %s" %
(traceback.format_exc(), ))
self.log_error(caddr,
"Error processing connection: %s" % (exc, ))
if csock is not None:
try:
csock.close()
except:
pass
self.debug("--- accept loop port=%d end" % (self.Port, ))
# overridable
def createConnection(self, cid, csock, caddr):
return HTTPConnection(cid, self, csock, caddr)
def wrap_socket(self, sock):
return sock, None
print(f"No flights for {airport} (lo:[{lomin}:{lomax}], la:[{lamin}:{lamax})]")
return []
timestamp = data["time"]
data = [
{"icao":d[ICAO24].strip(), "callsign":d[CALL].strip(), "country":d[COUNTRY]}
for d in states
]
self.DB.add(timestamp, airport, data)
if __name__ == "__main__":
tstart = time.time()
window = 20 # km
db = Database("flights.sqlite")
db.start()
scanner_queue = TaskQueue(10, stagger=0.5)
for airport in Airports.keys():
scanner_queue.addTask(ScanTask(db, airport))
scanner_queue.waitUntilEmpty()
db.stop()
db.join()
tend = time.time()
print("runtime:", tend-tstart)
return "Task<Error>"
class Delegate(object):
def taskFailed(self, queue, task, etype, evalue, etb):
print(f"Delegate: exception reported in task {task}")
traceback.print_exception(etype, evalue, etb)
def taskEnded(self, queue, task):
print(f"Delegate: task ended: {task}")
d = Delegate()
q1 = TaskQueue(2)
q2 = TaskQueue(2, delegate = d)
tok = TaskOK()
terr = TaskError()
print("-------- Queue without delegae --------")
for _ in range(100):
q1 << tok << terr
q1.join()
print("-------- Queue with delegae --------")
for _ in range(100):
q2 << tok << terr
q2.join()