def __init__(self, node, ip, edge_port, diagonal_port):
PyThread.__init__(self)
self.Node = node
self.Index = node.Index
self.ListenSock = None
self.DownStream = None
self.DownIndex = None
def __init__(self, server, root, recursive_threshold, max_scanners, timeout, quiet, display_progress, max_files = None,
include_sizes=True, ignore_subdirs=[]):
PyThread.__init__(self)
self.RecursiveThreshold = recursive_threshold
self.Server = server
self.Root = canonic_path(root)
self.MaxScanners = max_scanners
self.Results = DEQueue()
self.ScannerQueue = TaskQueue(max_scanners)
self.Timeout = timeout
self.Done = False
self.Error = None
self.Failed = False
self.RootFailed = False
self.Directories = set()
self.RecursiveFailed = {} # parent path -> count
self.Errors = {} # location -> count
self.GaveUp = {}
self.LastReport = time.time()
self.EmptyDirs = set()
self.NScanned = 0
self.NToScan = 1
self.Quiet = quiet
self.DisplayProgress = display_progress and Use_tqdm and not quiet
if self.DisplayProgress:
self.TQ = tqdm.tqdm(total=self.NToScan, unit="dir")
self.LastV = 0
self.NFiles = self.NDirectories = 0
self.MaxFiles = max_files # will stop after number of files found exceeds this number. Used for debugging
self.IgnoreSubdirs = ignore_subdirs
self.IgnoredFiles = self.IgnoredDirs = 0
self.IncludeSizes = include_sizes
self.TotalSize = 0.0 if include_sizes else None # Megabytes
def __init__(self, dbfile):
PyThread.__init__(self)
self.DBFile = dbfile
self.DB = None
self.InputQueue = DEQueue()
self.Stop = False
def __init__(self, config, logger=None):
name = config["name"]
self.AppName = name
PyThread.__init__(self, name=f"[app {name}]", daemon=True)
Logged.__init__(self, f"[app {name}]", logger, debug=True)
self.Config = None
self.configure(config)
self.Stop = False
def __init__(self, node, seed_nodes):
PyThread.__init__(self)
self.Node = node
self.SeedNodes = seed_nodes #[(inx, ip, port), ...]
self.UpStream = None
self.UpNodeID = None
self.UpAddress = None
self.Shutdown = False
def __init__(self, mgr, sock):
PyThread.__init__(self)
self.Sock = sock
self.Manager = mgr
self.NodeID = None
self.Address = None
self.Stream = None
self.Stop = False
def __init__(self, node, nodes):
PyThread.__init__(self)
self.Node = node
self.ListenSock = None
self.DownStream = None
self.Index, self.ListenSock, self.Address = self.bind(nodes)
#print("DownLink bound to:", self.Address)
self.ListenSock.listen()
self.DownNodeAddress = None
def __init__(self, backend, dataset_name):
PyThread.__init__(self)
self.Queue = Queue(self.QUEUESIZE)
self.DatasetName = dataset_name
self.Backend = backend
self.Buf = {}
self.TotalBytes = 0
self.Shutdown = False
self.T = Tracer()
def __init__(self, node, seed_nodes):
PyThread.__init__(self)
self.Node = node
self.ListenSock = None
self.DownConnection = None
self.Index, self.ListenSock, self.Address = self.bind(seed_nodes)
self.ListenSock.listen()
self.DownNodeAddress = None
self.Shutdown = False
def __init__(self, config_file, logger=None):
PyThread.__init__(self)
Logged.__init__(self, "[Multiserver]", logger, debug=True)
self.ConfigFile = config_file
self.Servers = []
self.ServersByPort = {}
self.ReconfiguredTime = 0
self.reconfigure()
self.debug("debug is enabled")
def __init__(self, ether, topics):
PyThread.__init__(self)
PubSubAgent.__init__(self, ether)
self.Topics = topics
self.Words = None
try:
with open("/usr/share/dict/words", "r") as words:
self.Words = [w.strip() for w in words]
except:
pass
def __init__(self, node, address, max_diagonals=3):
PyThread.__init__(self)
self.Node = node
self.IP, self.Port = address
self.Sock = None
self.DiagonalNodes = []
self.NDiagonals = max_diagonals
self.Sock = dsock = socket(AF_INET, SOCK_DGRAM)
dsock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
#print("DiagonalLink: binding to:", (self.IP, self.Port))
dsock.bind((self.IP, self.Port))
def __init__(self,
port,
app,
max_connections=10,
max_queued=30,
**ws_args):
PyThread.__init__(self)
self.Port = port
self.HandlerQueue = TaskQueue(max_connections, capacity=max_queued)
self.App = app
self.WSArgs = ws_args
def __init__(self, myid, cfg, storage):
PyThread.__init__(self)
self.ID = myid
self.DSAddr = (cfg['host'], cfg['cellif_port'])
self.Connected = 0
self.Reconciled = 0
self.LastIdle = 0
self.NextReconnect = 0
self.NextProbeTime = 0
#self.connect()
self.CellStorage = storage
def __init__(self, config_file):
self.ConfigFile = config_file
self.Config = config = expand(
yaml.load(open(self.ConfigFile, 'r'), Loader=yaml.SafeLoader))
#self.Config = config = yaml.load(open(self.ConfigFile, 'r'), Loader=yaml.SafeLoader)
log_file = config.get("log", "-")
self.Logger = Logger(log_file)
Logged.__init__(self, "[router]", self.Logger)
PyThread.__init__(self, name="[router]", daemon=True)
self.ConfigMTime = mtime(self.ConfigFile)
self.Apps = None
self.configure(config)
self.Stop = False
def __init__(self, node, nodes, ndiagonals):
PyThread.__init__(self)
self.Node = node
self.Index = node.Index
# nodes: [(ip, edge_port, diagonal_port)] indexed by the node index
nodes = [(inx, ip, edge_port, diagonal_port)
for inx, (ip, edge_port, diagonal_port) in enumerate(nodes)]
self.UpNodes = (nodes[inx:] +
nodes[:inx])[::-1] # sorted in upwards order
self.DiagonalNodes = []
self.NDiagonals = ndiagonals
self.UpStream = None
self.UpIndex = None
def __init__(self, myid, cfg, data_server, cell_storage, vfs_server_if):
PyThread.__init__(self)
self.MyID = myid
self.Port = cfg['listen_port']
self.FarmName = cfg['farm_name']
self.Sock = socket(AF_INET, SOCK_DGRAM)
self.Sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
self.Sock.bind(('', self.Port))
self.MyHost = gethostbyname(gethostname())
self.Enabled = False
self.DataServer = data_server
self.CellStorage = cell_storage
self.VFSSrvIF = vfs_server_if
def __init__(self, node, inx, ip, port):
PyThread.__init__(self)
self.Node = node
self.Index = inx
self.IP = ip
self.Port = port
self.ListenSock = None
self.DownStream = None
self.DownIndex = None
self.ListenSock = socket(AF_INET, SOCK_STREAM)
self.ListenSock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
self.ListenSock.bind((self.IP, self.Port))
self.ListenSock.listen(1)
def __init__(self, config_file, logger=None, debug=False):
PyThread.__init__(self)
Logged.__init__(self, "[Multiserver]", logger, debug=debug)
self.ConfigFile = config_file
self.Server = None
self.Port = None
self.ReconfiguredTime = 0
self.Subprocesses = []
self.Sock = None
self.Stop = False
self.MPLogger = None
if logger is not None:
self.MPLogger = MPLogger(logger, debug=debug)
self.MPLogger.start()
self.Debug = debug
self.reconfigure()
def __init__(self,
port,
ip='',
max_clients=None,
queue_capacity=None,
stagger=None,
enabled=True):
PyThread.__init__(self)
self.Sock = None
self.Clients = TaskQueue(max_clients,
capacity=queue_capacity,
stagger=stagger)
self.Port = port
self.IP = ip
self.Enabled = enabled
self.Shutdown = False
self.LastIdle = 0.0
def __init__(self,
port,
app=None,
services=[],
sock=None,
logger=None,
max_connections=100,
timeout=20.0,
enabled=True,
max_queued=100,
logging=False,
log_file="-",
debug=None,
certfile=None,
keyfile=None,
verify="none",
ca_file=None,
password=None):
PyThread.__init__(self)
self.Port = port
self.Sock = sock
assert self.Port is not None, "Port must be specified"
if logger is None and logging:
logger = Logger(log_file)
#print("logs sent to:", f)
Logged.__init__(self, f"[server {self.Port}]", logger, debug=True)
self.Logger = logger
self.Timeout = timeout
max_connections = max_connections
queue_capacity = max_queued
self.RequestReaderQueue = TaskQueue(max_connections,
capacity=queue_capacity,
delegate=self)
self.SocketWrapper = SSLSocketWrapper(
certfile, keyfile, verify, ca_file, password) if keyfile else None
if app is not None:
services = [Service(app, logger)]
self.Services = services
self.Stop = False
def __init__(self, inx, map, sel):
PyThread.__init__(self)
self.UpSock = None
self.UpStr = None
self.DnSock = None
self.DnStr = None
self.Sel = sel
self.Map = map
self.UpInx = None
self.DnInx = None
self.Inx = self.initServerSock(inx, map)
if self.Inx is None:
# some error
raise ChainException('Can not allocate Chain port')
self.Sel.register(self, rd = self.SSock.fileno())
self.Token = None
self.PusherSeq = None
self.IgnorePusher = -1
self.LastPushSeq = 0
self.LastPing = 0
print ("ChainSegment: connected")
def __init__(self,
port,
app,
max_connections=100,
timeout=10.0,
enabled=True,
max_queued=100,
logging=False,
log_file=None,
debug=None):
PyThread.__init__(self)
#self.debug("Server started")
self.Port = port
self.Timeout = timeout
self.WSGIApp = app
self.Enabled = False
self.Logging = logging
self.LogFile = sys.stdout if log_file is None else log_file
self.Connections = TaskQueue(max_connections, capacity=max_queued)
if enabled:
self.enableServer()
self.Debug = debug
def __init__(self,
port,
app,
remove_prefix="",
url_pattern="*",
max_connections=100,
enabled=True,
max_queued=100,
logging=True,
log_file=None):
PyThread.__init__(self)
#self.debug("Server started")
self.Port = port
self.WSGIApp = app
self.Match = url_pattern
self.Enabled = False
self.Logging = logging
self.LogFile = sys.stdout if log_file is None else log_file
self.Connections = TaskQueue(max_connections, capacity=max_queued)
self.RemovePrefix = remove_prefix
if enabled:
self.enableServer()
def __init__(self, config):
PyThread.__init__(self)
config = self.read_config(config)
uid = uuid.uuid4()
self.ID_bytes = uid.bytes[:self.ID_Length]
self.ID = self.ID_bytes.hex()
self.Seen = SeenMemory(10000) # data: (seen, sent_edge, sent_diagonal)
seed_nodes = config["seed_nodes"]
self.DownLink = DownLink(self, seed_nodes)
inx = self.DownLink.Index
if inx is not None:
seed_nodes = seed_nodes[inx + 1:] + seed_nodes[:inx + 1]
self.SeedNodes = seed_nodes
self.UpLink = UpLink(self, seed_nodes)
self.DiagonalLink = DiagonalLink(self, self.DownLink.Address)
self.Map = []
self.Initialized = False
self.Futures = {} # tid -> (future, exp_time)
self.Shutdown = False
self.DiagonalCheckInterval = 60.0
self.NextDiagonalCheck = time.time() + self.DiagonalCheckInterval
self.Debug = True
def __init__(self, master):
PyThread.__init__(self)
self.Master = master
self.Stop = False
def __init__(self, logger, queue_size=-1, debug=False, name=None):
import multiprocessing
PyThread.__init__(self, name=name, daemon=True)
self.Logger = logger
self.Queue = multiprocessing.Queue(queue_size)
self.Debug = debug
def __init__(self, ether, topics):
PyThread.__init__(self)
PubSubAgent.__init__(self, ether, topics)
def __init__(self, port, server, reader, logger):
PyThread.__init__(self)
self.Reader = reader
self.Logger = logger
self.Port = port
self.Server = server
def __init__(self, ws):
PyThread.__init__(self)
self.WS = ws
