def __init__(self, helper_candidate, send_request):
self.event = Event()
self.helper_candidate = helper_candidate
self.helper_candidate.set_timeout(self)
self.send_request = send_request
self.thread = CallFunctionThread()
self.thread.put(self.wait_and_see)
self.thread.start()
def set_receive(self):
logger.debug("Set receive on %s", self.group)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.bind(("", self.port))
mreq = struct.pack("!4sl", socket.inet_aton(self.group),
socket.INADDR_ANY)
self.sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq)
self.thread = CallFunctionThread(timeout=1.0)
self.thread.start()
self.event = Event()
self.thread.put(self._receive)
class IntroductionRequestTimeout(object):
'''
classdocs
'''
def __init__(self, helper_candidate, send_request):
self.event = Event()
self.helper_candidate = helper_candidate
self.helper_candidate.set_timeout(self)
self.send_request = send_request
self.thread = CallFunctionThread()
self.thread.put(self.wait_and_see)
self.thread.start()
def wait_and_see(self):
self.event.wait(TIMEOUT_INTRODUCTION_REQUEST)
if not self.helper_candidate.introduction_response_received():
self.send_request()
self.thread.put(self.wait_and_see)
else:
self.thread.stop()
@property
def candidate(self):
return self.helper_candidate
def stop(self):
self.event.set()
class IntroductionRequestTimeout(object):
'''
classdocs
'''
def __init__(self, helper_candidate, send_request):
self.event = Event()
self.helper_candidate = helper_candidate
self.helper_candidate.set_timeout(self)
self.send_request = send_request
self.thread = CallFunctionThread()
self.thread.put(self.wait_and_see)
self.thread.start()
def wait_and_see(self):
self.event.wait(TIMEOUT_INTRODUCTION_REQUEST)
if not self.helper_candidate.introduction_response_received():
self.send_request()
self.thread.put(self.wait_and_see)
else:
self.thread.stop()
@property
def candidate(self):
return self.helper_candidate
def stop(self):
self.event.set()
def __init__(self, connection, name=""):
self.conn = connection
self.name = name
# Receive from pipe
self.stop_receiving_event = Event()
self.is_alive_event = Event() # Wait until subclass tells you it is ready
self._receiver = Thread(target=self.wait_on_recv, name=name + "_receiver")
self._receiver.start()
# Start send thread
self.sender = CallFunctionThread(timeout=1.0, name=name + "_sender")
self.sender.start()
def __init__(self, callback, swift_path, directories=[], files=[], file_size=MAX_MESSAGE_SIZE, hidden=False, min_timestamp=None):
'''
@param callback: The function that will be called with a FileHashCarrier or SimpleFileCarrier object
@param swift_path: Path to swift executable
@param directory: The directory to search for files
@param files: The list of files to monitor
@param file_size: The decision variable for choosing callback object
@param hidden: List hidden downloads as well
@param min_timestamp: Oldest modification time to use for new files
'''
Thread.__init__(self, name="Filepusher")
self.setDaemon(True)
self._dirs = set()
for d in directories:
self.add_directory(d)
self._files = set()
self.add_files(files)
self._recent_files = []
self._callback = callback
self._file_size = file_size
self.swift_path = swift_path
self._hidden = hidden
self._min_timestamp = datetime.fromtimestamp(min_timestamp) if min_timestamp is not None else datetime.min
self._stop_event = Event()
self._thread_func = CallFunctionThread(timeout=1.0, name="Filepusher")
self._paused = False
def __init__(self, helper_candidate, send_request):
self.event = Event()
self.helper_candidate = helper_candidate
self.helper_candidate.set_timeout(self)
self.send_request = send_request
self.thread = CallFunctionThread()
self.thread.put(self.wait_and_see)
self.thread.start()
def set_receive(self):
logger.debug("Set receive on %s", self.group)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.bind(("", self.port))
mreq = struct.pack("!4sl", socket.inet_aton(self.group), socket.INADDR_ANY)
self.sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq)
self.thread = CallFunctionThread(timeout=1.0)
self.thread.start()
self.event = Event()
self.thread.put(self._receive)
def __init__(self,
callback,
swift_path,
directories=[],
files=[],
file_size=MAX_MESSAGE_SIZE,
hidden=False,
min_timestamp=None):
'''
@param callback: The function that will be called with a FileHashCarrier or SimpleFileCarrier object
@param swift_path: Path to swift executable
@param directory: The directory to search for files
@param files: The list of files to monitor
@param file_size: The decision variable for choosing callback object
@param hidden: List hidden downloads as well
@param min_timestamp: Oldest modification time to use for new files
'''
Thread.__init__(self, name="Filepusher")
self.setDaemon(True)
self._dirs = set()
for d in directories:
self.add_directory(d)
self._files = set()
self.add_files(files)
self._recent_files = []
self._callback = callback
self._file_size = file_size
self.swift_path = swift_path
self._hidden = hidden
self._min_timestamp = datetime.fromtimestamp(
min_timestamp) if min_timestamp is not None else datetime.min
self._stop_event = Event()
self._thread_func = CallFunctionThread(timeout=1.0, name="Filepusher")
self._paused = False
class MultiCast(object):
'''
This class will allow to send and receive UDP multicast
'''
def __init__(self, group, port, receive):
self.group = group
self.port = port
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.thread = None
self.event = None
if receive:
self.set_receive()
self._receiving = True
else:
self.set_send()
self._ready_to_send = True
def set_receive(self):
logger.debug("Set receive on %s", self.group)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.bind(("", self.port))
mreq = struct.pack("!4sl", socket.inet_aton(self.group),
socket.INADDR_ANY)
self.sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq)
self.thread = CallFunctionThread(timeout=1.0)
self.thread.start()
self.event = Event()
self.thread.put(self._receive)
def _receive(self):
logger.debug("Start receive on %s", self.sock.getsockname())
while not self.event.is_set():
# Select can be used for timeout (import select)
ret = select.select([self.sock], [], [], 0.1)
if ret[0]:
message = self.sock.recv(
4096) # Small power of 2 for optimal performance
logger.info("Received %s", message)
def set_send(self):
logger.debug("Set send")
self.sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL,
1) # Time to live, 1 for local network..
def send(self, message):
if self.ready_to_send:
logger.debug("Send %s to %s", message, self.group)
self.sock.sendto(message, (self.group, self.port))
@property
def receiving(self):
return self._receiving
@property
def ready_to_send(self):
return self._ready_to_send
def stop(self):
logger.debug("Stop")
# This is apparently not sufficient to stop the thread with daemon=False
if self.event:
self.event.set()
self.sock.close()
if self.thread:
self.thread.stop()
class MultiCast(object):
'''
This class will allow to send and receive UDP multicast
'''
def __init__(self, group, port, receive):
self.group = group
self.port = port
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.thread = None
self.event = None
if receive:
self.set_receive()
self._receiving = True
else:
self.set_send()
self._ready_to_send = True
def set_receive(self):
logger.debug("Set receive on %s", self.group)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.bind(("", self.port))
mreq = struct.pack("!4sl", socket.inet_aton(self.group), socket.INADDR_ANY)
self.sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq)
self.thread = CallFunctionThread(timeout=1.0)
self.thread.start()
self.event = Event()
self.thread.put(self._receive)
def _receive(self):
logger.debug("Start receive on %s", self.sock.getsockname())
while not self.event.is_set():
# Select can be used for timeout (import select)
ret = select.select([self.sock], [], [], 0.1)
if ret[0]:
message = self.sock.recv(4096) # Small power of 2 for optimal performance
logger.info("Received %s", message)
def set_send(self):
logger.debug("Set send")
self.sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 1) # Time to live, 1 for local network..
def send(self, message):
if self.ready_to_send:
logger.debug("Send %s to %s", message, self.group)
self.sock.sendto(message, (self.group, self.port))
@property
def receiving(self):
return self._receiving
@property
def ready_to_send(self):
return self._ready_to_send
def stop(self):
logger.debug("Stop")
# This is apparently not sufficient to stop the thread with daemon=False
if self.event:
self.event.set()
self.sock.close()
if self.thread:
self.thread.stop()
class FilePusher(Thread):
'''
FilePusher goes through a directory or a list of files to find either new files or updated files,
and does a callback with the list of these files. It distinguishes between files larger and smaller than _file_size.
In the former case the filename is send back, whereas in the latter case the contents of the file (string) is send back.
'''
def __init__(self,
callback,
swift_path,
directories=[],
files=[],
file_size=MAX_MESSAGE_SIZE,
hidden=False,
min_timestamp=None):
'''
@param callback: The function that will be called with a FileHashCarrier or SimpleFileCarrier object
@param swift_path: Path to swift executable
@param directory: The directory to search for files
@param files: The list of files to monitor
@param file_size: The decision variable for choosing callback object
@param hidden: List hidden downloads as well
@param min_timestamp: Oldest modification time to use for new files
'''
Thread.__init__(self, name="Filepusher")
self.setDaemon(True)
self._dirs = set()
for d in directories:
self.add_directory(d)
self._files = set()
self.add_files(files)
self._recent_files = []
self._callback = callback
self._file_size = file_size
self.swift_path = swift_path
self._hidden = hidden
self._min_timestamp = datetime.fromtimestamp(
min_timestamp) if min_timestamp is not None else datetime.min
self._stop_event = Event()
self._thread_func = CallFunctionThread(timeout=1.0, name="Filepusher")
self._paused = False
def add_directory(self, directory):
if directory and exists(directory) and isdir(directory):
self._dirs.add(
abspath(directory)) # Add only absolute paths, no ambiguities
def add_files(self, files):
for f in files:
if exists(f) and isfile(f):
self._files.add(f)
def run(self):
"""
Run until _stop_event is set.
Determine list of files that have are new or have changed since previous iteration.
Call _callback with each of these files.
"""
self._thread_func.start()
while not self._stop_event.is_set():
diff = self._list_files_to_send()
for absfilename in diff:
if datetime.fromtimestamp(
getmtime(absfilename)) < self._min_timestamp:
continue # Only go for files that are older than self._min_timestamp
logger.debug("New file to be sent: %s", absfilename)
if getsize(absfilename) > self._file_size:
self._thread_func.put(self.send_file_hash_message,
absfilename,
dirs=self._get_dir(absfilename),
queue_priority=getmtime(absfilename))
else:
with file(absfilename) as f:
s = f.read()
self._callback(
message=SmallFileCarrier(absfilename, s))
self._stop_event.wait(SLEEP_TIME)
def _get_dir(self, absfilename):
"""
Find the longest path in the available directories.
Return None if no found.
@param absfilename: The absolute path to the filename
"""
longest = "" # Longest directory path
for d in self._dirs:
if d in absfilename and len(longest) < len(d):
longest = d
if len(longest): # Not length 0
return absfilename[len(longest) + 1:-len(basename(absfilename))]
return None
def send_file_hash_message(self, absfilename, dirs=None):
roothash = get_hash(absfilename, self.swift_path)
size = getsize(absfilename)
modified = getmtime(absfilename)
logger.debug(
"Determined roothash %s, for %s, with dirs %s of size %d at time %f",
roothash, absfilename, dirs, size, modified)
self._callback(message=FileHashCarrier(absfilename, dirs, roothash,
size, modified, None))
def pause(self):
if not self._paused:
self._thread_func.pause()
self._paused = True
logger.info("Paused")
def unpause(self):
if self._paused:
self._thread_func.unpause()
self._paused = False
logger.info("Unpaused")
@property
def paused(self):
return self._paused
def stop(self):
"""
Stop thread
"""
self._stop_event.set()
self._thread_func.stop()
def _list_files_to_send(self):
"""
Compare all files in _directory and _files, combined in file_updates with the current list _recent_files,
which both hold tuples of filename and last modified time.
@return: the difference between _recent_files and the file_updates
"""
def recur(dir_):
# all_files should only contain absolute filename paths in dir_
all_files = [
join(dir_, f) for f in listdir(dir_)
if isfile(join(dir_, f)) and
# which do not end in any of the FILETYPES_NOT_TO_SEND or contain FILENAMES_NOT_TO_SEND
not (any(f.endswith(t) for t in FILETYPES_NOT_TO_SEND) or any(
f.find(n) >= 0 for n in FILENAMES_NOT_TO_SEND)) and
# which if not hidden, do no start with a dot
not (not self._hidden and f[0] == ".")
]
all_dir = [
join(dir_, d) for d in listdir(dir_)
if isdir(join(dir_, d)) and
# If not hidden, don't go into directories starting with a dot
not (not self._hidden and d[0] == ".")
]
for d in all_dir:
all_files.extend(recur(d))
return all_files
# Get all files in the directory and subdirectories
all_files = set([f for d in self._dirs for f in recur(d)])
all_files.update(self._files)
file_updates = [(f, getmtime(f)) for f in all_files
] # create tuple of file and last modified timestamp
# Each file in the directory should be send at least once
# If renewed they should be sent again
diff = [
_ft[0] for _ft in file_updates if _ft not in self._recent_files
]
self._recent_files = file_updates
return diff
class FilePusher(Thread):
'''
FilePusher goes through a directory or a list of files to find either new files or updated files,
and does a callback with the list of these files. It distinguishes between files larger and smaller than _file_size.
In the former case the filename is send back, whereas in the latter case the contents of the file (string) is send back.
'''
def __init__(self, callback, swift_path, directories=[], files=[], file_size=MAX_MESSAGE_SIZE, hidden=False, min_timestamp=None):
'''
@param callback: The function that will be called with a FileHashCarrier or SimpleFileCarrier object
@param swift_path: Path to swift executable
@param directory: The directory to search for files
@param files: The list of files to monitor
@param file_size: The decision variable for choosing callback object
@param hidden: List hidden downloads as well
@param min_timestamp: Oldest modification time to use for new files
'''
Thread.__init__(self, name="Filepusher")
self.setDaemon(True)
self._dirs = set()
for d in directories:
self.add_directory(d)
self._files = set()
self.add_files(files)
self._recent_files = []
self._callback = callback
self._file_size = file_size
self.swift_path = swift_path
self._hidden = hidden
self._min_timestamp = datetime.fromtimestamp(min_timestamp) if min_timestamp is not None else datetime.min
self._stop_event = Event()
self._thread_func = CallFunctionThread(timeout=1.0, name="Filepusher")
self._paused = False
def add_directory(self, directory):
if directory and exists(directory) and isdir(directory):
self._dirs.add(abspath(directory)) # Add only absolute paths, no ambiguities
def add_files(self, files):
for f in files:
if exists(f) and isfile(f):
self._files.add(f)
def run(self):
"""
Run until _stop_event is set.
Determine list of files that have are new or have changed since previous iteration.
Call _callback with each of these files.
"""
self._thread_func.start()
while not self._stop_event.is_set():
diff = self._list_files_to_send()
for absfilename in diff:
if datetime.fromtimestamp(getmtime(absfilename)) < self._min_timestamp:
continue # Only go for files that are older than self._min_timestamp
logger.debug("New file to be sent: %s", absfilename)
if getsize(absfilename) > self._file_size:
self._thread_func.put(self.send_file_hash_message, absfilename, dirs=self._get_dir(absfilename),
queue_priority=getmtime(absfilename))
else:
with file(absfilename) as f:
s = f.read()
self._callback(message=SmallFileCarrier(absfilename, s))
self._stop_event.wait(SLEEP_TIME)
def _get_dir(self, absfilename):
"""
Find the longest path in the available directories.
Return None if no found.
@param absfilename: The absolute path to the filename
"""
longest = "" # Longest directory path
for d in self._dirs:
if d in absfilename and len(longest) < len(d):
longest = d
if len(longest): # Not length 0
return absfilename[len(longest) + 1:-len(basename(absfilename))]
return None
def send_file_hash_message(self, absfilename, dirs=None):
roothash = get_hash(absfilename, self.swift_path)
size = getsize(absfilename)
modified = getmtime(absfilename)
logger.debug("Determined roothash %s, for %s, with dirs %s of size %d at time %f",
roothash, absfilename, dirs, size, modified)
self._callback(message=FileHashCarrier(absfilename, dirs, roothash, size, modified, None))
def pause(self):
if not self._paused:
self._thread_func.pause()
self._paused = True
logger.info("Paused")
def unpause(self):
if self._paused:
self._thread_func.unpause()
self._paused = False
logger.info("Unpaused")
@property
def paused(self):
return self._paused
def stop(self):
"""
Stop thread
"""
self._stop_event.set()
self._thread_func.stop()
def _list_files_to_send(self):
"""
Compare all files in _directory and _files, combined in file_updates with the current list _recent_files,
which both hold tuples of filename and last modified time.
@return: the difference between _recent_files and the file_updates
"""
def recur(dir_):
# all_files should only contain absolute filename paths in dir_
all_files = [ join(dir_,f) for f in listdir(dir_) if isfile(join(dir_,f)) and
# which do not end in any of the FILETYPES_NOT_TO_SEND or contain FILENAMES_NOT_TO_SEND
not (any(f.endswith(t) for t in FILETYPES_NOT_TO_SEND)
or any(f.find(n) >= 0 for n in FILENAMES_NOT_TO_SEND)) and
# which if not hidden, do no start with a dot
not (not self._hidden and f[0] == ".") ]
all_dir = [join(dir_, d) for d in listdir(dir_) if isdir(join(dir_, d)) and
# If not hidden, don't go into directories starting with a dot
not (not self._hidden and d[0] == ".") ]
for d in all_dir:
all_files.extend(recur(d))
return all_files
# Get all files in the directory and subdirectories
all_files = set([f for d in self._dirs for f in recur(d)])
all_files.update(self._files)
file_updates = [ (f, getmtime(f)) for f in all_files] # create tuple of file and last modified timestamp
# Each file in the directory should be send at least once
# If renewed they should be sent again
diff = [_ft[0] for _ft in file_updates if _ft not in self._recent_files]
self._recent_files = file_updates
return diff
def setUp(self):
self._thread = CallFunctionThread()
self._thread.start()
class RunnerTest(unittest.TestCase):
def setUp(self):
self._thread = CallFunctionThread()
self._thread.start()
def tearDown(self):
self._thread.stop()
def test_function(self):
res = [1]
res[0] = False;
def callback():
res[0] = True
self._thread.put(callback)
event = Event()
event.wait(SMALL_TASK_TIMEOUT)
self.assertTrue(res[0])
def test_function_with_args(self):
res = [1]
res[0] = False;
def callback(b, b2):
res[0] = b and b2
self._thread.put(callback,(True,),{"b2":True})
event = Event()
event.wait(SMALL_TASK_TIMEOUT)
self.assertTrue(res[0])
def test_stop_after_tasks_are_done(self):
s = 0.5
def sleep():
time.sleep(s)
self._thread.put(sleep)
t = time.time()
self._thread.stop(wait_for_tasks=True, timeout=s * 2)
d = time.time() - t
self.assertTrue(self._thread.empty())
self.assertGreater(d, s)
self.assertLess(d, 2 * s)
class PipeHandler(object):
"""
This PipeHandler handles one end of a Pipe connection.
Both receiving and and sending are done in separate threads.
This will not start until is_alive_event has been set.
The Message Key Map allows child classes to define callback functions for specific Message Keys,
which they can also define.
When the other end of the connection is gone, _connection_process_gone() will be called,
which therefore needs to be implemented by all subclasses
"""
# This dictionary needs to be overwritten for the messages to be handled
# It should hold a MESSAGE_KEY_* as key and a function as value
MESSAGE_KEY_MAP = {}
def __init__(self, connection, name=""):
self.conn = connection
self.name = name
# Receive from pipe
self.stop_receiving_event = Event()
self.is_alive_event = Event() # Wait until subclass tells you it is ready
self._receiver = Thread(target=self.wait_on_recv, name=name + "_receiver")
self._receiver.start()
# Start send thread
self.sender = CallFunctionThread(timeout=1.0, name=name + "_sender")
self.sender.start()
def close_connection(self):
self.stop_receiving_event.set()
self.sender.stop(wait_for_tasks=True, timeout=1.0) # Wait at most timeout till queue is empty
self.conn.close()
logger.debug("Connection closed for %s", self.name)
def wait_on_recv(self):
"""
Listen to pipe for incoming messages, which are dispatched to handle_message
"""
while not self.stop_receiving_event.is_set():
self.is_alive_event.wait()
message = None
try:
message = self.conn.recv() # Blocking
except EOFError: # Other end is dead
logger.exception("Connection with process is gone for %s", self.name)
self._connection_process_gone() # Should be implemented!!
except:
logger.exception("Could not receive message over pipe for %s", self.name)
self.handle_message(message)
def send_message(self, key, *args, **kwargs):
"""
Send message via pipe to parent process
@param key: MESSAGE_KEY
"""
def send():
self.is_alive_event.wait()
try:
self.conn.send((key, args, kwargs))
except ValueError:
logger.exception("%s failed to send %s %s %s", self.name, key, args, kwargs)
self.sender.put(send)
def handle_message(self, message):
"""
Handles an incoming message. Relies on MESSAGE_KEY_MAP to hold the specified key.
If the key is available, an attempt to call the function it points to is made,
with the arguments provided.
@param message: (KEY, ARGS, KWARGS)
"""
if not message:
return
try:
func = self.MESSAGE_KEY_MAP[message[0]]
func(*message[1], **message[2])
except KeyError:
logger.exception("%s failed to dispatch incoming message %d %s %s", self.name, message[0], message[1], message[2])
def _connection_process_gone(self):
raise NotImplementedError()