def __init__(self, helper_candidate, send_request): self.event = Event() self.helper_candidate = helper_candidate self.helper_candidate.set_timeout(self) self.send_request = send_request self.thread = CallFunctionThread() self.thread.put(self.wait_and_see) self.thread.start()
def set_receive(self): logger.debug("Set receive on %s", self.group) self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.sock.bind(("", self.port)) mreq = struct.pack("!4sl", socket.inet_aton(self.group), socket.INADDR_ANY) self.sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) self.thread = CallFunctionThread(timeout=1.0) self.thread.start() self.event = Event() self.thread.put(self._receive)
class IntroductionRequestTimeout(object): ''' classdocs ''' def __init__(self, helper_candidate, send_request): self.event = Event() self.helper_candidate = helper_candidate self.helper_candidate.set_timeout(self) self.send_request = send_request self.thread = CallFunctionThread() self.thread.put(self.wait_and_see) self.thread.start() def wait_and_see(self): self.event.wait(TIMEOUT_INTRODUCTION_REQUEST) if not self.helper_candidate.introduction_response_received(): self.send_request() self.thread.put(self.wait_and_see) else: self.thread.stop() @property def candidate(self): return self.helper_candidate def stop(self): self.event.set()
def __init__(self, connection, name=""): self.conn = connection self.name = name # Receive from pipe self.stop_receiving_event = Event() self.is_alive_event = Event() # Wait until subclass tells you it is ready self._receiver = Thread(target=self.wait_on_recv, name=name + "_receiver") self._receiver.start() # Start send thread self.sender = CallFunctionThread(timeout=1.0, name=name + "_sender") self.sender.start()
def __init__(self, callback, swift_path, directories=[], files=[], file_size=MAX_MESSAGE_SIZE, hidden=False, min_timestamp=None): ''' @param callback: The function that will be called with a FileHashCarrier or SimpleFileCarrier object @param swift_path: Path to swift executable @param directory: The directory to search for files @param files: The list of files to monitor @param file_size: The decision variable for choosing callback object @param hidden: List hidden downloads as well @param min_timestamp: Oldest modification time to use for new files ''' Thread.__init__(self, name="Filepusher") self.setDaemon(True) self._dirs = set() for d in directories: self.add_directory(d) self._files = set() self.add_files(files) self._recent_files = [] self._callback = callback self._file_size = file_size self.swift_path = swift_path self._hidden = hidden self._min_timestamp = datetime.fromtimestamp(min_timestamp) if min_timestamp is not None else datetime.min self._stop_event = Event() self._thread_func = CallFunctionThread(timeout=1.0, name="Filepusher") self._paused = False
def __init__(self, callback, swift_path, directories=[], files=[], file_size=MAX_MESSAGE_SIZE, hidden=False, min_timestamp=None): ''' @param callback: The function that will be called with a FileHashCarrier or SimpleFileCarrier object @param swift_path: Path to swift executable @param directory: The directory to search for files @param files: The list of files to monitor @param file_size: The decision variable for choosing callback object @param hidden: List hidden downloads as well @param min_timestamp: Oldest modification time to use for new files ''' Thread.__init__(self, name="Filepusher") self.setDaemon(True) self._dirs = set() for d in directories: self.add_directory(d) self._files = set() self.add_files(files) self._recent_files = [] self._callback = callback self._file_size = file_size self.swift_path = swift_path self._hidden = hidden self._min_timestamp = datetime.fromtimestamp( min_timestamp) if min_timestamp is not None else datetime.min self._stop_event = Event() self._thread_func = CallFunctionThread(timeout=1.0, name="Filepusher") self._paused = False
class MultiCast(object): ''' This class will allow to send and receive UDP multicast ''' def __init__(self, group, port, receive): self.group = group self.port = port self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.thread = None self.event = None if receive: self.set_receive() self._receiving = True else: self.set_send() self._ready_to_send = True def set_receive(self): logger.debug("Set receive on %s", self.group) self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.sock.bind(("", self.port)) mreq = struct.pack("!4sl", socket.inet_aton(self.group), socket.INADDR_ANY) self.sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) self.thread = CallFunctionThread(timeout=1.0) self.thread.start() self.event = Event() self.thread.put(self._receive) def _receive(self): logger.debug("Start receive on %s", self.sock.getsockname()) while not self.event.is_set(): # Select can be used for timeout (import select) ret = select.select([self.sock], [], [], 0.1) if ret[0]: message = self.sock.recv( 4096) # Small power of 2 for optimal performance logger.info("Received %s", message) def set_send(self): logger.debug("Set send") self.sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 1) # Time to live, 1 for local network.. def send(self, message): if self.ready_to_send: logger.debug("Send %s to %s", message, self.group) self.sock.sendto(message, (self.group, self.port)) @property def receiving(self): return self._receiving @property def ready_to_send(self): return self._ready_to_send def stop(self): logger.debug("Stop") # This is apparently not sufficient to stop the thread with daemon=False if self.event: self.event.set() self.sock.close() if self.thread: self.thread.stop()
class MultiCast(object): ''' This class will allow to send and receive UDP multicast ''' def __init__(self, group, port, receive): self.group = group self.port = port self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.thread = None self.event = None if receive: self.set_receive() self._receiving = True else: self.set_send() self._ready_to_send = True def set_receive(self): logger.debug("Set receive on %s", self.group) self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.sock.bind(("", self.port)) mreq = struct.pack("!4sl", socket.inet_aton(self.group), socket.INADDR_ANY) self.sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) self.thread = CallFunctionThread(timeout=1.0) self.thread.start() self.event = Event() self.thread.put(self._receive) def _receive(self): logger.debug("Start receive on %s", self.sock.getsockname()) while not self.event.is_set(): # Select can be used for timeout (import select) ret = select.select([self.sock], [], [], 0.1) if ret[0]: message = self.sock.recv(4096) # Small power of 2 for optimal performance logger.info("Received %s", message) def set_send(self): logger.debug("Set send") self.sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 1) # Time to live, 1 for local network.. def send(self, message): if self.ready_to_send: logger.debug("Send %s to %s", message, self.group) self.sock.sendto(message, (self.group, self.port)) @property def receiving(self): return self._receiving @property def ready_to_send(self): return self._ready_to_send def stop(self): logger.debug("Stop") # This is apparently not sufficient to stop the thread with daemon=False if self.event: self.event.set() self.sock.close() if self.thread: self.thread.stop()
class FilePusher(Thread): ''' FilePusher goes through a directory or a list of files to find either new files or updated files, and does a callback with the list of these files. It distinguishes between files larger and smaller than _file_size. In the former case the filename is send back, whereas in the latter case the contents of the file (string) is send back. ''' def __init__(self, callback, swift_path, directories=[], files=[], file_size=MAX_MESSAGE_SIZE, hidden=False, min_timestamp=None): ''' @param callback: The function that will be called with a FileHashCarrier or SimpleFileCarrier object @param swift_path: Path to swift executable @param directory: The directory to search for files @param files: The list of files to monitor @param file_size: The decision variable for choosing callback object @param hidden: List hidden downloads as well @param min_timestamp: Oldest modification time to use for new files ''' Thread.__init__(self, name="Filepusher") self.setDaemon(True) self._dirs = set() for d in directories: self.add_directory(d) self._files = set() self.add_files(files) self._recent_files = [] self._callback = callback self._file_size = file_size self.swift_path = swift_path self._hidden = hidden self._min_timestamp = datetime.fromtimestamp( min_timestamp) if min_timestamp is not None else datetime.min self._stop_event = Event() self._thread_func = CallFunctionThread(timeout=1.0, name="Filepusher") self._paused = False def add_directory(self, directory): if directory and exists(directory) and isdir(directory): self._dirs.add( abspath(directory)) # Add only absolute paths, no ambiguities def add_files(self, files): for f in files: if exists(f) and isfile(f): self._files.add(f) def run(self): """ Run until _stop_event is set. Determine list of files that have are new or have changed since previous iteration. Call _callback with each of these files. """ self._thread_func.start() while not self._stop_event.is_set(): diff = self._list_files_to_send() for absfilename in diff: if datetime.fromtimestamp( getmtime(absfilename)) < self._min_timestamp: continue # Only go for files that are older than self._min_timestamp logger.debug("New file to be sent: %s", absfilename) if getsize(absfilename) > self._file_size: self._thread_func.put(self.send_file_hash_message, absfilename, dirs=self._get_dir(absfilename), queue_priority=getmtime(absfilename)) else: with file(absfilename) as f: s = f.read() self._callback( message=SmallFileCarrier(absfilename, s)) self._stop_event.wait(SLEEP_TIME) def _get_dir(self, absfilename): """ Find the longest path in the available directories. Return None if no found. @param absfilename: The absolute path to the filename """ longest = "" # Longest directory path for d in self._dirs: if d in absfilename and len(longest) < len(d): longest = d if len(longest): # Not length 0 return absfilename[len(longest) + 1:-len(basename(absfilename))] return None def send_file_hash_message(self, absfilename, dirs=None): roothash = get_hash(absfilename, self.swift_path) size = getsize(absfilename) modified = getmtime(absfilename) logger.debug( "Determined roothash %s, for %s, with dirs %s of size %d at time %f", roothash, absfilename, dirs, size, modified) self._callback(message=FileHashCarrier(absfilename, dirs, roothash, size, modified, None)) def pause(self): if not self._paused: self._thread_func.pause() self._paused = True logger.info("Paused") def unpause(self): if self._paused: self._thread_func.unpause() self._paused = False logger.info("Unpaused") @property def paused(self): return self._paused def stop(self): """ Stop thread """ self._stop_event.set() self._thread_func.stop() def _list_files_to_send(self): """ Compare all files in _directory and _files, combined in file_updates with the current list _recent_files, which both hold tuples of filename and last modified time. @return: the difference between _recent_files and the file_updates """ def recur(dir_): # all_files should only contain absolute filename paths in dir_ all_files = [ join(dir_, f) for f in listdir(dir_) if isfile(join(dir_, f)) and # which do not end in any of the FILETYPES_NOT_TO_SEND or contain FILENAMES_NOT_TO_SEND not (any(f.endswith(t) for t in FILETYPES_NOT_TO_SEND) or any( f.find(n) >= 0 for n in FILENAMES_NOT_TO_SEND)) and # which if not hidden, do no start with a dot not (not self._hidden and f[0] == ".") ] all_dir = [ join(dir_, d) for d in listdir(dir_) if isdir(join(dir_, d)) and # If not hidden, don't go into directories starting with a dot not (not self._hidden and d[0] == ".") ] for d in all_dir: all_files.extend(recur(d)) return all_files # Get all files in the directory and subdirectories all_files = set([f for d in self._dirs for f in recur(d)]) all_files.update(self._files) file_updates = [(f, getmtime(f)) for f in all_files ] # create tuple of file and last modified timestamp # Each file in the directory should be send at least once # If renewed they should be sent again diff = [ _ft[0] for _ft in file_updates if _ft not in self._recent_files ] self._recent_files = file_updates return diff
class FilePusher(Thread): ''' FilePusher goes through a directory or a list of files to find either new files or updated files, and does a callback with the list of these files. It distinguishes between files larger and smaller than _file_size. In the former case the filename is send back, whereas in the latter case the contents of the file (string) is send back. ''' def __init__(self, callback, swift_path, directories=[], files=[], file_size=MAX_MESSAGE_SIZE, hidden=False, min_timestamp=None): ''' @param callback: The function that will be called with a FileHashCarrier or SimpleFileCarrier object @param swift_path: Path to swift executable @param directory: The directory to search for files @param files: The list of files to monitor @param file_size: The decision variable for choosing callback object @param hidden: List hidden downloads as well @param min_timestamp: Oldest modification time to use for new files ''' Thread.__init__(self, name="Filepusher") self.setDaemon(True) self._dirs = set() for d in directories: self.add_directory(d) self._files = set() self.add_files(files) self._recent_files = [] self._callback = callback self._file_size = file_size self.swift_path = swift_path self._hidden = hidden self._min_timestamp = datetime.fromtimestamp(min_timestamp) if min_timestamp is not None else datetime.min self._stop_event = Event() self._thread_func = CallFunctionThread(timeout=1.0, name="Filepusher") self._paused = False def add_directory(self, directory): if directory and exists(directory) and isdir(directory): self._dirs.add(abspath(directory)) # Add only absolute paths, no ambiguities def add_files(self, files): for f in files: if exists(f) and isfile(f): self._files.add(f) def run(self): """ Run until _stop_event is set. Determine list of files that have are new or have changed since previous iteration. Call _callback with each of these files. """ self._thread_func.start() while not self._stop_event.is_set(): diff = self._list_files_to_send() for absfilename in diff: if datetime.fromtimestamp(getmtime(absfilename)) < self._min_timestamp: continue # Only go for files that are older than self._min_timestamp logger.debug("New file to be sent: %s", absfilename) if getsize(absfilename) > self._file_size: self._thread_func.put(self.send_file_hash_message, absfilename, dirs=self._get_dir(absfilename), queue_priority=getmtime(absfilename)) else: with file(absfilename) as f: s = f.read() self._callback(message=SmallFileCarrier(absfilename, s)) self._stop_event.wait(SLEEP_TIME) def _get_dir(self, absfilename): """ Find the longest path in the available directories. Return None if no found. @param absfilename: The absolute path to the filename """ longest = "" # Longest directory path for d in self._dirs: if d in absfilename and len(longest) < len(d): longest = d if len(longest): # Not length 0 return absfilename[len(longest) + 1:-len(basename(absfilename))] return None def send_file_hash_message(self, absfilename, dirs=None): roothash = get_hash(absfilename, self.swift_path) size = getsize(absfilename) modified = getmtime(absfilename) logger.debug("Determined roothash %s, for %s, with dirs %s of size %d at time %f", roothash, absfilename, dirs, size, modified) self._callback(message=FileHashCarrier(absfilename, dirs, roothash, size, modified, None)) def pause(self): if not self._paused: self._thread_func.pause() self._paused = True logger.info("Paused") def unpause(self): if self._paused: self._thread_func.unpause() self._paused = False logger.info("Unpaused") @property def paused(self): return self._paused def stop(self): """ Stop thread """ self._stop_event.set() self._thread_func.stop() def _list_files_to_send(self): """ Compare all files in _directory and _files, combined in file_updates with the current list _recent_files, which both hold tuples of filename and last modified time. @return: the difference between _recent_files and the file_updates """ def recur(dir_): # all_files should only contain absolute filename paths in dir_ all_files = [ join(dir_,f) for f in listdir(dir_) if isfile(join(dir_,f)) and # which do not end in any of the FILETYPES_NOT_TO_SEND or contain FILENAMES_NOT_TO_SEND not (any(f.endswith(t) for t in FILETYPES_NOT_TO_SEND) or any(f.find(n) >= 0 for n in FILENAMES_NOT_TO_SEND)) and # which if not hidden, do no start with a dot not (not self._hidden and f[0] == ".") ] all_dir = [join(dir_, d) for d in listdir(dir_) if isdir(join(dir_, d)) and # If not hidden, don't go into directories starting with a dot not (not self._hidden and d[0] == ".") ] for d in all_dir: all_files.extend(recur(d)) return all_files # Get all files in the directory and subdirectories all_files = set([f for d in self._dirs for f in recur(d)]) all_files.update(self._files) file_updates = [ (f, getmtime(f)) for f in all_files] # create tuple of file and last modified timestamp # Each file in the directory should be send at least once # If renewed they should be sent again diff = [_ft[0] for _ft in file_updates if _ft not in self._recent_files] self._recent_files = file_updates return diff
def setUp(self): self._thread = CallFunctionThread() self._thread.start()
class RunnerTest(unittest.TestCase): def setUp(self): self._thread = CallFunctionThread() self._thread.start() def tearDown(self): self._thread.stop() def test_function(self): res = [1] res[0] = False; def callback(): res[0] = True self._thread.put(callback) event = Event() event.wait(SMALL_TASK_TIMEOUT) self.assertTrue(res[0]) def test_function_with_args(self): res = [1] res[0] = False; def callback(b, b2): res[0] = b and b2 self._thread.put(callback,(True,),{"b2":True}) event = Event() event.wait(SMALL_TASK_TIMEOUT) self.assertTrue(res[0]) def test_stop_after_tasks_are_done(self): s = 0.5 def sleep(): time.sleep(s) self._thread.put(sleep) t = time.time() self._thread.stop(wait_for_tasks=True, timeout=s * 2) d = time.time() - t self.assertTrue(self._thread.empty()) self.assertGreater(d, s) self.assertLess(d, 2 * s)
class PipeHandler(object): """ This PipeHandler handles one end of a Pipe connection. Both receiving and and sending are done in separate threads. This will not start until is_alive_event has been set. The Message Key Map allows child classes to define callback functions for specific Message Keys, which they can also define. When the other end of the connection is gone, _connection_process_gone() will be called, which therefore needs to be implemented by all subclasses """ # This dictionary needs to be overwritten for the messages to be handled # It should hold a MESSAGE_KEY_* as key and a function as value MESSAGE_KEY_MAP = {} def __init__(self, connection, name=""): self.conn = connection self.name = name # Receive from pipe self.stop_receiving_event = Event() self.is_alive_event = Event() # Wait until subclass tells you it is ready self._receiver = Thread(target=self.wait_on_recv, name=name + "_receiver") self._receiver.start() # Start send thread self.sender = CallFunctionThread(timeout=1.0, name=name + "_sender") self.sender.start() def close_connection(self): self.stop_receiving_event.set() self.sender.stop(wait_for_tasks=True, timeout=1.0) # Wait at most timeout till queue is empty self.conn.close() logger.debug("Connection closed for %s", self.name) def wait_on_recv(self): """ Listen to pipe for incoming messages, which are dispatched to handle_message """ while not self.stop_receiving_event.is_set(): self.is_alive_event.wait() message = None try: message = self.conn.recv() # Blocking except EOFError: # Other end is dead logger.exception("Connection with process is gone for %s", self.name) self._connection_process_gone() # Should be implemented!! except: logger.exception("Could not receive message over pipe for %s", self.name) self.handle_message(message) def send_message(self, key, *args, **kwargs): """ Send message via pipe to parent process @param key: MESSAGE_KEY """ def send(): self.is_alive_event.wait() try: self.conn.send((key, args, kwargs)) except ValueError: logger.exception("%s failed to send %s %s %s", self.name, key, args, kwargs) self.sender.put(send) def handle_message(self, message): """ Handles an incoming message. Relies on MESSAGE_KEY_MAP to hold the specified key. If the key is available, an attempt to call the function it points to is made, with the arguments provided. @param message: (KEY, ARGS, KWARGS) """ if not message: return try: func = self.MESSAGE_KEY_MAP[message[0]] func(*message[1], **message[2]) except KeyError: logger.exception("%s failed to dispatch incoming message %d %s %s", self.name, message[0], message[1], message[2]) def _connection_process_gone(self): raise NotImplementedError()