def __setupInotifyMonitor(self, path, inotify_types, glob_pattern):
'''
Initializes an inotify monitor process on ``path`` and yields the
defined ``inotify_types`` generated on the paths matching the
``glob_pattern``.
:param str path: The path to monitor
:param list inotify_types: A list of inotify types to monitor
:param str glob_pattern: Paths need to match in order to be returned.
:return: generator
'''
file_exists = True
i = Inotify()
i.add_watch(path)
while file_exists and self.loop():
for event in i.event_gen(yield_nones=False):
if event is not None:
for inotify_type in event[1]:
if inotify_type in inotify_types or inotify_types == []:
abs_path = os.path.abspath("%s/%s" % (event[2], event[3]))
if fnmatch.fnmatch(abs_path, glob_pattern):
yield abs_path.rstrip('/'), inotify_type
if inotify_type == "IN_DELETE_SELF":
file_exists = False
break
else:
sleep(1)
def main():
from inotify.adapters import Inotify
i = Inotify()
i.add_watch('.')
watched_files = sys.argv[1:]
# watched_file = (sys.argv + [None])[1]
# import os
# common = os.path.commonpath([os.getcwd(), watched_file])
# watched_file = watched_file[1+len(common):]
# print(common, watched_file)
with kernel() as k:
for f in watched_files:
try:
processed_to_stdout(k.process(open(f, 'r').read()))
except FileNotFoundError:
pass
for event in i.event_gen(yield_nones=False):
(_, event_type, _, filename) = event
# print(event_type, filename)
if event_type == ['IN_CLOSE_WRITE'] and filename in watched_files:
processed_to_stdout(k.process(open(filename, 'r').read()))
if event_type == ['IN_CLOSE_WRITE'] and filename == '.requests':
try:
request, body = open(filename, 'r').read().split('\n', 1)
handle_request(k, body, *request.split(' '))
except:
print('Invalid request:', str(open(filename, 'r').read()).split('\n')[0])
import traceback
print(traceback.format_exc())
continue
def _get_events(self):
inotify_watcher = Inotify()
inotify_watcher.add_watch(self.partialsPath, mask=self.MASK)
for event in inotify_watcher.event_gen():
if not event:
continue
yield event
def __init__(self, extra_files=None, callback=None):
super().__init__()
self.setDaemon(True)
self._callback = callback
self._dirs = set()
self._watcher = Inotify()
for extra_file in extra_files:
self.add_extra_file(extra_file)
def __init__(self, path, mask=IN_ALL_EVENTS, block_duration_s=1):
self.__root_path = path
# No matter what we actually received as the mask, make sure we have
# the minimum that we require to curate our list of watches.
self.__mask = mask | IN_ISDIR | IN_CREATE | IN_DELETE
self.__i = Inotify(block_duration_s=block_duration_s)
self.__load_tree(path)
def __init__(self):
self._inotify = Inotify()
self._inotify_mask = IN_ALL_EVENTS & (
~IN_ACCESS & ~IN_OPEN & ~IN_CLOSE_NOWRITE & ~IN_CLOSE_WRITE)
self._dir_queue = multiprocessing.Queue()
self._sync_queue = multiprocessing.Queue()
self._watched_dirs = {} # {lib_name: set(dirs)}
super().__init__(target=self._main,
args=(self._dir_queue, self._sync_queue))
def watch(self):
print(
"--- Orchestator mode engaged. Watching for changes in SFC configuration at "
+ str(self._sfc_config_directory))
i = Inotify()
i.add_watch(str(self._sfc_config_directory))
for event in i.event_gen(yield_nones=False):
(_, type_name, path, filename) = event
# if changes have been written to the file
if 'IN_CLOSE_WRITE' in type_name and self._sfc_config_filename == filename:
print("Write-event to {}{} registered {}, reevaluating...\n".
format(path, filename, type_name))
new_sfcs = self.read_json_config(file=self._sfc_config_path)
print('New SFCs: {}\n'.format(
simplejson.dumps(new_sfcs, indent=4)))
deleted_sfcs = [x for x in self.sfcs if x not in new_sfcs]
for sfc_id in deleted_sfcs:
print('Delete SFC {}:'.format(sfc_id))
if '/' in str(
self.sfcs[sfc_id]['trafficType']['ipAddress']):
split = str(self.sfcs[sfc_id]['trafficType']
['ipAddress']).split('/')
ip_address = split[0]
prefix_length = split[1]
else:
print(
'Traffic destination IP given without prefix! Aborting...'
)
exit(1)
# Traffic from customer
self._ingress_switch_controller.delete_mpls_sr_rule(
self._label_stacks[sfc_id], 'src', ip_address,
prefix_length)
label_stacks_backwards = self.generate_label_stacks_backwards(
)
# Traffic to customer
self._ingress_switch_controller.delete_mpls_sr_rule(
label_stacks_backwards[sfc_id], 'dst', ip_address,
prefix_length)
self.apply_sfc_policies(new_sfcs)
print(
"----- Orchestator mode engaged. Watching for changes in SFC configuration at {} ----- "
.format(self._sfc_config_directory))
def tail(self):
watch_mask = (constants.IN_ALL_EVENTS ^ constants.IN_ACCESS ^
constants.IN_OPEN ^ constants.IN_CLOSE_NOWRITE)
i = Inotify()
i.add_watch(self.filename, mask=watch_mask)
while True:
# Consume as much as possible.
yield from self.resume()
# Now wait for a change that we can react to
ev = self.wait_actionable(i)
if ev == 'append':
continue
if ev == 'swap':
# Need to reach around since file-deletion removes C watches,
# but not the python one...
try:
i.remove_watch(self.filename)
except Exception:
pass
i.add_watch(self.filename, mask=watch_mask)
self.pos = 1
continue
class InotifyReloader(threading.Thread):
event_mask = (inotify.constants.IN_CREATE | inotify.constants.IN_DELETE
| inotify.constants.IN_DELETE_SELF
| inotify.constants.IN_MODIFY
| inotify.constants.IN_MOVE_SELF
| inotify.constants.IN_MOVED_FROM
| inotify.constants.IN_MOVED_TO)
def __init__(self, extra_files=None, callback=None):
super().__init__()
self.setDaemon(True)
self._callback = callback
self._dirs = set()
self._watcher = Inotify()
for extra_file in extra_files:
self.add_extra_file(extra_file)
def add_extra_file(self, filename):
dirname = os.path.dirname(filename)
if dirname in self._dirs:
return
self._watcher.add_watch(dirname, mask=self.event_mask)
self._dirs.add(dirname)
def get_dirs(self):
fnames = [
os.path.dirname(COMPILED_EXT_RE.sub('py', module.__file__))
for module in tuple(sys.modules.values())
if getattr(module, '__file__', None)
]
return set(fnames)
def run(self):
self._dirs = self.get_dirs()
for dirname in self._dirs:
self._watcher.add_watch(dirname, mask=self.event_mask)
for event in self._watcher.event_gen():
if event is None:
continue
filename = event[3]
self._callback(filename)
def run(self):
watch_mask = constants.IN_ALL_EVENTS
print("Starting FsMonitor")
i = Inotify()
i.add_watch('gossip_store', mask=watch_mask)
for event in i.event_gen(yield_nones=False):
(e, type_names, path, filename) = event
if e.mask & constants.IN_DELETE_SELF:
i.remove_watch('gossip_store')
i.add_watch('gossip_store', mask=watch_mask)
def __init__(self, sets: Sets) -> None:
self.se = sets
self.addons = ["venv"]
unknown_addons = [a for a in self.se.addons if a not in self.addons]
if unknown_addons:
raise RuntimeError(f"Unknown addons {unknown_addons}")
self.inotify = Inotify()
self._environ_before: Dict[str, Any] = os.environ.copy()
self.shell = Shell.create()
self.env_dirs = self._get_env_dirs()
def run_inotify(watch_dir, move_to_dir):
logging.info("Starting move_and_process, watch: %s, move_to: %s",
watch_dir, move_to_dir)
i = Inotify(block_duration_s=300)
i.add_watch(watch_dir, constants.IN_MOVED_TO)
for evt in i.event_gen():
if evt is None:
# Call every block_duration_s seconds when there is nothing else to do
run_periodic(watch_dir, move_to_dir)
continue
(_header, type_names, _path, fn) = evt
if 'IN_ISDIR' not in type_names or not fn.isdigit():
logging.info('Skipped %s' % fn)
continue
logging.info('Found %s' % fn)
handle_file(watch_dir, move_to_dir, int(fn))
class InotifyReloader(threading.Thread):
event_mask = (inotify.constants.IN_CREATE | inotify.constants.IN_DELETE
| inotify.constants.IN_DELETE_SELF | inotify.constants.IN_MODIFY
| inotify.constants.IN_MOVE_SELF | inotify.constants.IN_MOVED_FROM
| inotify.constants.IN_MOVED_TO)
def __init__(self, extra_files=None, callback=None):
super().__init__()
self.setDaemon(True)
self._callback = callback
self._dirs = set()
self._watcher = Inotify()
for extra_file in extra_files:
self.add_extra_file(extra_file)
def add_extra_file(self, filename):
dirname = os.path.dirname(filename)
if dirname in self._dirs:
return
self._watcher.add_watch(dirname, mask=self.event_mask)
self._dirs.add(dirname)
def get_dirs(self):
fnames = [
os.path.dirname(COMPILED_EXT_RE.sub('py', module.__file__))
for module in tuple(sys.modules.values())
if getattr(module, '__file__', None)
]
return set(fnames)
def run(self):
self._dirs = self.get_dirs()
for dirname in self._dirs:
self._watcher.add_watch(dirname, mask=self.event_mask)
for event in self._watcher.event_gen():
if event is None:
continue
filename = event[3]
self._callback(filename)
def __init__(self, sets: Sets) -> None:
self.se = sets
self.addons = ["venv"]
unknown_addons = [a for a in self.se.addons if a not in self.addons]
if unknown_addons:
raise EnvoError(f"Unknown addons {unknown_addons}")
self.inotify = Inotify()
self.env_dirs = self._get_env_dirs()
self.quit: bool = False
self.environ_before = os.environ.copy() # type: ignore
self._set_context_thread: Optional[Thread] = None
def __init__(self, extra_files=None, callback=None):
super().__init__()
self.setDaemon(True)
self._callback = callback
self._dirs = set()
self._watcher = Inotify()
for extra_file in extra_files:
self.add_extra_file(extra_file)
def run(self):
logging.info("Running notify thread. First scan files from last run")
for f in os.listdir(self.folder):
if os.path.isfile(os.path.join(self.folder,f)):
logging.info("File %s noticed unprocessed notify thread for project %s in %s"%(f,
self.project_id,self.folder))
processing.Processing.process_incoming_file(self.folder,f,self.project_id)
logging.info("Scan complete, start watching events")
i = Inotify()
i.add_watch(self.folder)
for event in i.event_gen(yield_nones=False):
(header, type_names, watch_path, filename) = event
if ('IN_CLOSE_WRITE' in type_names):
logging.info("File %s noticed by notify thread for project %s in %s"%(filename,
self.project_id,watch_path))
processing.Processing.process_incoming_file(watch_path,filename,self.project_id)
def run_daemon(args):
# Main loop waiting on inotify file events
inotify = Inotify(block_duration_s=1) # event_gen blocks for 1 second
inotify.add_watch(args.synapse_service_dir.encode(),
IN_MOVED_TO | IN_MODIFY)
services_by_dependencies_time = 0
for event in inotify.event_gen(
): # blocks for only up to 1 second at a time
if services_by_dependencies_time + args.update_secs < time.time():
services_by_dependencies = smartstack_dependencies_of_running_firewalled_services(
soa_dir=args.soa_dir, )
services_by_dependencies_time = time.time()
if event is None:
continue
process_inotify_event(event, services_by_dependencies, args.soa_dir,
args.synapse_service_dir)
class InotifyRecursive(object):
def __init__(self, path, mask=IN_ALL_EVENTS, block_duration_s=1):
self.__root_path = path
# No matter what we actually received as the mask, make sure we have
# the minimum that we require to curate our list of watches.
self.__mask = mask | IN_ISDIR | IN_CREATE | IN_DELETE
self.__i = Inotify(block_duration_s=block_duration_s)
self.__load_tree(path)
def __load_tree(self, path):
q = [path]
while q:
current_path = q[0]
del q[0]
self.__i.add_watch(current_path, self.__mask)
for filename in os.listdir(current_path):
entry_filepath = os.path.join(current_path, filename)
if os.path.isdir(entry_filepath) is False:
continue
q.append(entry_filepath)
def event_gen(self):
for event in self.__i.event_gen():
if event is not None:
(header, type_names, path, filename) = event
if header.mask & IN_ISDIR:
full_path = os.path.join(path, filename)
if header.mask & IN_CREATE:
self.__i.add_watch(full_path, self.__mask)
# solved minor situations that the inotify cannot discover the change
for root, dirs, files in os.walk(full_path):
for name in dirs:
self.__i.add_watch(os.path.join(root, name))
elif header.mask & IN_DELETE:
self.__i.remove_watch(full_path, superficial=True)
yield event
def __init__(self):
self._inotify = Inotify()
self._inotify_mask = IN_ALL_EVENTS & (~IN_ACCESS &
~IN_OPEN &
~IN_CLOSE_NOWRITE &
~IN_CLOSE_WRITE)
self._dir_queue = multiprocessing.Queue()
self._sync_queue = multiprocessing.Queue()
self._watched_dirs = {} # {lib_name: set(dirs)}
super().__init__(target=self._main, args=(self._dir_queue,
self._sync_queue))
class InotifyReloader(Reloader):
event_mask = (inotify.constants.IN_CREATE | inotify.constants.IN_DELETE
| inotify.constants.IN_DELETE_SELF
| inotify.constants.IN_MODIFY
| inotify.constants.IN_MOVE_SELF
| inotify.constants.IN_MOVED_FROM
| inotify.constants.IN_MOVED_TO)
def __init__(self, extra_files=None, callback=None):
super(InotifyReloader, self).__init__(extra_files, callback)
self.watcher = Inotify()
def get_files_or_dir(self):
dirs = set([os.path.dirname(path) for path in self.file])
self.file = dirs
def add_extra(self, extra):
dirname = os.path.dirname(extra)
if dirname in self.file:
return
self.watcher.add_watch(dirname, mask=self.event_mask)
self.file.add(dirname)
def run(self):
for dirname in self.file:
self.watcher.add_watch(dirname, mask=self.event_mask)
for event in self.watcher.event_gen():
if event is None:
continue
filename = event[3]
self.callback(filename)
def inotify(args):
"Setup to run inotify loop"
setup_logging(args)
inot = Inotify()
inot.add_watch(str(args.session), mask=IN_CREATE ^ IN_DELETE)
global hooks
if args.hooks:
hooks = import_user(args.hooks)
else:
hooks = None
try:
con = sqlite3.connect(str(args.sql_file),
detect_types=sqlite3.PARSE_DECLTYPES)
logging.info("Repopulating database.")
populate_session_tbl(con, args.session, args.no_action, args=args)
queues = OrderedDict((('create', deque()), ('retry_create', deque()),
('remove', deque())))
qfuncs = {
'create': qfunc_create,
'retry_create': qfunc_retry_create,
'remove': hooks_and_remove_torrent
}
inot_funcs = {
('torrent', ('IN_CREATE', )): queues['create'].append,
('torrent', ('IN_DELETE', )): queues['remove'].append
}
complete_hook = getattr(hooks, 'complete', None)
if complete_hook:
queues['complete'] = deque()
inot_funcs[('complete',
('IN_CREATE', ))] = queues['complete'].append
qfuncs['complete'] = complete_hook
logging.info("Entering inotify loop.")
preloop_hook = getattr(hooks, 'pre_loop', None)
if preloop_hook:
preloop_hook(con, inot, args, queues, qfuncs, inot_funcs)
queues.move_to_end('remove')
while True:
try:
inotify_loop(con, inot, args, queues, qfuncs, inot_funcs)
except rTorEventException as e:
logging.exception("Something happened.")
except (KeyboardInterrupt, SystemExit):
logging.info("Exiting due to interrupt.")
raise
except Exception:
logging.exception("Unhandled exception.")
raise
finally:
postloop_hook = getattr(hooks, 'post_loop', None)
if postloop_hook:
postloop_hook(con, inot, args)
inot.remove_watch(bytes(args.session))
con.close()
def watch_prefix_file(file_name):
""" Using inotify function is looking for IN_CLOSE_WRITE events, that happens when pmacct is pushing new data to
_PMACCT_DATA file. write_to_db is called to store new data into database. On every iteration main thread
status is checked.
"""
inotify_obj = Inotify()
inotify_obj.add_watch(file_name)
try:
for event in inotify_obj.event_gen():
if event is not None:
if event[1] == ['IN_CLOSE_WRITE']:
logger.debug("Found IN_CLOSE_WRITE event")
write_to_db()
else:
if not main_thread().is_alive():
logger.error('Main thread died, stopping all child threads')
# Canceling Timer thread
timer_obj.cancel()
# Breaking watcher thread loop
break
finally:
inotify_obj.remove_watch(file_name)
def main():
monitor = Inotify()
monitor_messaging_manager = MessagingManager(MessagingManagerType.SERVER, "tcp://127.0.0.1:5555")
watch_paths_list = [ b'/home/anton/test/' ]
pair_events_list = []
stop_events_processing_flag = Event()
for path in watch_paths_list:
monitor.add_watch(path, inotify_constants.IN_MOVE | inotify_constants.IN_MOVED_TO | inotify_constants.IN_MOVED_FROM | inotify_constants.IN_CREATE)
try:
while True:
messages = monitor_messaging_manager.get_all_received_messages()
process_received_messages(monitor, messages)
stop_events_processing_timer = Timer(10, stop_events_processing_flag.set)
stop_events_processing_timer.start()
process_received_events(monitor, monitor_messaging_manager, pair_events_list, stop_events_processing_flag)
except KeyboardInterrupt:
for path in watch_paths_list:
monitor.remove_watch(path)
raise SystemExit()
#!/usr/bin/env python3
from pathlib import Path
import sched
import signal
from time import strftime, localtime, time, sleep, mktime, strptime
from exif import Image
from inotify.adapters import Inotify
import magic
i = Inotify()
i.add_watch('/mnt/current')
s = sched.scheduler(time)
signal.signal(signal.SIGTERM, signal.default_int_handler)
def get_creation_time(file):
if magic.from_file(str(file), mime=True).startswith('image/'):
try:
with file.open('rb') as content:
exif = Image(content)
return mktime(strptime(exif.datetime, "%Y:%m:%d %H:%M:%S"))
except:
return file.stat().st_mtime
else:
return file.stat().st_mtime
def do_move(file):
creation_time = get_creation_time(file)
levelnum = logging.INFO
logging.basicConfig(level=levelnum)
# Load up system inforamtion into a Node object representing this host
localnode = Node(localhost, ifaddrs(), bool(sysctl_ipv4_forward()))
network.add_node(localnode)
# Dump this host's information to a file on the share
with open(path.join(sharedir, localhost + '.json'), 'w') as f:
json.dump(localnode, f, cls=Encoder)
logging.debug("Wrote localhost node inforamtion to %s",
path.join(sharedir, localhost + '.json'))
# Add a filsytem watcher to get notified of changes
notify = Inotify()
notify.add_watch(sharedir, IN_CLOSE_WRITE)
# Load the current state of every written node
# This is done after adding the watcher to prevent race conditions
for name, node in load_nodes(sharedir).items():
network.add_node(node)
logging.info("Loaded %d nodes: %s", len(network.nodes),
" ".join(n.name for n in network.nodes.values()))
update_routes()
# Set up a DNS server to resolve names across subnets
resolver = DnsResolver(network, localhost)
dnsserver = DNSServer(resolver, port=dnsport, address=dnsaddr)
dnsserver.start_thread()
class Envo:
@dataclass
class Sets:
stage: str
addons: List[str]
init: bool
root: Path
stage: str
selected_addons: List[str]
addons: List[str]
files_watchdog_thread: Thread
def __init__(self, sets: Sets) -> None:
self.se = sets
self.addons = ["venv"]
unknown_addons = [a for a in self.se.addons if a not in self.addons]
if unknown_addons:
raise RuntimeError(f"Unknown addons {unknown_addons}")
self.inotify = Inotify()
self._environ_before: Dict[str, Any] = os.environ.copy()
self.shell = Shell.create()
self.env_dirs = self._get_env_dirs()
def spawn_shell(self) -> None:
sys.path.insert(0, str(self.env_dirs[0].parent))
self._start_files_watchdog()
self.send_env_to_shell()
self.shell.start()
def send_env_to_shell(self) -> None:
try:
env: Env = self.get_env()
env_prefix = f"{env.meta.emoji}({env.get_full_name()})"
env.validate()
env.activate()
self.shell.set_variable("env", env)
self.shell.set_variable("environ", os.environ)
self.shell.environ.update(os.environ)
self.shell.set_prompt_prefix(env_prefix)
except Env.EnvException as exc:
logger.error(exc)
self.shell.set_prompt_prefix(f"❌{env_prefix})")
except Exception:
from traceback import print_exc
print_exc()
def _files_watchdog(self) -> None:
for event in self.inotify.event_gen(yield_nones=False):
(_, type_names, path, filename) = event
if "IN_CLOSE_WRITE" in type_names:
logger.info(f'\nDetected changes in "{str(path)}".')
logger.info("Reloading...")
self.send_env_to_shell()
print("\r" + self.shell.formated_prompt, end="")
def _start_files_watchdog(self) -> None:
for d in self.env_dirs:
comm_env_file = d / "env_comm.py"
env_file = d / f"env_{self.se.stage}.py"
self.inotify.add_watch(str(comm_env_file))
self.inotify.add_watch(str(env_file))
self.files_watchdog_thread = Thread(target=self._files_watchdog)
self.files_watchdog_thread.start()
def _get_env_dirs(self) -> List[Path]:
ret = []
path = Path(".").absolute()
while True:
env_file = path / f"env_{self.se.stage}.py"
if env_file.exists():
ret.append(path)
else:
if path == Path("/"):
break
path = path.parent
return ret
def _create_init_files(self) -> None:
for d in self.env_dirs:
init_file = d / "__init__.py"
if init_file.exists():
init_file_tmp = d / Path("__init__.py.tmp")
init_file_tmp.touch()
init_file_tmp.write_text(init_file.read_text())
if not init_file.exists():
init_file.touch()
init_file.write_text("# __envo_delete__")
def _delete_init_files(self) -> None:
for d in self.env_dirs:
init_file = d / Path("__init__.py")
init_file_tmp = d / Path("__init__.py.tmp")
if init_file.read_text() == "# __envo_delete__":
init_file.unlink()
if init_file_tmp.exists():
init_file.touch()
init_file.write_text(init_file_tmp.read_text())
init_file_tmp.unlink()
def _unload_modules(self) -> None:
modules = list(sys.modules.keys())[:]
for m in modules:
for d in self.env_dirs:
package = d.name
if m.startswith(package):
sys.modules.pop(m)
def get_env(self) -> Env:
env_dir = self.env_dirs[0]
package = env_dir.name
env_name = f"env_{self.se.stage}"
env_file = env_dir / f"{env_name}.py"
module_name = f"{package}.{env_name}"
with ILock("envo_lock"):
self._create_init_files()
self._unload_modules()
try:
module = import_module_from_file(env_file)
env: Env
env = module.Env()
return env
except ImportError as exc:
logger.error(f"""Couldn't import "{module_name}" ({exc}).""")
raise
finally:
self._delete_init_files()
def _create_from_templ(self,
templ_file: Path,
output_file: Path,
is_comm: bool = False) -> None:
Environment(keep_trailing_newline=True)
template = Template((templates_dir / templ_file).read_text())
if output_file.exists():
logger.error(f"{str(output_file)} file already exists.")
exit(1)
output_file.touch()
env_dir = Path(".").absolute()
package_name = comm.dir_name_to_pkg_name(env_dir.name)
class_name = comm.dir_name_to_class_name(package_name) + "Env"
context = {
"class_name": class_name,
"name": env_dir.name,
"package_name": package_name,
"stage": self.se.stage,
"emoji": stage_emoji_mapping[self.se.stage],
"selected_addons": self.se.addons,
}
if not is_comm:
context["stage"] = self.se.stage
output_file.write_text(template.render(**context))
def init_files(self) -> None:
env_comm_file = Path("env_comm.py")
if not env_comm_file.exists():
self._create_from_templ(Path("env_comm.py.templ"),
env_comm_file,
is_comm=True)
env_file = Path(f"env_{self.se.stage}.py")
self._create_from_templ(Path("env.py.templ"), env_file)
logger.info(f"Created {self.se.stage} environment 🍰!")
def handle_command(self, args: argparse.Namespace) -> None:
if args.version:
from envo.__version__ import __version__
logger.info(__version__)
return
if args.init:
self.init_files()
return
if args.save:
self.get_env().dump_dot_env()
return
if args.dry_run:
self.get_env().print_envs()
else:
self.spawn_shell()
class Monitor(multiprocessing.Process):
def __init__(self):
self._inotify = Inotify()
self._inotify_mask = IN_ALL_EVENTS & (
~IN_ACCESS & ~IN_OPEN & ~IN_CLOSE_NOWRITE & ~IN_CLOSE_WRITE)
self._dir_queue = multiprocessing.Queue()
self._sync_queue = multiprocessing.Queue()
self._watched_dirs = {} # {lib_name: set(dirs)}
super().__init__(target=self._main,
args=(self._dir_queue, self._sync_queue))
def _main(self, dir_queue, sync_queue):
next_sync_t = time() + SYNC_INTERVAL
sync_dirs = set()
try:
while True:
# Check for new directories to watch
while not dir_queue.empty():
lib, path = dir_queue.get()
watched = (path
in set(chain(*self._watched_dirs.values())))
if lib not in self._watched_dirs:
self._watched_dirs[lib] = set()
self._watched_dirs[lib].add(path)
if not watched:
self._inotify.add_watch(
str(path).encode(LOCAL_FS_ENCODING),
self._inotify_mask)
print("Watching {} (lib: {}) (total dirs: {})".format(
path, lib, len(watched)))
# Process Inotify
for event in self._inotify.event_gen():
if event is None:
break
(header, type_names, watch_path, filename) = event
watch_path = Path(str(watch_path, LOCAL_FS_ENCODING))
filename = Path(str(filename, LOCAL_FS_ENCODING))
print("WD=({:d}) MASK=({:d}) "
"MASK->NAMES={} WATCH-PATH={} FILENAME={}".format(
header.wd, header.mask, type_names, watch_path,
filename))
if header.mask & (IN_ATTRIB | IN_CREATE | IN_DELETE |
IN_MODIFY | IN_MOVED_TO | IN_MOVED_FROM):
if IN_ISDIR & header.mask and header.mask & IN_CREATE:
watch_path = watch_path / filename
elif IN_ISDIR & header.mask and header.mask & IN_DELETE:
self._inotify.remove_watch(
str(watch_path).encode(LOCAL_FS_ENCODING))
sync_dirs.add(watch_path)
def _reqSync(l, d):
if d.exists():
sync_queue.put((lib, d))
print("Requesting sync {} (lib: {})".format(d, l))
if time() > next_sync_t:
for d in sync_dirs:
for lib in self._watched_dirs:
lib_paths = self._watched_dirs[lib]
if d in lib_paths:
_reqSync(lib, d)
if not d.exists():
self._watched_dirs[lib].remove(d)
elif d.parent in lib_paths:
_reqSync(lib, d)
self.dir_queue.put((lib, d))
sync_dirs.clear()
next_sync_t = time() + SYNC_INTERVAL
except KeyboardInterrupt:
pass
finally:
for path in set(chain(*self._watched_dirs.values())):
self._inotify.remove_watch(str(path).encode(LOCAL_FS_ENCODING))
@property
def dir_queue(self):
return self._dir_queue
@property
def sync_queue(self):
return self._sync_queue
def __init__(self, extra_files=None, callback=None):
super(InotifyReloader, self).__init__(extra_files, callback)
self.watcher = Inotify()
class Monitor(multiprocessing.Process):
def __init__(self):
self._inotify = Inotify()
self._inotify_mask = IN_ALL_EVENTS & (~IN_ACCESS &
~IN_OPEN &
~IN_CLOSE_NOWRITE &
~IN_CLOSE_WRITE)
self._dir_queue = multiprocessing.Queue()
self._sync_queue = multiprocessing.Queue()
self._watched_dirs = {} # {lib_name: set(dirs)}
super().__init__(target=self._main, args=(self._dir_queue,
self._sync_queue))
def _main(self, dir_queue, sync_queue):
next_sync_t = time() + SYNC_INTERVAL
sync_dirs = set()
try:
while True:
# Check for new directories to watch
while not dir_queue.empty():
lib, path = dir_queue.get()
watched = (path in set(chain(*self._watched_dirs.values())))
if lib not in self._watched_dirs:
self._watched_dirs[lib] = set()
self._watched_dirs[lib].add(path)
if not watched:
self._inotify.add_watch(str(path), self._inotify_mask)
log.info(f"Watching {path} (lib: {lib}) "
f"(total dirs: {len(self._watched_dirs[lib])}")
# Process Inotify
for event in self._inotify.event_gen():
if event is None:
break
(header,
type_names,
watch_path,
filename) = event
watch_path = Path(watch_path)
filename = Path(filename)
log.debug(
f"WD=({header.wd}) MASK=({header.mask}) "
f"MASK->NAMES={type_names} WATCH-PATH={watch_path} FILENAME={filename}")
if header.mask & (IN_ATTRIB | IN_CREATE | IN_DELETE |
IN_MODIFY | IN_MOVED_TO | IN_MOVED_FROM):
if IN_ISDIR & header.mask and header.mask & IN_CREATE:
watch_path = watch_path / filename
elif IN_ISDIR & header.mask and header.mask & IN_DELETE:
self._inotify.remove_watch(str(watch_path))
sync_dirs.add(watch_path)
def _reqSync(l, d):
if d.exists():
sync_queue.put((lib, d))
log.info(f"Requesting sync {d} (lib: {l})")
if time() > next_sync_t:
for d in sync_dirs:
for lib in self._watched_dirs:
lib_paths = self._watched_dirs[lib]
if d in lib_paths:
_reqSync(lib, d)
if not d.exists():
self._watched_dirs[lib].remove(d)
elif d.parent in lib_paths:
_reqSync(lib, d)
self.dir_queue.put((lib, d))
sync_dirs.clear()
next_sync_t = time() + SYNC_INTERVAL
except KeyboardInterrupt:
pass
finally:
for path in set(chain(*self._watched_dirs.values())):
self._inotify.remove_watch(str(path))
@property
def dir_queue(self):
return self._dir_queue
@property
def sync_queue(self):
return self._sync_queue
def process_loop(clone, label, tmp_mount, concurrent, exclude, sudo):
"""Drop into the iNotify loop continually scanning for new devices.
Args:
clone: list of files/folders that need to be put on formatted drives.
label: name to apply to the formatted drives.
tmp_mount: mount folder for the formatted device to clone files to.
concurrent (int): number of concurrent threads for writing.
exclude (Set[str]): file extensions to ignore while cloning.
sudo (Command): sudo/root password (if we need it)
"""
# what we want to do with each device
data = {
'clone': clone,
'label': label,
'tmp_mount': tmp_mount,
'concurrent': concurrent,
'exclude': exclude,
'sudo': sudo
}
manager = CallbackManager(data)
notify = Inotify(paths=['/dev'])
# map the device identifier of our "original usb sticks" so
# we don't accidentally overwrite something that was already plugged in.
# However, this will not count against original devices that are
# unplugged and plugged back in.
original_devices = {o: None for o in usb_storage_devices()}
current_devices = {}
for device in original_devices.keys():
logger.debug('original device, %s', device)
# track the Inotify events in a loop.
last_events = deque(list(), maxlen=6)
# prime the state; curry
get_state = process_state(notify, last_events, original_devices)
logger.info('~ready')
try:
while True:
new_devices = get_state(current_devices)
if new_devices:
# device was added
for device, mount in new_devices.items():
logger.info('found new device %s at %s', device, mount)
current_devices[device] = mount
manager.on_new_device(device, mount)
continue
current_usb = usb_storage_devices()
for device in list(current_devices.keys()):
if device not in current_usb:
# device was removed
mount = current_devices.pop(device)
logger.info('device was removed %s from %s', device, mount)
manager.on_removed_device(device, mount)
# infinite loop
except KeyboardInterrupt:
manager.stop()
raise
except Exception as e:
logger.error(e)
raise
def __init__(self, extra_files=None, callback=None):
super(InotifyReloader, self).__init__()
self.setDaemon(True)
self._callback = callback
self._dirs = set()
self._watcher = Inotify()
class Envo:
@dataclass
class Sets:
stage: str
addons: List[str]
init: bool
environ_before = Dict[str, str]
selected_addons: List[str]
addons: List[str]
files_watchdog_thread: Thread
shell: shell.Shell
inotify: Inotify
env_dirs: List[Path]
quit: bool
env: Env
def __init__(self, sets: Sets) -> None:
self.se = sets
self.addons = ["venv"]
unknown_addons = [a for a in self.se.addons if a not in self.addons]
if unknown_addons:
raise EnvoError(f"Unknown addons {unknown_addons}")
self.inotify = Inotify()
self.env_dirs = self._get_env_dirs()
self.quit: bool = False
self.environ_before = os.environ.copy() # type: ignore
self._set_context_thread: Optional[Thread] = None
def spawn_shell(self, type: Literal["fancy", "simple",
"headless"]) -> None:
"""
:param type: shell type
"""
self.shell = shell.shells[type].create()
self._start_files_watchdog()
self.restart()
self.shell.start()
self._on_unload()
self._stop_files_watchdog()
self._on_destroy()
def restart(self) -> None:
try:
os.environ = self.environ_before.copy() # type: ignore
if not hasattr(self, "env"):
self.env = self.create_env()
self._on_create()
else:
self._on_unload()
self.env = self.create_env()
self.env.validate()
self.env.activate()
self._on_load()
self.shell.reset()
self.shell.set_variable("env", self.env)
self.shell.set_variable("environ", self.shell.environ)
self._set_context_thread = Thread(target=self._set_context)
self._set_context_thread.start()
glob_cmds = [
c for c in self.env.get_magic_functions()["command"]
if c.kwargs["glob"]
]
for c in glob_cmds:
self.shell.set_variable(c.name, c)
self.shell.pre_cmd = self._on_precmd
self.shell.on_stdout = self._on_stdout
self.shell.on_stderr = self._on_stderr
self.shell.post_cmd = self._on_postcmd
self.shell.environ.update(self.env.get_env_vars())
self.shell.set_prompt_prefix(
self._get_prompt_prefix(
loading=self._set_context_thread.is_alive()))
except EnvoError as exc:
logger.error(exc)
self.shell.set_prompt_prefix("❌")
except Exception:
from traceback import print_exc
print_exc()
self.shell.set_prompt_prefix("❌")
def _get_prompt_prefix(self, loading: bool = False) -> str:
env_prefix = f"{self.env.meta.emoji}({self.env.get_full_name()})"
if loading:
env_prefix = "⏳" + env_prefix
return env_prefix
def _set_context(self) -> None:
for c in self.env.get_magic_functions()["context"]:
context = c()
self.shell.update_context(context)
self.shell.set_prompt_prefix(self._get_prompt_prefix(loading=False))
def _on_create(self) -> None:
for h in self.env.get_magic_functions()["oncreate"]:
h()
def _on_destroy(self) -> None:
for h in self.env.get_magic_functions()["ondestroy"]:
h()
def _on_load(self) -> None:
for h in self.env.get_magic_functions()["onload"]:
h()
def _on_unload(self) -> None:
for h in self.env.get_magic_functions()["onunload"]:
h()
def _on_precmd(self, command: str) -> str:
for h in self.env.get_magic_functions()["precmd"]:
if re.match(h.kwargs["cmd_regex"], command):
ret = h(command=command) # type: ignore
if ret:
command = ret
return command
def _on_stdout(self, command: str, out: str) -> str:
for h in self.env.get_magic_functions()["onstdout"]:
if re.match(h.kwargs["cmd_regex"], command):
ret = h(command=command, out=out) # type: ignore
if ret:
out = ret
return out
def _on_stderr(self, command: str, out: str) -> str:
for h in self.env.get_magic_functions()["onstderr"]:
if re.match(h.kwargs["cmd_regex"], command):
ret = h(command=command, out=out) # type: ignore
if ret:
out = ret
return out
def _on_postcmd(self, command: str, stdout: List[str],
stderr: List[str]) -> None:
for h in self.env.get_magic_functions()["postcmd"]:
if re.match(h.kwargs["cmd_regex"], command):
h(command=command, stdout=stdout,
stderr=stderr) # type: ignore
def _files_watchdog(self) -> None:
for event in self.inotify.event_gen(yield_nones=False):
if self.quit:
return
(_, type_names, path, filename) = event
if "IN_CLOSE_WRITE" in type_names:
logger.info(f'\nDetected changes in "{str(path)}".')
logger.info("Reloading...")
self.restart()
print("\r" + self.shell.prompt, end="")
def _start_files_watchdog(self) -> None:
for d in self.env_dirs:
comm_env_file = d / "env_comm.py"
env_file = d / f"env_{self.se.stage}.py"
self.inotify.add_watch(str(comm_env_file))
self.inotify.add_watch(str(env_file))
self.files_watchdog_thread = Thread(target=self._files_watchdog)
self.files_watchdog_thread.start()
def _stop_files_watchdog(self) -> None:
self.quit = True
env_comm = self.env_dirs[0] / "env_comm.py"
# Save the same content to trigger inotify event
env_comm.read_text()
def _get_env_dirs(self) -> List[Path]:
ret = []
path = Path(".").absolute()
while True:
env_file = path / f"env_{self.se.stage}.py"
if env_file.exists():
ret.append(path)
else:
if path == Path("/"):
break
path = path.parent
return ret
def _create_init_files(self) -> None:
"""
Create __init__.py files if not exist.
If exist save them to __init__.py.tmp to recover later.
This step is needed because there might be some content in existing that might crash envo.
"""
for d in self.env_dirs:
init_file = d / "__init__.py"
if init_file.exists():
init_file_tmp = d / Path("__init__.py.tmp")
init_file_tmp.touch()
init_file_tmp.write_text(init_file.read_text())
if not init_file.exists():
init_file.touch()
init_file.write_text("# __envo_delete__")
def _delete_init_files(self) -> None:
"""
Delete __init__.py files if crated otherwise recover.
:return:
"""
for d in self.env_dirs:
init_file = d / Path("__init__.py")
init_file_tmp = d / Path("__init__.py.tmp")
if init_file.read_text() == "# __envo_delete__":
init_file.unlink()
if init_file_tmp.exists():
init_file.touch()
init_file.write_text(init_file_tmp.read_text())
init_file_tmp.unlink()
def create_env(self) -> Env:
env_dir = self.env_dirs[0]
package = env_dir.name
env_name = f"env_{self.se.stage}"
env_file = env_dir / f"{env_name}.py"
module_name = f"{package}.{env_name}"
# We have to lock this part in case there's other shells concurrently executing this code
with ILock("envo_lock"):
self._create_init_files()
# unload modules
for m in list(sys.modules.keys())[:]:
if m.startswith("env_"):
sys.modules.pop(m)
try:
module = import_from_file(env_file)
env: Env
env = module.Env()
return env
except ImportError as exc:
raise EnvoError(
f"""Couldn't import "{module_name}" ({exc}).""")
finally:
self._delete_init_files()
def _create_from_templ(self,
templ_file: Path,
output_file: Path,
is_comm: bool = False) -> None:
"""
Create env file from template.
:param templ_file:
:param output_file:
:param is_comm:
:return:
"""
Environment(keep_trailing_newline=True)
if output_file.exists():
raise EnvoError(f"{str(output_file)} file already exists.")
env_dir = Path(".").absolute()
package_name = misc.dir_name_to_pkg_name(env_dir.name)
class_name = misc.dir_name_to_class_name(package_name) + "Env"
if misc.is_valid_module_name(env_dir.name):
env_comm_import = f"from env_comm import {class_name}Comm"
else:
env_comm_import = (
"from pathlib import Path\n"
f"from envo.misc import import_from_file\n\n\n"
f'{class_name}Comm = import_from_file(Path("env_comm.py")).{class_name}Comm'
)
context = {
"class_name": class_name,
"name": env_dir.name,
"package_name": package_name,
"stage": self.se.stage,
"emoji": stage_emoji_mapping.get(self.se.stage, "🙂"),
"selected_addons": self.se.addons,
"env_comm_import": env_comm_import,
}
if not is_comm:
context["stage"] = self.se.stage
misc.render_py_file(templates_dir / templ_file,
output=output_file,
context=context)
def init_files(self) -> None:
env_comm_file = Path("env_comm.py")
if not env_comm_file.exists():
self._create_from_templ(Path("env_comm.py.templ"),
env_comm_file,
is_comm=True)
env_file = Path(f"env_{self.se.stage}.py")
self._create_from_templ(Path("env.py.templ"), env_file)
logger.info(f"Created {self.se.stage} environment 🍰!")
def handle_command(self, args: argparse.Namespace) -> None:
if args.version:
from envo.__version__ import __version__
logger.info(__version__)
return
if args.init:
self.init_files()
return
if not self.env_dirs:
raise EnvoError("Couldn't find any env!\n"
'Forgot to run envo --init" first?')
sys.path.insert(0, str(self.env_dirs[0]))
if args.save:
self.create_env().dump_dot_env()
return
if args.command:
self.spawn_shell("headless")
try:
self.shell.default(args.command)
except SystemExit as e:
sys.exit(e.code)
else:
sys.exit(self.shell.history[-1].rtn)
if args.dry_run:
content = "\n".join([
f'export {k}="{v}"'
for k, v in self.create_env().get_env_vars().items()
])
print(content)
else:
self.spawn_shell(args.shell)
if not el[2] == 2:
query.append([(
el[0][0],
len(query),
), el[1], el[2]])
except (IOError, UnicodeDecodeError) as e:
print("File query.pkl not found or corrupt, creating new empty query")
chdir("/home/catomaior/Documents/Scuola/A.S. 2019-2020")
if isConnected():
for el in query:
el[2] = 1
start_new_thread(el[1], el[0])
i = Inotify()
for dir in DIRS:
i.add_watch(dir)
lastEvent = ()
operations = {"temp": "temp"}
for event in i.event_gen(yield_nones=False):
(_, type_names, path, filename) = event
if (filename.endswith(".pdf") or filename.endswith(".pptx")) and not (
filename[0] == "." or (path, filename, type_names) == lastEvent):
if not filename in operations:
operations[filename] = []
operations[filename].append(type_names[0])
#print(operations[filename], filename, path)
def __init__(self, process_number, fields, input_directory, output_directory, user=None):
self._pcap_to_csv_pool = PcapToCsvPool(process_number, fields, output_directory, user)
self._inotify = i = Inotify()
self._inotify.add_watch(input_directory.encode("utf8"))
self._log = logging.getLogger("PcapToCsv")
def __init__(self, extra_files=None, callback=None):
super(InotifyReloader, self).__init__()
self.setDaemon(True)
self._callback = callback
self._dirs = set()
self._watcher = Inotify()