def __init__(self, rmr_port=4562, rmr_wait_for_ready=True, use_fake_sdl=False, post_init=None):
"""
Documented in the class comment.
"""
# PUBLIC, can be used by xapps using self.(name):
self.logger = Logger(name=__name__)
# Start rmr rcv thread
self._rmr_loop = xapp_rmr.RmrLoop(port=rmr_port, wait_for_ready=rmr_wait_for_ready)
self._mrc = self._rmr_loop.mrc # for convenience
# SDL
self.sdl = SDLWrapper(use_fake_sdl)
# Config
# The environment variable specifies the path to the Xapp config file
self._config_path = os.environ.get(CONFIG_FILE_ENV, None)
if self._config_path and os.path.isfile(self._config_path):
self._inotify = inotify_simple.INotify()
self._inotify.add_watch(self._config_path, inotify_simple.flags.MODIFY)
self.logger.debug("__init__: watching config file {}".format(self._config_path))
else:
self._inotify = None
self.logger.warning("__init__: NOT watching any config file")
# run the optionally provided user post init
if post_init:
post_init(self)
def __init__(self, dname):
self._ino = inotify_simple.INotify()
self._wd = self._ino.add_watch(
dname, inotify_simple.flags.CREATE | inotify_simple.flags.MODIFY
| inotify_simple.flags.CLOSE_WRITE)
self._unhandled_events = []
self.dname = dname
def __init__(self, log):
self._inotify = inotify_simple.INotify()
self.rewind_needed = threading.Event()
self._mask = inotify_simple.flags.MODIFY
self._mask |= inotify_simple.flags.ATTRIB
self._mask |= inotify_simple.flags.CLOSE_WRITE
self._mask |= inotify_simple.flags.MOVED_TO
self._mask |= inotify_simple.flags.CREATE
self._mask |= inotify_simple.flags.DELETE_SELF
def _event_loop():
log.debug(f'inotify blocks')
for event in self._inotify.read():
log.debug(f'inotify {event}')
if not _is_event_rerun_worthy(event):
continue
log.debug(f'that was rerun-worthy')
# exhausting the events queue / debouncing
debounce_end_time = time.time() + WATCHER_DEBOUNCE_TIMEOUT
while True:
time_left = debounce_end_time - time.time()
if time_left < 0:
break
_ = self._inotify.read(timeout=time_left)
# finally, set the flag and cease functioning
self.rewind_needed.set()
threading.Thread(target=_event_loop, daemon=True).start()
def __init__(self,
vertexpath: Path,
fragmentpath: Path,
geometrypath: Path = None):
super(HotloadingShader, self).__init__()
self.shaderprogram = DebugShader().getshaderprogram(
) # Have default shader program
self.vertShader = None
self.fragShader = None
self.geomShader = None
fl = inotify_simple.flags.CREATE | inotify_simple.flags.MODIFY | inotify_simple.flags.MOVED_TO
self.inotify = inotify_simple.INotify()
self.vertexPath = vertexpath
self.vertWatch = self.inotify.add_watch(self.vertexPath.parent, fl)
self.fragmentPath = fragmentpath
self.fragWatch = self.inotify.add_watch(self.fragmentPath.parent, fl)
self.geometryPath = geometrypath
if geometrypath is not None:
self.geomWatch = self.inotify.add_watch(self.geometryPath.parent,
fl)
self.regenShader()
def watch_for_changes(self) -> None:
"""
Monitor the sysfs brightness file and notify on modifications.
Create an inotify watch for the sysfs brightness file and wait
for IN_MODIFY events. Once an event is received, start a timer
after which the notification is sent. It gets (re)started with
each event, so that consecutive brightness changes only result
in a single notification.
:return: None
"""
i = inotify_simple.INotify()
i.add_watch(self.brightness_file, mask=inotify_simple.flags.MODIFY)
timer: Optional[Timer] = None
while True:
i.read() # Block until an event is received
if timer is not None:
timer.cancel()
self.new_brightness = self.get_current_brightness()
if self.prev_brightness == self.new_brightness:
continue
timer = Timer(self.keypress_timeout, self.show_notification)
timer.start()
def __init__(self, url, recurse=False, **kw):
# normalize filesystem paths by removing trailing slashes;
# this is necessary for the check at the beginning of
# `self.watch()` to work correctly when path == self.url.path
try:
path = os.path.abspath(url.path)
except AttributeError:
path = os.path.abspath(url)
super(INotifyPoller, self).__init__(path, **kw)
self._recurse = recurse
self._ifd = inotify.INotify()
self._wd = {}
# Ensure inbox directory exists
if not os.path.exists(self.url.path):
gc3libs.log.info(
"Inbox directory `%s` does not exist,"
" creating it ...", self.url.path)
os.makedirs(self.url.path)
self.watch(self.url.path)
if self._recurse:
for dirpath, dirnames, filenames in os.walk(self.url.path):
self.watch(dirpath)
for name in filenames:
path = os.path.join(self.url.path, dirpath, name)
self.watch(path)
def start(self):
self.log.debug("Starting TorrentWatcher")
self.inotify = inotify_simple.INotify()
self.set_watches(self.conf['dirs'])
try:
self.loop()
except KeyboardInterrupt:
self.log.info("\nExited")
def start_sync(s3_output_location, region, endpoint_url=None):
"""Starts intermediate folder sync which copies files from 'opt/ml/output/intermediate'
directory to the provided s3 output location as files created or modified.
If files are deleted it doesn't delete them from s3.
It starts intermediate folder behavior as a daemonic process and
only if the directory doesn't exists yet, if it does - it indicates
that platform is taking care of syncing files to S3 and container should not interfere.
Args:
s3_output_location (str): name of the script or module.
region (str): the location of the module.
Returns:
(multiprocessing.Process): the intermediate output sync daemonic process.
"""
if not s3_output_location or os.path.exists(intermediate_path):
logger.debug('Could not initialize intermediate folder sync to s3.')
return
# create intermediate and intermediate_tmp directories
os.makedirs(intermediate_path)
os.makedirs(tmp_dir_path)
# configure unique s3 output location similar to how SageMaker platform does it
# or link it to the local output directory
url = urlparse(s3_output_location)
if url.scheme == 'file':
logger.debug('Local directory is used for output. No need to sync any intermediate output.')
return
elif url.scheme != 's3':
raise ValueError("Expecting 's3' scheme, got: %s in %s" % (url.scheme, url))
# create s3 transfer client
client = boto3.client('s3', region, endpoint_url=endpoint_url)
s3_transfer = s3transfer.S3Transfer(client)
s3_uploader = {
'transfer': s3_transfer,
'bucket': url.netloc,
'key_prefix': os.path.join(url.path.lstrip('/'), os.environ.get('TRAINING_JOB_NAME', ''),
'output', 'intermediate'),
}
# Add intermediate folder to the watch list
inotify = inotify_simple.INotify()
watch_flags = inotify_simple.flags.CLOSE_WRITE | inotify_simple.flags.CREATE
watchers = {}
wd = inotify.add_watch(intermediate_path, watch_flags)
watchers[wd] = ''
# start subprocess to sync any files from intermediate folder to s3
p = multiprocessing.Process(target=_watch, args=[inotify, watchers, watch_flags, s3_uploader])
# Make the process daemonic as a safety switch to prevent training job from hanging forever
# in case if something goes wrong and main container process exits in an unexpected way
p.daemon = True
p.start()
return p
def __init__(self, base_path, trigger, complete, cb, cb_ctx):
self.base_path = base_path
self.trigger = trigger
self.complete = complete
self.cb = cb
self.cb_ctx = cb_ctx
self.wd2name = {}
self.inotify = inotify.INotify()
self.main_wd = None
def __init__(self,
rmr_port=4562,
rmr_wait_for_ready=True,
use_fake_sdl=False,
post_init=None):
"""
Documented in the class comment.
"""
# PUBLIC, can be used by xapps using self.(name):
self.logger = Logger(name=__name__)
self._appthread = None
# Start rmr rcv thread
self._rmr_loop = xapp_rmr.RmrLoop(port=rmr_port,
wait_for_ready=rmr_wait_for_ready)
self._mrc = self._rmr_loop.mrc # for convenience
# SDL
self.sdl = SDLWrapper(use_fake_sdl)
# Config
# The environment variable specifies the path to the Xapp config file
self._config_path = os.environ.get(Constants.CONFIG_FILE_ENV, None)
if self._config_path and os.path.isfile(self._config_path):
self._inotify = inotify_simple.INotify()
self._inotify.add_watch(self._config_path,
inotify_simple.flags.MODIFY)
self.logger.debug("__init__: watching config file {}".format(
self._config_path))
else:
self._inotify = None
self.logger.warning("__init__: NOT watching any config file")
# used for thread control of Registration of Xapp
self._keep_registration = True
# configuration data for xapp registration and deregistration
self._config_data = None
if self._config_path and os.path.isfile(self._config_path):
with open(self._config_path) as json_file:
self._config_data = json.load(json_file)
else:
self._keep_registration = False
self.logger.error(
"__init__: Cannot Read config file for xapp Registration")
self._config_data = {}
self._appthread = Thread(target=self.registerXapp).start()
# run the optionally provided user post init
if post_init:
post_init(self)
def __init__(self, fpaths: List[PurePath]) -> None:
self._fpaths = list(fpaths)
self._fpath_by_id: Dict[int, Path] = {}
self._inotify = inotify_simple.INotify()
for fpath in self._fpaths:
LOG.info(f"Resolving {fpath!r}")
real_path = Path(fpath).resolve(strict=True)
LOG.info(f"Watching {real_path!r}")
watch_id = self._inotify.add_watch(str(real_path),
(0
| inotify_flags.OPEN
| inotify_flags.CLOSE_NOWRITE
),
)
LOG.debug(f"Watch ID = {watch_id!r}")
self._fpath_by_id[watch_id] = real_path
def __init__(self,
path: str,
print_func=None,
timeout: typing.Optional[int] = None):
if timeout is not None:
self.timeout = timeout
self.print_func = print_func
self._inotify = _inotify.INotify()
dirpath, fpath = os.path.split(path)
if not dirpath:
dirpath = "."
if not fpath:
raise ValueError("a file path is required")
self._dir = dirpath
self._name = fpath
self._mask = _inotify.flags.DELETE | _inotify.flags.CLOSE_WRITE
self._inotify.add_watch(self._dir, self._mask)
def log_watcher():
inotify = inotify_simple.INotify()
cwl = boto3.client('logs', region_name=region)
cwl.create_log_stream(logGroupName=data['log_group'], logStreamName=data['jobid'])
wait_for_files = dict(map(lambda k: (os.path.join(workflow_path,k),1), data['extra_logs']))
wait_for_files[log_path] = 1
watching = {}
kvargs = {'logGroupName':data['log_group'], 'logStreamName':data['jobid']}
while True:
for f in list(wait_for_files.keys()):
if os.path.exists(f):
wd = inotify.add_watch(f, inotify_simple.flags.MODIFY | inotify_simple.flags.ATTRIB)
watching[wd] = open(f)
del wait_for_files[f]
os.utime(f) # trigger inotify now
for e in inotify.read(timeout=5000,read_delay=1000):
t = round(datetime.datetime.now().timestamp()*1000)
logs = list(map(lambda l: {'timestamp': t, 'message': l}, watching[e.wd].readlines()))
if len(logs)>0:
kvargs['logEvents'] = logs
r = cwl.put_log_events(**kvargs)
kvargs['sequenceToken'] = r['nextSequenceToken']
def __inotify(self):
"""
Setup a watch and return its
"""
logger = logging.getLogger(f'{self.logger_name}__inotify::')
inotify = inotify_simple.INotify()
# Ouput by flag over one numeric value
get_flags = inotify_simple.flags.from_mask
try:
log_wd = inotify.add_watch(self.caller['path'],
self.caller['flags'])
except OSError as error:
logger.error(f"Inotify watch crash: Using:" +
f" '{self.caller['path']}'.")
logger.error(f"{error}: Exiting with status '1'.")
sys.exit(1)
else:
logger.debug(
f"Started monitoring: '{self.caller['path']}', flags:" +
f" '{get_flags(self.caller['flags'])}'," +
f" timeout={self.timeout}.")
return inotify
def __init__(self, directory, flags=inotify_simple.flags.CLOSE_WRITE):
self._inotify = inotify_simple.INotify()
self._inotify_flags = flags
self._dir = directory
def main():
def shutdown(signum, frame):
"""Signal handler for shutting down main."""
logging.debug("main: shutdown triggered")
global run
run = 0
signal.signal(signal.SIGTERM, shutdown)
signal.signal(signal.SIGINT, shutdown)
parser = argparse.ArgumentParser(prog="strelka_dirstream.py",
description="sends files from a directory"
" to a Strelka cluster in"
" near real-time.",
usage="%(prog)s [options]")
parser.add_argument("-d", "--debug",
action="store_true",
default=False,
dest="debug",
help="enable debug messages to the console")
parser.add_argument("-c", "--dirstream-config",
action="store",
dest="dirstream_cfg",
help="path to dirstream configuration file")
args = parser.parse_args()
if args.debug:
logging.basicConfig(
level=logging.DEBUG,
format="%(asctime)s %(levelname)-8s %(message)s",
datefmt="%Y-%m-%d %H:%M:%S")
else:
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)-8s %(message)s",
datefmt="%Y-%m-%d %H:%M:%S")
dirstream_cfg = None
if args.dirstream_cfg:
if not os.path.exists(args.dirstream_cfg):
sys.exit(f"main: stream directory config {args.dirstream_cfg}"
" does not exist")
dirstream_cfg = args.dirstream_cfg
elif os.path.exists(DEFAULT_CONFIGS['sys_dirstream_cfg']):
dirstream_cfg = DEFAULT_CONFIGS['sys_dirstream_cfg']
elif os.path.exists(DEFAULT_CONFIGS['dev_dirstream_cfg']):
dirstream_cfg = DEFAULT_CONFIGS['dev_dirstream_cfg']
if dirstream_cfg is None:
sys.exit("main: no dirstream configuration found")
logging.info(f"main: using dirstream configuration {dirstream_cfg}")
dirstream_cfg = conf.parse_yaml(path=dirstream_cfg, section="dirstream")
directory_cfg = dirstream_cfg.get("directory", {})
network_cfg = dirstream_cfg.get("network", {})
processes_cfg = dirstream_cfg.get("processes", {})
directory = directory_cfg.get("directory")
shutdown_timeout = processes_cfg.get("shutdown_timeout", 10)
worker_count = processes_cfg.get("worker_count", 1)
worker_processes = []
if not os.path.isdir(directory):
sys.exit(f"main: directory {directory} does not exist")
manager = multiprocessing.Manager()
intake_queue = manager.Queue()
inotify = inotify_simple.INotify()
watch_flags = inotify_simple.flags.CLOSE_WRITE
inotify.add_watch(directory, watch_flags)
for _ in range(worker_count):
worker_process = Worker(intake_queue, directory_cfg, network_cfg)
worker_process.start()
worker_processes.append(worker_process)
with os.scandir(directory) as sd:
for entry in sd:
if not entry.name.startswith(".") and entry.is_file():
file_path = os.path.join(directory, entry.name)
intake_queue.put(file_path)
while run:
for process in list(worker_processes):
if not process.is_alive():
process.join()
worker_processes.remove(process)
worker_process = Worker(intake_queue,
directory_cfg,
network_cfg)
worker_process.start()
worker_processes.append(worker_process)
for evt in inotify.read(timeout=100, read_delay=500):
file_path = os.path.join(directory, evt.name)
intake_queue.put(file_path)
logging.info("main: starting shutdown of running child processes"
f" (using timeout value {shutdown_timeout})")
try:
with interruptingcow.timeout(shutdown_timeout,
exception=errors.QuitDirStream):
utils.signal_children(worker_processes, signal.SIGUSR1)
logging.debug("main: finished shutdown of running"
" child processes")
except errors.QuitDirStream:
logging.debug("main: starting forcible shutdown of running"
" child processes")
utils.signal_children(worker_processes, signal.SIGKILL)
logging.info("main: finished")
def track(self):
with inotify_simple.INotify() as notify:
self.add_watch(notify)
self.manage_events(notify)
def start_sync(s3_output_location, region, endpoint_url=None): # pylint: disable=inconsistent-return-statements
"""Start intermediate folder sync, which copies files from 'opt/ml/output/intermediate'
directory to the provided s3 output location as files created or modified.
If files are deleted, it doesn't delete them from s3.
It starts intermediate folder behavior as a daemonic process only if the directory
doesn't exists yet. If the directory does exist, it indicates that the platform is
taking care of syncing files to S3 and the container should not interfere.
Args:
s3_output_location (str): Name of the script or module.
region (str): The location of the module.
endpoint_url (str): An alternative endpoint URL to connect to.
Returns:
(multiprocessing.Process): The intermediate output sync daemonic process.
"""
if not s3_output_location or os.path.exists(intermediate_path):
logger.debug("Could not initialize intermediate folder sync to s3.")
return None
# create intermediate and intermediate_tmp directories
os.makedirs(intermediate_path)
os.makedirs(tmp_dir_path)
# configure unique s3 output location similar to how SageMaker platform does it
# or link it to the local output directory
url = urlparse(s3_output_location)
if url.scheme == "file":
logger.debug(
"Local directory is used for output. No need to sync any intermediate output."
)
return None
elif url.scheme != "s3":
raise ValueError("Expecting 's3' scheme, got: %s in %s" %
(url.scheme, url))
# create s3 transfer client
client = boto3.client("s3", region, endpoint_url=endpoint_url)
s3_transfer = s3transfer.S3Transfer(client)
s3_uploader = {
"transfer":
s3_transfer,
"bucket":
url.netloc,
"key_prefix":
os.path.join(url.path.lstrip("/"),
os.environ.get("TRAINING_JOB_NAME", ""), "output",
"intermediate"),
}
# Add intermediate folder to the watch list
inotify = inotify_simple.INotify()
watch_flags = inotify_simple.flags.CLOSE_WRITE | inotify_simple.flags.CREATE
watchers = {}
wd = inotify.add_watch(intermediate_path, watch_flags)
watchers[wd] = ""
# start subprocess to sync any files from intermediate folder to s3
p = multiprocessing.Process(
target=_watch, args=[inotify, watchers, watch_flags, s3_uploader])
# Make the process daemonic as a safety switch to prevent training job from hanging forever
# in case if something goes wrong and main container process exits in an unexpected way
p.daemon = True
p.start()
return p
def ParseNZB(cfg, dirs, lock, mp_loggerqueue):
setproctitle("gzbx." + os.path.basename(__file__))
logger = mplogging.setup_logger(mp_loggerqueue, __file__)
nzbdir = dirs["nzb"]
incompletedir = dirs["incomplete"]
global TERMINATED
sh = SigHandler_Parser(logger)
signal.signal(signal.SIGINT, sh.sighandler)
signal.signal(signal.SIGTERM, sh.sighandler)
pwdb = PWDBSender()
cwd0 = os.getcwd()
os.chdir(nzbdir)
inotify = inotify_simple.INotify()
watch_flags = inotify_simple.flags.CREATE | inotify_simple.flags.DELETE | inotify_simple.flags.MODIFY | inotify_simple.flags.DELETE_SELF
inotify.add_watch(nzbdir, watch_flags)
isfirstrun = True
while not TERMINATED:
events = get_inotify_events(inotify)
if isfirstrun or events: # and events not in eventslist):
if isfirstrun:
logger.debug(whoami() + "scanning nzb dir ...")
else:
logger.debug(whoami() + "got event in nzb_dir")
for nzb in glob.glob("*.nzb") + glob.glob(".*.nzb"):
if TERMINATED:
break
nzb0 = nzb.split("/")[-1]
if pwdb.exc("db_nzb_exists", [nzb0], {}):
logger.warning(whoami() + " NZB file " + nzb0 +
" already exists in DB")
continue
# replace [ and ] brackets in nzb0 / this is a bug in re!?
if ("[" in nzb0) or ("]" in nzb0):
nzb0 = nzb0.replace("[", "(")
nzb0 = nzb0.replace("]", ")")
try:
os.rename(nzb, dirs["nzb"] + nzb0)
except Exception as e:
logger.error(whoami() + "Cannot rename NZB file " +
nzb0 + " :" + str(e))
continue
logger.info(whoami() + "inserting " + nzb0 + "into db")
newnzb = pwdb.exc("db_nzb_insert", [nzb0], {})
if newnzb:
logger.info(whoami() + "new NZB file " + nzb0 +
" detected")
# update size
# rename nzb here to ....processed
filedic, bytescount = decompose_nzb(nzb0, logger)
if not filedic:
logger.warning("Could not interpret nzb " + nzb0 +
", setting to obsolete")
pwdb.exc("db_nzb_update_status", [nzb0, -1],
{"usefasttrack": False
}) # status "cannot queue / -1"
else:
size_gb = bytescount / (1024 * 1024 * 1024)
infostr = nzb0 + " / " + "{0:.3f}".format(
size_gb) + " GB"
logger.debug(whoami() + "analysing NZB: " + infostr)
# insert files + articles
for key, items in filedic.items():
if TERMINATED:
break
data = []
fsize = 0
for i, it in enumerate(items):
if TERMINATED:
break
if i == 0:
age, nr0 = it
ftype = par2lib.get_file_type(key)
logger.debug(
whoami() +
"analysing and inserting file " + key +
" + articles: age=" + str(age) +
" / nr=" + str(nr0) + " / type=" +
ftype)
newfile = pwdb.exc(
"db_file_insert",
[key, newnzb, nr0, age, ftype], {})
else:
fn, no, size = it
fsize += size
data.append(
(fn, newfile, size, no, time.time()))
data.sort(key=lambda tup: tup[3])
pwdb.exc("db_file_update_size", [key, fsize], {})
pwdb.exc("db_article_insert_many", [data], {})
# if there are nzbs in the download queue, pause after insert
if not pwdb.exc("db_nzb_are_all_nzb_idle", [], {}):
time.sleep(0.3)
logger.info(whoami() + "Added NZB: " + infostr +
" to database / queue")
pwdb.exc(
"db_nzb_update_status", [nzb0, 1],
{"usefasttrack": False}) # status "queued / 1"
logger.debug(whoami() + "Added NZB: " + infostr +
" to GUI")
pwdb.exc("store_sorted_nzbs", [], {})
pwdb.exc("create_allfile_list_via_name",
[nzb0, incompletedir], {})
time.sleep(0.25)
isfirstrun = False
else:
time.sleep(0.25)
os.chdir(cwd0)
logger.debug(whoami() + "exited!")
def renamer_old(child_pipe, renamer_result_queue, mp_loggerqueue,
filewrite_lock):
setproctitle("gzbx." + os.path.basename(__file__))
logger = mplogging.setup_logger(mp_loggerqueue, __file__)
logger.debug(whoami() + "starting renamer process")
sh = SigHandler_Renamer(logger)
signal.signal(signal.SIGINT, sh.sighandler_renamer)
signal.signal(signal.SIGTERM, sh.sighandler_renamer)
pwdb = PWDBSender()
cwd0 = os.getcwd()
while not TERMINATED:
# wait for start command
logger.debug(whoami() + "waiting for start command")
while not TERMINATED:
if child_pipe.poll():
command = child_pipe.recv()
try:
cmd0, source_dir, dest_dir = command
if cmd0 == "start":
break
except Exception as e:
logger.warning(whoami() + str(e))
time.sleep(0.1)
if TERMINATED:
break
if source_dir[-1] != "/":
source_dir += "/"
if dest_dir[-1] != "/":
dest_dir += "/"
try:
os.chdir(dest_dir)
except FileNotFoundError:
os.mkdir(dest_dir)
os.chdir(source_dir)
# init inotify
inotify = inotify_simple.INotify()
watch_flags = inotify_simple.flags.CREATE | inotify_simple.flags.DELETE | inotify_simple.flags.MODIFY | inotify_simple.flags.DELETE_SELF
wd = inotify.add_watch(source_dir, watch_flags)
p2obj = None
p2basename = None
# eventslist = []
isfirstrun = True
while not TERMINATED:
events = get_inotify_events(inotify, filewrite_lock)
if isfirstrun or events: # and events not in eventslist):
logger.debug(whoami() + "Events: " + str(events))
# get all files not yet .renamed
logger.debug(
whoami() +
"reading not yet downloaded files in _downloaded0")
notrenamedfiles = get_not_yet_renamed_files(
source_dir, filewrite_lock)
# get all renames filed & trying to get .par2 file
logger.debug(
whoami() +
"Reading files in _renamed0 & trying to get .par2 file")
renamedfiles, p2obj, p2basename = scan_renamed_dir(
dest_dir, p2obj, filewrite_lock, logger)
# if no par2 in _renamed, check _downloaded0
if not p2obj:
logger.debug(whoami() +
"No p2obj yet found, looking in _downloaded0")
p2obj, p2basename = scan_for_par2(notrenamedfiles, logger)
if p2obj:
logger.debug(whoami() + "p2obj found: " + p2basename)
# rename par2 and move them
p2obj, p2objname = renamer_process_par2s(
source_dir, dest_dir, p2obj, p2basename, notrenamedfiles,
pwdb, renamer_result_queue, filewrite_lock)
# rename & move rar + remaining files
rename_and_move_rarandremainingfiles_old(
p2obj, notrenamedfiles, source_dir, dest_dir, pwdb,
renamer_result_queue, filewrite_lock, logger)
isfirstrun = False
if child_pipe.poll():
command, _, _ = child_pipe.recv()
if command == "pause":
break
os.chdir(cwd0)
try:
if wd:
inotify.rm_watch(wd)
except Exception as e:
logger.warning(whoami() + str(e))
logger.debug(whoami() + "renamer paused")
logger.info(whoami() + "exited!")
def renamer(child_pipe, renamer_result_queue, mp_loggerqueue, filewrite_lock):
setproctitle("gzbx." + os.path.basename(__file__))
logger = mplogging.setup_logger(mp_loggerqueue, __file__)
logger.debug(whoami() + "starting renamer process")
sh = SigHandler_Renamer(logger)
signal.signal(signal.SIGINT, sh.sighandler_renamer)
signal.signal(signal.SIGTERM, sh.sighandler_renamer)
pwdb = PWDBSender()
cwd0 = os.getcwd()
while not TERMINATED:
# wait for start command
logger.debug(whoami() + "waiting for start command")
while not TERMINATED:
if child_pipe.poll():
command = child_pipe.recv()
try:
cmd0, source_dir, dest_dir, nzbname = command
if cmd0 == "start":
break
except Exception as e:
logger.warning(whoami() + str(e))
time.sleep(0.1)
if TERMINATED:
break
if source_dir[-1] != "/":
source_dir += "/"
if dest_dir[-1] != "/":
dest_dir += "/"
try:
os.chdir(dest_dir)
except FileNotFoundError:
os.mkdir(dest_dir)
os.chdir(source_dir)
# init inotify
inotify = inotify_simple.INotify()
watch_flags = inotify_simple.flags.CREATE | inotify_simple.flags.DELETE | inotify_simple.flags.MODIFY | inotify_simple.flags.DELETE_SELF
wd = inotify.add_watch(source_dir, watch_flags)
isfirstrun = True
p2list = pwdb.exc("db_p2_get_p2list", [nzbname], {})
while not TERMINATED:
events = get_inotify_events(inotify, filewrite_lock)
if isfirstrun or events: # and events not in eventslist):
logger.debug(whoami() + "Events: " + str(events))
# in downloaded_dir: look for not renamed files
logger.debug(
whoami() +
"reading not yet downloaded files in _downloaded0")
notrenamedfiles = []
with filewrite_lock:
for fnfull in glob.glob(source_dir +
"*") + glob.glob(source_dir +
".*"):
fnshort = fnfull.split("/")[-1]
rn = pwdb.exc("db_file_get_renamed_name", [fnshort],
{})
is_renamed = rn and (rn != "N/A")
if not is_renamed:
p2 = par2lib.Par2File(fnfull)
if p2:
oldft = pwdb.exc("db_file_get_orig_filetype",
[fnshort], {})
if p2.is_par2():
rarfiles = [
(fn, md5)
for fn, md5 in p2.md5_16khash()
]
p2list.append(
(p2, fnshort, dest_dir + fnshort,
rarfiles))
pwdb.exc("db_p2_insert_p2", [
nzbname, p2, fnshort,
dest_dir + fnshort, rarfiles
], {})
pwdb.exc("db_file_set_file_type",
[fnshort, "par2"], {})
try:
newshortname = fnshort.split(
".par2")[0] + ".par2"
shutil.copyfile(
source_dir + fnshort,
dest_dir + newshortname)
pwdb.exc("db_file_set_renamed_name",
[fnshort, newshortname], {})
renamer_result_queue.put(
(newshortname,
dest_dir + newshortname, "par2",
fnshort, oldft))
except Exception as e:
logger.error(
whoami() + str(e) +
": cannot rename par2 file!")
# par2vol
elif p2.is_par2vol():
pwdb.exc("db_file_set_file_type",
[fnshort, "par2vol"], {})
try:
newshortname = fnshort.split(
".PAR2")[0] + ".PAR2"
shutil.copyfile(
source_dir + fnshort,
dest_dir + newshortname)
pwdb.exc("db_file_set_renamed_name",
[fnshort, newshortname], {})
renamer_result_queue.put(
(newshortname,
dest_dir + newshortname,
"par2vol", fnshort, oldft))
except Exception as e:
logger.error(
whoami() + str(e) +
": cannot rename par2 file!")
# could set # of blocks here in gpeewee
try:
os.remove(fnfull)
except Exception:
pass
else:
notrenamedfiles.append(
(fnfull, fnshort,
par2lib.calc_file_md5hash_16k(fnfull)))
# rename & move rar + remaining files
if notrenamedfiles:
# da hats was
rename_and_move_rarandremainingfiles(
p2list, notrenamedfiles, source_dir, dest_dir, pwdb,
renamer_result_queue, filewrite_lock, logger)
isfirstrun = False
if child_pipe.poll():
command, _, _ = child_pipe.recv()
if command == "pause":
break
os.chdir(cwd0)
try:
if wd:
inotify.rm_watch(wd)
except Exception as e:
logger.warning(whoami() + str(e))
logger.debug(whoami() + "renamer paused")
logger.info(whoami() + "exited!")
def par_verifier(child_pipe, renamed_dir, verifiedrar_dir, main_dir,
mp_loggerqueue, nzbname, pvmode, event_idle, cfg):
setproctitle("gzbx." + os.path.basename(__file__))
logger = mplogging.setup_logger(mp_loggerqueue, __file__)
logger.debug(whoami() + "starting ...")
sh = SigHandler_Verifier(logger)
signal.signal(signal.SIGINT, sh.sighandler_verifier)
signal.signal(signal.SIGTERM, sh.sighandler_verifier)
pwdb = PWDBSender()
event_idle.clear()
if pvmode == "verify":
# p2 = pwdb.get_renamed_p2(renamed_dir, nzbname)
try:
p2list = pwdb.exc("db_p2_get_p2list", [nzbname], {})
p2 = P2(p2list)
except Exception as e:
logger.warning(whoami() + str(e))
# pwdb.db_nzb_update_verify_status(nzbname, 1)
pwdb.exc("db_nzb_update_verify_status", [nzbname, 1], {})
# a: verify all unverified files in "renamed"
unverified_rarfiles = None
try:
# unverified_rarfiles = pwdb.get_all_renamed_rar_files(nzbname)
unverified_rarfiles = pwdb.exc("get_all_renamed_rar_files", [nzbname],
{})
except Exception as e:
logger.debug(whoami() + str(e) +
": no unverified rarfiles met in first run, skipping!")
doloadpar2vols = False
if pvmode == "verify" and not p2:
logger.debug(whoami() + "no par2 file found")
if pvmode == "verify" and unverified_rarfiles and p2:
logger.debug(whoami() + "verifying all unchecked rarfiles")
for filename, f_origname in unverified_rarfiles:
f_short = filename.split("/")[-1]
md5 = calc_file_md5hash(renamed_dir + filename)
md5match = [(pmd5 == md5) for pname, pmd5 in p2.filenames()
if pname == filename]
if False in md5match:
logger.warning(whoami() +
" error in md5 hash match for file " + f_short)
pwdb.exc("db_msg_insert", [
nzbname, "error in md5 hash match for file " + f_short,
"warning"
], {})
pwdb.exc("db_nzb_update_verify_status", [nzbname, -2], {})
pwdb.exc("db_file_update_parstatus", [f_origname, -1], {})
child_pipe.send(True)
else:
logger.info(whoami() + f_short +
" md5 hash match ok, copying to verified_rar dir")
pwdb.exc("db_msg_insert", [
nzbname, f_short +
" md5 hash match ok, copying to verified_rar dir ", "info"
], {})
shutil.copy(renamed_dir + filename, verifiedrar_dir)
pwdb.exc("db_file_update_parstatus", [f_origname, 1], {})
elif (pvmode == "verify" and not p2) or (pvmode == "copy"):
logger.info(whoami() + "copying all rarfiles")
for filename, f_origname in unverified_rarfiles:
f_short = filename.split("/")[-1]
sfvcheck = pwdb.exc("db_nzb_check_sfvcrc32",
[nzbname, renamed_dir, renamed_dir + filename],
{})
if sfvcheck == -1:
logger.warning(whoami() + " error in crc32 check for file " +
f_short)
pwdb.exc("db_msg_insert", [
nzbname, "error in crc32 check for file " + f_short,
"warning"
], {})
pwdb.exc("db_nzb_update_verify_status", [nzbname, -2], {})
pwdb.exc("db_file_update_parstatus", [f_origname, -1], {})
child_pipe.send(True)
continue
logger.debug(whoami() + "copying " + f_short +
" to verified_rar dir")
pwdb.exc("db_msg_insert", [
nzbname, "copying " + f_short + " to verified_rar dir ", "info"
], {})
shutil.copy(renamed_dir + filename, verifiedrar_dir)
pwdb.exc("db_file_update_parstatus", [f_origname, 1], {})
# b: inotify renamed_dir
inotify = inotify_simple.INotify()
watch_flags = inotify_simple.flags.CREATE | inotify_simple.flags.DELETE | inotify_simple.flags.MODIFY | inotify_simple.flags.DELETE_SELF
inotify.add_watch(renamed_dir, watch_flags)
while not TERMINATED:
# allparstatus = pwdb.db_file_getallparstatus(0)
allparstatus = pwdb.exc("db_file_getallparstatus", [0], {})
if 0 not in allparstatus:
event_idle.clear()
logger.info(whoami() +
"all renamed rars checked, exiting par_verifier")
break
events = get_inotify_events(inotify)
event_idle.set()
if events or 0 in allparstatus:
event_idle.clear()
if pvmode == "verify" and not p2:
try:
p2list = pwdb.exc("db_p2_get_p2list", [nzbname], {})
p2 = P2(p2list)
except Exception as e:
logger.debug(whoami() + str(e))
if pvmode == "verify" and p2:
for rar in glob.glob(renamed_dir +
"*") + glob.glob(renamed_dir + ".*"):
rar0 = rar.split("/")[-1]
f0 = pwdb.exc("db_file_get_renamed", [rar0], {})
if not f0:
continue
f0_origname, f0_renamedname, f0_ftype = f0
if not f0_ftype == "rar":
continue
if pwdb.exc("db_file_getparstatus", [rar0],
{}) == 0 and f0_renamedname != "N/A":
f_short = f0_renamedname.split("/")[-1]
md5 = calc_file_md5hash(renamed_dir + rar0)
md5match = [(pmd5 == md5)
for pname, pmd5 in p2.filenames()
if pname == f0_renamedname]
#print(f0_renamedname, md5, " : ", p2.filenames())
#print(md5match)
#print("-" * 80)
if True in md5match:
logger.info(
whoami() + f_short +
" md5 hash match ok, copying to verified_rar dir"
)
pwdb.exc("db_msg_insert", [
nzbname, f_short +
" md5 hash match ok, copying to verified_rar dir ",
"info"
], {})
shutil.copy(renamed_dir + f0_renamedname,
verifiedrar_dir)
pwdb.exc("db_file_update_parstatus",
[f0_origname, 1], {})
elif False in md5match:
logger.warning(
whoami() +
"error in md5 hash match for file " + f_short)
pwdb.exc("db_msg_insert", [
nzbname, "error in md5 hash match for file " +
f_short, "warning"
], {})
pwdb.exc("db_nzb_update_verify_status",
[nzbname, -2], {})
pwdb.exc("db_file_update_parstatus",
[f0_origname, -1], {})
child_pipe.send(True)
else: # if no match at all in p2list -> try sfvcheck / or just copy
sfvcheck = pwdb.exc("db_nzb_check_sfvcrc32",
[nzbname, renamed_dir, rar],
{})
if sfvcheck == -1:
logger.warning(
whoami() +
" error in crc32 check for file " + rar0)
pwdb.exc("db_msg_insert", [
nzbname, "error in crc32 check for file " +
rar0, "warning"
], {})
pwdb.exc("db_nzb_update_verify_status",
[nzbname, -2], {})
pwdb.exc("db_file_update_parstatus",
[f0_origname, -1], {})
child_pipe.send(True)
continue
if pwdb.exc("db_file_getparstatus", [rar0],
{}) == 0 and f0_renamedname != "N/A":
logger.debug(whoami() +
"no md5 check, copying " +
f0_renamedname.split("/")[-1] +
" to verified_rar dir")
pwdb.exc("db_msg_insert", [
nzbname, "no md5 check, copying " +
f0_renamedname.split("/")[-1] +
" to verified_rar dir", "info"
], {})
shutil.copy(renamed_dir + f0_renamedname,
verifiedrar_dir)
pwdb.exc("db_file_update_parstatus",
[f0_origname, 1], {})
# free rars or copy mode?
elif (pvmode == "verify" and not p2) or (pvmode == "copy"):
# maybe we can check via sfv file?
for file0full in glob.glob(renamed_dir +
"*") + glob.glob(renamed_dir +
".*"):
file0short = file0full.split("/")[-1]
ft = pwdb.exc("db_file_getftype_renamed", [file0short], {})
if ft == "rar":
rar0 = file0short
f0 = pwdb.exc("db_file_get_renamed", [rar0], {})
if not f0:
continue
f0_origname, f0_renamedname, f0_ftype = f0
sfvcheck = pwdb.exc("db_nzb_check_sfvcrc32",
[nzbname, renamed_dir, file0full],
{})
if sfvcheck == -1:
logger.warning(whoami() +
" error in crc32 check for file " +
rar0)
pwdb.exc("db_msg_insert", [
nzbname, "error in crc32 check for file " +
rar0, "warning"
], {})
pwdb.exc("db_nzb_update_verify_status",
[nzbname, -2], {})
pwdb.exc("db_file_update_parstatus",
[f0_origname, -1], {})
child_pipe.send(True)
continue
if pwdb.exc("db_file_getparstatus", [rar0],
{}) == 0 and f0_renamedname != "N/A":
logger.debug(whoami() + "copying " +
f0_renamedname.split("/")[-1] +
" to verified_rar dir")
pwdb.exc("db_msg_insert", [
nzbname,
"copying " + f0_renamedname.split("/")[-1] +
" to verified_rar dir", "info"
], {})
shutil.copy(renamed_dir + f0_renamedname,
verifiedrar_dir)
pwdb.exc("db_file_update_parstatus",
[f0_origname, 1], {})
allrarsverified, rvlist = pwdb.exc("db_only_verified_rars", [nzbname],
{})
if allrarsverified:
break
time.sleep(1)
if TERMINATED:
logger.info(whoami() + "terminated!")
sys.exit()
logger.debug(whoami() + "all rars are checked!")
corruptrars = pwdb.exc("get_all_corrupt_rar_files", [nzbname], {})
if not corruptrars:
logger.debug(whoami() +
"rar files ok, no repair needed, exiting par_verifier")
pwdb.exc("db_nzb_update_verify_status", [nzbname, 2], {})
elif p2list and corruptrars:
pwdb.exc("db_msg_insert", [nzbname, "repairing rar files", "info"], {})
logger.info(whoami() + "par2vol files present, repairing ...")
allok = True
allfound = True
corruptrars_1 = [c1 for c1, _ in corruptrars]
corruptrars_2 = [c2 for _, c2 in corruptrars]
for _, fnshort, fnlong, rarfiles in p2list:
rarf_match = [
rarf for rarf, _ in rarfiles
if rarf in corruptrars_1 or rarf in corruptrars_2
]
if len(rarf_match) == 0:
allfound = False
continue
lrar = str(len(rarfiles))
pwdb.exc("db_msg_insert",
[nzbname, "performing par2verify for " + fnshort, "info"],
{})
ssh = subprocess.Popen(['par2verify', fnlong],
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
sshres = ssh.stdout.readlines()
repair_is_required = False
repair_is_possible = False
for ss in sshres:
ss0 = ss.decode("utf-8")
if "Repair is required" in ss0:
repair_is_required = True
if "Repair is possible" in ss0:
repair_is_possible = True
if not repair_is_required:
pwdb.exc("db_msg_insert", [
nzbname, "par2verify for " + fnshort +
": repair is not required!", "info"
], {})
res0 = 1
elif repair_is_required and not repair_is_possible:
pwdb.exc("db_msg_insert", [
nzbname, "par2verify for " + fnshort +
": repair is required but not possible", "error"
], {})
res0 = -1
elif repair_is_required and repair_is_possible:
pwdb.exc("db_msg_insert", [
nzbname, "par2verify for " + fnshort +
": repair is required and possible, repairing files",
"info"
], {})
logger.info(
whoami() +
"repair is required and possible, performing par2repair")
# repair
ssh = subprocess.Popen(['par2repair', fnlong],
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
sshres = ssh.stdout.readlines()
repair_complete = False
for ss in sshres:
ss0 = ss.decode("utf-8")
if "Repair complete" in ss0:
repair_complete = True
if not repair_complete:
res0 = -1
else:
res0 = 1
# res0 = multipartrar_repair(renamed_dir, fnshort, pwdb, nzbname, logger)
else:
res0 = -1
if res0 != 1:
allok = False
logger.error(whoami() + "repair failed for " + lrar +
"rarfiles in " + fnshort)
pwdb.exc("db_msg_insert", [
nzbname, "rar file repair failed for " + lrar +
" rarfiles in " + fnshort + "!", "error"
], {})
else:
logger.info(whoami() + "repair success for " + lrar +
"rarfiles in " + fnshort)
pwdb.exc("db_msg_insert", [
nzbname, "rar file repair success for " + lrar +
" rarfiles in " + fnshort + "!", "info"
], {})
if not allfound:
allok = False
logger.error(
whoami() +
"cannot attempt one or more par2repairs due to missing par2 file(s)!"
)
pwdb.exc("db_msg_insert", [
nzbname,
"cannot attempt one or more par2repairs due to missing par2 file(s)!",
"error"
], {})
if allok:
logger.info(whoami() + "repair success")
pwdb.exc("db_nzb_update_verify_status", [nzbname, 2], {})
# copy all no yet copied rars to verifiedrar_dir
for c_origname, c_renamedname in corruptrars:
logger.info(whoami() + "copying " + c_renamedname +
" to verifiedrar_dir")
pwdb.exc("db_file_update_parstatus", [c_origname, 1], {})
pwdb.exc("db_file_update_status", [c_origname, 2], {})
shutil.copy(renamed_dir + c_renamedname, verifiedrar_dir)
else:
logger.error(whoami() + "repair failed!")
pwdb.exc("db_nzb_update_verify_status", [nzbname, -1], {})
for _, c_origname in corruptrars:
pwdb.exc("db_file_update_parstatus", [c_origname, -2], {})
else:
pwdb.exc("db_msg_insert", [
"nzbname", "rar file repair failed, no par files available",
"error"
], {})
logger.warning(
whoami() +
"some rars are corrupt but cannot repair (no par2 files)")
pwdb.exc("db_nzb_update_verify_status", [nzbname, -1], {})
logger.info(whoami() + "terminated!")
sys.exit()
