def _on_sigterm(self, num, stackframe):
try:
# self.logger.debug("GOT %s, %s FM worker killed! (pid = %s)" % (num, self.__class__.__name__, self.pid))
if self.pid is not None:
kill(self.pid, signal.SIGKILL, self.logger)
for p in self.processes:
try:
# no logs here during deadlock
kill(p.pid, signal.SIGKILL, self.logger)
except OSError:
pass
except Exception:
# self.logger.error("Error on_sigterm() %s , error %s" % (str(e), traceback.format_exc()))
sys.exit(1)
def _on_sigterm(self, num, stackframe):
try:
# self.logger.debug("GOT %s, %s FM worker killed! (pid = %s)" % (num, self.__class__.__name__, self.pid))
if self.pid is not None:
kill(self.pid, signal.SIGKILL, self.logger)
for p in self.processes:
try:
# no logs here during deadlock
kill(p.pid, signal.SIGKILL, self.logger)
except OSError:
pass
except Exception:
# self.logger.error("Error on_sigterm() %s , error %s" % (str(e), traceback.format_exc()))
sys.exit(1)
def list(self, path):
flist = {
"path": path,
"items": []
}
try:
self.webdavClient.check('/')
except Exception:
raise Exception("Error during establishing webdav connection")
listdir = self.webdavClient.list(self.to_byte(path))
self.logger.info("listdir=%s", listdir)
time_limit = int(time.time()) + TIMEOUT_LIMIT
self.file_queue = JoinableQueue(maxsize=0)
self.result_queue = Queue(maxsize=0)
for i in range(self.NUM_WORKING_PROCESSES):
p = Process(target=self._make_file_info, args=(self.file_queue, self.result_queue, self.logger, time_limit))
p.start()
proc = psutil.Process(p.pid)
proc.ionice(psutil.IOPRIO_CLASS_IDLE)
proc.nice(20)
self.logger.debug(
"ListDir worker #%s, set ionice = idle and nice = 20 for pid %s" % (
str(i), str(p.pid)))
self.processes.append(p)
for name in listdir:
try:
item_path = '{0}/{1}'.format(path, name)
self.file_queue.put(item_path)
except UnicodeDecodeError as e:
self.logger.error(
"UnicodeDecodeError %s, %s" % (str(e), traceback.format_exc()))
except IOError as e:
self.logger.error("IOError %s, %s" % (str(e), traceback.format_exc()))
except Exception as e:
self.logger.error(
"Exception %s, %s" % (str(e), traceback.format_exc()))
while not self.file_queue.empty():
self.logger.debug("file_queue size = %s , empty = %s (timeout: %s/%s)" % (
self.file_queue.qsize(), self.file_queue.empty(), str(int(time.time())), time_limit))
time.sleep(REQUEST_DELAY)
if self.file_queue.empty():
self.logger.debug("join() file_queue until workers done jobs")
self.file_queue.join()
for p in self.processes:
try:
self.logger.debug("WebDav ListDir terminate worker process, pid = %s" % p.pid)
kill(p.pid, signal.SIGKILL, self.logger)
except OSError:
self.logger.error(
"ListDir unable to terminate worker process, pid = %s" % p.pid)
if self.is_alive['status'] is True:
while not self.result_queue.empty():
file_info = self.result_queue.get()
flist["items"].append(file_info)
return flist
def run(self):
try:
self.preload()
sftp = self.get_sftp_connection(self.session)
self.logger.debug("findText started with timeout = %s" % TIMEOUT_LIMIT)
time_limit = int(time.time()) + TIMEOUT_LIMIT
# Launches a number of worker threads to perform operations using the queue of inputs
sftp_managers = []
for i in range(self.NUM_WORKING_PROCESSES):
p = Process(target=self.worker,
args=(self.re_text, self.file_queue, self.result_queue, time_limit))
p.start()
proc = psutil.Process(p.pid)
proc.ionice(psutil.IOPRIO_CLASS_IDLE)
proc.nice(20)
self.logger.debug(
"Search worker #%s, set ionice = idle and nice = 20 for pid %s" % (
str(i), str(p.pid)))
self.processes.append(p)
abs_path = self.path
self.logger.debug("FM FindText worker run(), abs_path = %s" % abs_path)
if not sftp.exists(abs_path):
raise Exception("Provided path not exist")
self.on_running(self.status_id, pid=self.pid, pname=self.name)
for current, dirs, files in sftp.walk(abs_path):
for f in files:
try:
file_path = os.path.join(current, f)
self.file_queue.put(file_path)
except UnicodeDecodeError as e:
self.logger.error(
"UnicodeDecodeError %s, %s" % (str(e), traceback.format_exc()))
except IOError as e:
self.logger.error("IOError %s, %s" % (str(e), traceback.format_exc()))
except Exception as e:
self.logger.error(
"Exception %s, %s" % (str(e), traceback.format_exc()))
while int(time.time()) <= time_limit:
self.logger.debug("file_queue size = %s , empty = %s (timeout: %s/%s)" % (
self.file_queue.qsize(), self.file_queue.empty(), str(int(time.time())), time_limit))
if self.file_queue.empty():
self.logger.debug("join() file_queue until workers done jobs")
self.file_queue.join()
break
else:
time.sleep(REQUEST_DELAY)
if int(time.time()) > time_limit:
self.is_alive['status'] = False
for sftp in sftp_managers:
sftp.conn.close()
for p in self.processes:
try:
self.logger.debug("FM FindText terminate worker process, pid = %s" % p.pid)
kill(p.pid, signal.SIGKILL, self.logger)
except OSError:
self.logger.error(
"FindText unable to terminate worker process, pid = %s" % p.pid)
if self.is_alive['status'] is True:
while not self.result_queue.empty():
file_path = self.result_queue.get()
self.result.append(sftp.make_file_info(file_path))
self.on_success(self.status_id, data=self.result, pid=self.pid, pname=self.name)
else:
result = {
"error": True,
"message": "Operation timeout exceeded",
"traceback": ""
}
self.on_error(self.status_id, result, pid=self.pid, pname=self.name)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(self.status_id, result, pid=self.pid, pname=self.name)
def run(self):
try:
self.preload()
except Exception as e:
result = {"error": True, "message": str(e), "traceback": traceback.format_exc()}
self.on_error(self.status_id, result, pid=self.pid, pname=self.name)
return
def worker(re_text, file_queue, result_queue, logger, timeout):
while int(time.time()) < timeout:
if file_queue.empty() is not True:
f_path = file_queue.get()
try:
if not is_binary(f_path):
mime = mimetypes.guess_type(f_path)[0]
# исключаем некоторые mime типы из поиска
if mime not in ["application/pdf", "application/rar"]:
with open(f_path, "rb") as fp:
for line in fp:
try:
line = as_unicode(line)
except UnicodeDecodeError:
charset = chardet.detect(line)
if charset.get("encoding") in ["MacCyrillic"]:
detected = "windows-1251"
else:
detected = charset.get("encoding")
if detected is None:
break
try:
line = str(line, detected, "replace")
except LookupError:
pass
if re_text.match(line) is not None:
result_queue.put(f_path)
# logger.debug("matched file = %s " % f_path)
break
except UnicodeDecodeError as unicode_e:
logger.error("UnicodeDecodeError %s, %s" % (str(unicode_e), traceback.format_exc()))
except IOError as io_e:
logger.error("IOError %s, %s" % (str(io_e), traceback.format_exc()))
except Exception as other_e:
logger.error("Exception %s, %s" % (str(other_e), traceback.format_exc()))
finally:
file_queue.task_done()
else:
time.sleep(REQUEST_DELAY)
try:
self.logger.debug("findText started with timeout = %s" % TIMEOUT_LIMIT)
time_limit = int(time.time()) + TIMEOUT_LIMIT
# Launches a number of worker threads to perform operations using the queue of inputs
for i in range(self.NUM_WORKING_PROCESSES):
p = Process(
target=worker, args=(self.re_text, self.file_queue, self.result_queue, self.logger, time_limit)
)
p.start()
proc = psutil.Process(p.pid)
proc.ionice(psutil.IOPRIO_CLASS_IDLE)
proc.nice(20)
self.logger.debug(
"Search worker #%s, set ionice = idle and nice = 20 for pid %s" % (str(i), str(p.pid))
)
self.processes.append(p)
abs_path = self.get_abs_path(self.path)
self.logger.debug("FM FindText worker run(), abs_path = %s" % abs_path)
if not os.path.exists(abs_path):
raise Exception("Provided path not exist")
self.on_running(self.status_id, pid=self.pid, pname=self.name)
for current, dirs, files in os.walk(abs_path):
for f in files:
try:
file_path = os.path.join(current, f)
self.file_queue.put(file_path)
except UnicodeDecodeError as e:
self.logger.error("UnicodeDecodeError %s, %s" % (str(e), traceback.format_exc()))
except IOError as e:
self.logger.error("IOError %s, %s" % (str(e), traceback.format_exc()))
except Exception as e:
self.logger.error("Exception %s, %s" % (str(e), traceback.format_exc()))
while int(time.time()) <= time_limit:
self.logger.debug(
"file_queue size = %s , empty = %s (timeout: %s/%s)"
% (self.file_queue.qsize(), self.file_queue.empty(), str(int(time.time())), time_limit)
)
if self.file_queue.empty():
self.logger.debug("join() file_queue until workers done jobs")
self.file_queue.join()
break
else:
time.sleep(REQUEST_DELAY)
if int(time.time()) > time_limit:
self.is_alive["status"] = False
for p in self.processes:
try:
self.logger.debug("FM FindText terminate worker process, pid = %s" % p.pid)
kill(p.pid, signal.SIGKILL, self.logger)
except OSError:
self.logger.error("FindText unable to terminate worker process, pid = %s" % p.pid)
if self.is_alive["status"] is True:
while not self.result_queue.empty():
file_path = self.result_queue.get()
self.result.append(self._make_file_info(file_path))
self.on_success(self.status_id, data=self.result, pid=self.pid, pname=self.name)
else:
result = {"error": True, "message": "Operation timeout exceeded", "traceback": ""}
self.on_error(self.status_id, result, pid=self.pid, pname=self.name)
except Exception as e:
result = {"error": True, "message": str(e), "traceback": traceback.format_exc()}
self.on_error(self.status_id, result, pid=self.pid, pname=self.name)
def run(self):
try:
self.preload()
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(self.status_id,
result,
pid=self.pid,
pname=self.name)
return
def worker(re_text, file_queue, result_queue, logger, timeout):
while int(time.time()) < timeout:
if file_queue.empty() is not True:
f_path = file_queue.get()
try:
if not is_binary(f_path):
mime = mimetypes.guess_type(f_path)[0]
# исключаем некоторые mime типы из поиска
if mime not in [
'application/pdf', 'application/rar'
]:
with open(f_path, 'rb') as fp:
for line in fp:
try:
line = as_unicode(line)
except UnicodeDecodeError:
charset = chardet.detect(line)
if charset.get('encoding') in [
'MacCyrillic'
]:
detected = 'windows-1251'
else:
detected = charset.get(
'encoding')
if detected is None:
break
try:
line = str(
line, detected, "replace")
except LookupError:
pass
if re_text.match(line) is not None:
result_queue.put(f_path)
# logger.debug("matched file = %s " % f_path)
break
except UnicodeDecodeError as unicode_e:
logger.error("UnicodeDecodeError %s, %s" %
(str(unicode_e), traceback.format_exc()))
except IOError as io_e:
logger.error("IOError %s, %s" %
(str(io_e), traceback.format_exc()))
except Exception as other_e:
logger.error("Exception %s, %s" %
(str(other_e), traceback.format_exc()))
finally:
file_queue.task_done()
else:
time.sleep(REQUEST_DELAY)
try:
self.logger.debug("findText started with timeout = %s" %
TIMEOUT_LIMIT)
time_limit = int(time.time()) + TIMEOUT_LIMIT
# Launches a number of worker threads to perform operations using the queue of inputs
for i in range(self.NUM_WORKING_PROCESSES):
p = Process(target=worker,
args=(self.re_text, self.file_queue,
self.result_queue, self.logger, time_limit))
p.start()
proc = psutil.Process(p.pid)
proc.ionice(psutil.IOPRIO_CLASS_IDLE)
proc.nice(20)
self.logger.debug(
"Search worker #%s, set ionice = idle and nice = 20 for pid %s"
% (str(i), str(p.pid)))
self.processes.append(p)
abs_path = self.get_abs_path(self.path)
self.logger.debug("FM FindText worker run(), abs_path = %s" %
abs_path)
if not os.path.exists(abs_path):
raise Exception("Provided path not exist")
self.on_running(self.status_id, pid=self.pid, pname=self.name)
for current, dirs, files in os.walk(abs_path):
for f in files:
try:
file_path = os.path.join(current, f)
self.file_queue.put(file_path)
except UnicodeDecodeError as e:
self.logger.error("UnicodeDecodeError %s, %s" %
(str(e), traceback.format_exc()))
except IOError as e:
self.logger.error("IOError %s, %s" %
(str(e), traceback.format_exc()))
except Exception as e:
self.logger.error("Exception %s, %s" %
(str(e), traceback.format_exc()))
while int(time.time()) <= time_limit:
self.logger.debug(
"file_queue size = %s , empty = %s (timeout: %s/%s)" %
(self.file_queue.qsize(), self.file_queue.empty(),
str(int(time.time())), time_limit))
if self.file_queue.empty():
self.logger.debug(
"join() file_queue until workers done jobs")
self.file_queue.join()
break
else:
time.sleep(REQUEST_DELAY)
if int(time.time()) > time_limit:
self.is_alive['status'] = False
for p in self.processes:
try:
self.logger.debug(
"FM FindText terminate worker process, pid = %s" %
p.pid)
kill(p.pid, signal.SIGKILL, self.logger)
except OSError:
self.logger.error(
"FindText unable to terminate worker process, pid = %s"
% p.pid)
if self.is_alive['status'] is True:
while not self.result_queue.empty():
file_path = self.result_queue.get()
self.result.append(self._make_file_info(file_path))
self.on_success(self.status_id,
data=self.result,
pid=self.pid,
pname=self.name)
else:
result = {
"error": True,
"message": "Operation timeout exceeded",
"traceback": ""
}
self.on_error(self.status_id,
result,
pid=self.pid,
pname=self.name)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(self.status_id,
result,
pid=self.pid,
pname=self.name)
def run(self):
try:
self.preload()
sftp = self.get_sftp_connection(self.session)
self.logger.debug("findText started with timeout = %s" % TIMEOUT_LIMIT)
time_limit = int(time.time()) + TIMEOUT_LIMIT
# Launches a number of worker threads to perform operations using the queue of inputs
sftp_managers = []
for i in range(self.NUM_WORKING_PROCESSES):
p = Process(target=self.worker,
args=(self.re_text, self.file_queue, self.result_queue, time_limit))
p.start()
proc = psutil.Process(p.pid)
proc.ionice(psutil.IOPRIO_CLASS_IDLE)
proc.nice(20)
self.logger.debug(
"Search worker #%s, set ionice = idle and nice = 20 for pid %s" % (
str(i), str(p.pid)))
self.processes.append(p)
abs_path = self.path
self.logger.debug("FM FindText worker run(), abs_path = %s" % abs_path)
if not sftp.exists(abs_path):
raise Exception("Provided path not exist")
self.on_running(self.status_id, pid=self.pid, pname=self.name)
for current, dirs, files in sftp.walk(abs_path):
for f in files:
try:
file_path = os.path.join(current, f)
self.file_queue.put(file_path)
except UnicodeDecodeError as e:
self.logger.error(
"UnicodeDecodeError %s, %s" % (str(e), traceback.format_exc()))
except IOError as e:
self.logger.error("IOError %s, %s" % (str(e), traceback.format_exc()))
except Exception as e:
self.logger.error(
"Exception %s, %s" % (str(e), traceback.format_exc()))
while int(time.time()) <= time_limit:
self.logger.debug("file_queue size = %s , empty = %s (timeout: %s/%s)" % (
self.file_queue.qsize(), self.file_queue.empty(), str(int(time.time())), time_limit))
if self.file_queue.empty():
self.logger.debug("join() file_queue until workers done jobs")
self.file_queue.join()
break
else:
time.sleep(REQUEST_DELAY)
if int(time.time()) > time_limit:
self.is_alive['status'] = False
for sftp in sftp_managers:
sftp.conn.close()
for p in self.processes:
try:
self.logger.debug("FM FindText terminate worker process, pid = %s" % p.pid)
kill(p.pid, signal.SIGKILL, self.logger)
except OSError:
self.logger.error(
"FindText unable to terminate worker process, pid = %s" % p.pid)
if self.is_alive['status'] is True:
while not self.result_queue.empty():
file_path = self.result_queue.get()
self.result.append(sftp.make_file_info(file_path))
self.on_success(self.status_id, data=self.result, pid=self.pid, pname=self.name)
else:
result = {
"error": True,
"message": "Operation timeout exceeded",
"traceback": ""
}
self.on_error(self.status_id, result, pid=self.pid, pname=self.name)
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(self.status_id, result, pid=self.pid, pname=self.name)
def run(self):
p = pam.pam()
if not p.authenticate(self.login, self.password):
raise Exception('Not Authenticated')
self.logger.info("CancelOperation process started PID = %s",
str(self.pid))
try:
if self.operation_pid is None:
self.on_error({
"error": True,
"message": "Operation pid not provided",
"status": False
})
return
if self.operation_pname is None:
self.on_error({
"error": True,
"message": "Operation pname not provided",
"status": False
})
return
try:
proc = psutil.Process(self.operation_pid)
self.logger.info("PROC!!!!!!!!!!!! %s", proc)
self.logger.info("Process object = %s name = %s , cmd = %s" %
(pprint.pformat(proc), proc.name(),
pprint.pformat(proc.cmdline())))
self.logger.info("check = %s , (%s)" %
(str(self.operation_pname in proc.cmdline()),
self.operation_pname))
if self.operation_pname in str(proc.cmdline()):
self.logger.info('==== MATCHED ====')
kill(self.operation_pid, signal.SIGTERM, self.logger)
self.on_success({"status": True})
return
except psutil.NoSuchProcess:
self.on_error({
"error": True,
"message": "process not found",
"status": False
})
return
except Exception as e:
self.on_error({
"error":
True,
"message":
"%s %s" % (str(e), traceback.format_exc()),
"status":
False
})
self.on_error({
"error": True,
"message": "process not killed",
"status": False
})
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(result)
def run(self):
p = pam.pam()
if not p.authenticate(self.login, self.password):
raise Exception('Not Authenticated')
self.logger.info("CancelOperation process started PID = %s", str(self.pid))
try:
if self.operation_pid is None:
self.on_error({
"error": True,
"message": "Operation pid not provided",
"status": False
})
return
if self.operation_pname is None:
self.on_error({
"error": True,
"message": "Operation pname not provided",
"status": False
})
return
try:
proc = psutil.Process(self.operation_pid)
self.logger.info("PROC!!!!!!!!!!!! %s", proc)
self.logger.info(
"Process object = %s name = %s , cmd = %s" % (pprint.pformat(proc),
proc.name(), pprint.pformat(proc.cmdline())))
self.logger.info(
"check = %s , (%s)" % (str(self.operation_pname in proc.cmdline()), self.operation_pname))
if self.operation_pname in str(proc.cmdline()):
self.logger.info('==== MATCHED ====')
kill(self.operation_pid, signal.SIGTERM, self.logger)
self.on_success({
"status": True
})
return
except psutil.NoSuchProcess:
self.on_error({
"error": True,
"message": "process not found",
"status": False
})
return
except Exception as e:
self.on_error({
"error": True,
"message": "%s %s" % (str(e), traceback.format_exc()),
"status": False
})
self.on_error({
"error": True,
"message": "process not killed",
"status": False
})
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.on_error(result)
