def run(self):
try:
if self.logger:
self.logger.debug("%s : execute command %s" % (as_unicode(self.log_prefix), as_unicode(self.command)))
except:
# на случай если в комманде возникнет UNICODE/DECODE error
# может быть в случае передачи русских символов например в пути
if self.logger:
self.logger.error("%s : Error when write log" % (as_unicode(self.log_prefix)))
command = [as_default_string(item) for item in self.command]
self.process = subprocess.Popen(command, **self.process_options)
def wait(self, extended_return=False, write_output_in_log=True):
out, err = self.process.communicate()
try:
if err != "":
if self.logger:
self.logger.error("%s : Error: %s" % (as_unicode(self.log_prefix), as_unicode(err)))
if write_output_in_log and self.logger:
self.logger.debug("%s : command output: %s" % (as_unicode(self.log_prefix), as_unicode(out)))
except:
if self.logger:
self.logger.error("%s : Error when write log" % (as_unicode(self.log_prefix)))
return (out, err, self.process.returncode) if extended_return else out
def get_util(name):
command = ["/bin/which", name]
env = {"PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"}
p = SubprocessRunner(command=command, env=env)
p.run()
return as_unicode(p.wait()).rstrip("\n")
def worker(self, re_text, file_queue, result_queue, timeout):
try:
worker_sftp = self.get_sftp_connection(self.session)
while int(time.time()) < timeout:
if file_queue.empty() is not True:
f_path = file_queue.get()
try:
if not worker_sftp.is_binary(f_path):
mime = mimetypes.guess_type(f_path)[0]
# исключаем некоторые mime типы из поиска
if mime not in ['application/pdf', 'application/rar']:
with worker_sftp.open(f_path, 'rb') as fp:
for line in fp:
try:
line = as_unicode(line)
except UnicodeDecodeError:
charset = chardet.detect(line)
if charset.get('encoding') in ['MacCyrillic']:
detected = 'windows-1251'
else:
detected = charset.get('encoding')
if detected is None:
break
try:
line = str(line, detected, "replace")
except LookupError:
pass
if re_text.match(line) is not None:
result_queue.put(f_path)
self.logger.debug("matched file = %s " % f_path)
break
except UnicodeDecodeError as unicode_e:
self.logger.error(
"UnicodeDecodeError %s, %s" % (str(unicode_e), traceback.format_exc()))
except IOError as io_e:
self.logger.error("IOError %s, %s" % (str(io_e), traceback.format_exc()))
except Exception as other_e:
self.logger.error("Exception %s, %s" % (str(other_e), traceback.format_exc()))
finally:
file_queue.task_done()
else:
time.sleep(REQUEST_DELAY)
worker_sftp.close()
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.logger.error('SFTP FindText Worker Exception {}'.format(result))
def worker(self, re_text, file_queue, result_queue, timeout):
try:
worker_sftp = self.get_sftp_connection(self.session)
while int(time.time()) < timeout:
if file_queue.empty() is not True:
f_path = file_queue.get()
try:
if not worker_sftp.is_binary(f_path):
mime = mimetypes.guess_type(f_path)[0]
# исключаем некоторые mime типы из поиска
if mime not in ['application/pdf', 'application/rar']:
with worker_sftp.open(f_path, 'rb') as fp:
for line in fp:
try:
line = as_unicode(line)
except UnicodeDecodeError:
charset = chardet.detect(line)
if charset.get('encoding') in ['MacCyrillic']:
detected = 'windows-1251'
else:
detected = charset.get('encoding')
if detected is None:
break
try:
line = str(line, detected, "replace")
except LookupError:
pass
if re_text.match(line) is not None:
result_queue.put(f_path)
self.logger.debug("matched file = %s " % f_path)
break
except UnicodeDecodeError as unicode_e:
self.logger.error(
"UnicodeDecodeError %s, %s" % (str(unicode_e), traceback.format_exc()))
except IOError as io_e:
self.logger.error("IOError %s, %s" % (str(io_e), traceback.format_exc()))
except Exception as other_e:
self.logger.error("Exception %s, %s" % (str(other_e), traceback.format_exc()))
finally:
file_queue.task_done()
else:
time.sleep(REQUEST_DELAY)
worker_sftp.close()
except Exception as e:
result = {
"error": True,
"message": str(e),
"traceback": traceback.format_exc()
}
self.logger.error('SFTP FindText Worker Exception {}'.format(result))
def worker(re_text, file_queue, result_queue, logger, timeout):
while int(time.time()) < timeout:
if file_queue.empty() is not True:
f_path = file_queue.get()
try:
if not is_binary(f_path):
mime = mimetypes.guess_type(f_path)[0]
# исключаем некоторые mime типы из поиска
if mime not in ["application/pdf", "application/rar"]:
with open(f_path, "rb") as fp:
for line in fp:
try:
line = as_unicode(line)
except UnicodeDecodeError:
charset = chardet.detect(line)
if charset.get("encoding") in ["MacCyrillic"]:
detected = "windows-1251"
else:
detected = charset.get("encoding")
if detected is None:
break
try:
line = str(line, detected, "replace")
except LookupError:
pass
if re_text.match(line) is not None:
result_queue.put(f_path)
# logger.debug("matched file = %s " % f_path)
break
except UnicodeDecodeError as unicode_e:
logger.error("UnicodeDecodeError %s, %s" % (str(unicode_e), traceback.format_exc()))
except IOError as io_e:
logger.error("IOError %s, %s" % (str(io_e), traceback.format_exc()))
except Exception as other_e:
logger.error("Exception %s, %s" % (str(other_e), traceback.format_exc()))
finally:
file_queue.task_done()
else:
time.sleep(REQUEST_DELAY)
def enqueue_errors(err, queue):
for line in iter(err.readline, ""):
queue.put(as_unicode(line).rstrip('\n'))
err.close()
def enqueue_output(out, queue):
for line in iter(out.readline, ""):
queue.put(as_unicode(line).rstrip("\n"))
out.close()
def iterate(self):
try:
if self.logger:
self.logger.debug("%s : iterate command %s" % (as_unicode(self.log_prefix), as_unicode(self.command)))
except Exception as e:
# на случай если в комманде возникнет UNICODE/DECODE error
# может быть в случае передачи русских символов например в пути
if self.logger:
self.logger.error("%s : Error when write log: %s" % (as_unicode(self.log_prefix), str(e)))
def enqueue_output(out, queue):
for line in iter(out.readline, ""):
queue.put(as_unicode(line).rstrip("\n"))
out.close()
def enqueue_errors(err, queue):
for line in iter(err.readline, ""):
queue.put(as_unicode(line).rstrip('\n'))
err.close()
command = [as_default_string(item) for item in self.command]
self.process = subprocess.Popen(command, **self.process_options)
q_out = Queue()
q_err = Queue()
t_out = Thread(target=enqueue_output, args=(self.process.stdout, q_out))
t_out.daemon = True
t_out.start()
t_err = Thread(target=enqueue_errors, args=(self.process.stderr, q_err))
t_err.daemon = True
t_err.start()
while True:
if not q_err.empty():
err_output = q_err.get()
else:
err_output = ""
if err_output != "":
if self.logger:
self.logger.error("%s : Error: %s" % (as_unicode(self.log_prefix), as_unicode(err_output)))
raise Exception(err_output)
if not q_out.empty():
line_output = q_out.get()
else:
line_output = ""
code = self.process.poll()
if self.logger and line_output != "":
try:
self.logger.debug(
"%s : command iterate: %s" % (as_unicode(self.log_prefix), as_unicode(line_output)))
except Exception as e:
# на случай если в комманде возникнет UNICODE/DECODE error
# может быть в случае передачи русских символов например в пути
self.logger.error(
"%s : Error when write command iterate log: %s" % (as_unicode(self.log_prefix), str(e)))
if line_output == "":
if code is not None:
self.logger.debug("%s : command iterate end" % as_unicode(self.log_prefix))
break
yield line_output