def _set_dates(self, netloc, port):
"""
set dates from openssl (notAfter and not Before)
:param netloc: target netloc
:param port: target port
:return:
"""
with sp_Popen([self.echo_cmd], stdout=SP_PIPE) as proc_echo:
cmd = self.ssl_client_cmd[:]
cmd.append('{}:{}'.format(shlex_quote(netloc), str(port)))
with sp_Popen(cmd,
stdin=proc_echo.stdout,
stdout=SP_PIPE,
stderr=SP_DEVNULL) as proc_ssl:
cmd = self.ssl_x509_cmd[:]
cmd.append('-noout')
cmd.append('-dates')
with sp_Popen(cmd,
stdin=proc_ssl.stdout,
stdout=SP_PIPE,
stderr=SP_DEVNULL) as proc_x509:
data = proc_x509.stdout.read()
ddata = data.decode('utf-8')
if ddata is not None:
reg = re_compile('notBefore=([^\n]+)')
res = reg.search(ddata)
if res is not None:
self.notBefore = datetime.fromtimestamp(
cert_time_to_seconds(res.group(1)))
reg = re_compile('notAfter=([^\n]+)')
res = reg.search(ddata)
if res is not None:
self.notAfter = datetime.fromtimestamp(
cert_time_to_seconds(res.group(1)))
def _set_serial(self, netloc, port):
"""
set serial from openssl
:param netloc: target netloc
:param port: target port
:return:
"""
with sp_Popen([self.echo_cmd], stdout=SP_PIPE) as proc_echo:
cmd = self.ssl_client_cmd[:]
cmd.append('{}:{}'.format(shlex_quote(netloc), str(port)))
with sp_Popen(cmd,
stdin=proc_echo.stdout,
stdout=SP_PIPE,
stderr=SP_DEVNULL) as proc_ssl:
cmd = self.ssl_x509_cmd[:]
cmd.append('-noout')
cmd.append('-serial')
with sp_Popen(cmd,
stdin=proc_ssl.stdout,
stdout=SP_PIPE,
stderr=SP_DEVNULL) as proc_x509:
data = proc_x509.stdout.read()
ddata = data.decode('utf-8')
if ddata is not None:
reg = re_compile('serial=([^\n]+)')
res = reg.search(ddata)
if res is not None:
self.serialNumber = res.group(1)
def _set_issuer(self, netloc, port):
"""
set issuer dict from openssl
:param netloc: target netloc
:param port: target port
:return:
"""
with sp_Popen([self.echo_cmd], stdout=SP_PIPE) as proc_echo:
cmd = self.ssl_client_cmd[:]
cmd.append('{}:{}'.format(shlex_quote(netloc), str(port)))
with sp_Popen(cmd,
stdin=proc_echo.stdout,
stdout=SP_PIPE,
stderr=SP_DEVNULL) as proc_ssl:
cmd = self.ssl_x509_cmd[:]
cmd.append('-noout')
cmd.append('-issuer')
with sp_Popen(cmd,
stdin=proc_ssl.stdout,
stdout=SP_PIPE,
stderr=SP_DEVNULL) as proc_x509:
data = proc_x509.stdout.read()
ddata = data.decode('utf-8')
if ddata is not None:
issuer_dict = dict()
reg = re_compile('[A-Z]+=[^/\n]+')
for res in reg.finditer(ddata):
_split = res.group().split('=')
if str(_split[0]) in self.keys:
issuer_dict[self.keys[str(
_split[0])]] = _split[1]
self.issuer = issuer_dict
def _set_error_message(self, netloc, port):
"""
set basic error from openssl (certificate not valid)
:param netloc: target netloc
:param port: target port
:return:
"""
with sp_Popen([self.echo_cmd], stdout=SP_PIPE) as proc_echo:
cmd = self.ssl_client_cmd[:]
cmd.append('{}:{}'.format(shlex_quote(netloc), str(port)))
cmd.append('-CAfile')
cmd.append(CA_CERTS)
with sp_Popen(cmd,
stdin=proc_echo.stdout,
stdout=SP_DEVNULL,
stderr=SP_PIPE) as proc_ssl:
data = proc_ssl.stderr.read()
ddata = data.decode('utf-8')
if ddata is not None:
reg = re_compile('error:num=[0-9]+:([^\n]+)')
for res in reg.finditer(ddata):
# expired checked at the end
# print('error msg: {}'.format(res.group(0)))
if 'expired' not in res.group(1):
self.add_error(err=NameError(res.group(1)))
def _set_other_params(self, netloc, port):
"""
set other params from openssl (OCSP, CRL, caIssuers)
:param netloc: target netloc
:param port: target port
:return:
"""
with sp_Popen([self.echo_cmd], stdout=SP_PIPE) as proc_echo:
cmd = self.ssl_client_cmd[:]
cmd.append('{}:{}'.format(shlex_quote(netloc), str(port)))
with sp_Popen(cmd,
stdin=proc_echo.stdout,
stdout=SP_PIPE,
stderr=SP_DEVNULL) as proc_ssl:
cmd = self.ssl_x509_cmd[:]
cmd.append('-noout')
cmd.append('-text')
with sp_Popen(cmd,
stdin=proc_ssl.stdout,
stdout=SP_PIPE,
stderr=SP_DEVNULL) as proc_x509:
data = proc_x509.stdout.read()
ddata = data.decode('utf-8')
if ddata is not None:
reg = re_compile('Version: ([0-9]+)')
res = reg.search(ddata)
if res is not None:
self.version = int(res.group(1))
reg = re_compile('OCSP - URI:([^\n]+)')
res = reg.search(ddata)
if res is not None:
self.OCSP = res.group(1),
reg = re_compile('CA Issuers - URI:([^\n]+)')
res = reg.search(ddata)
if res is not None:
self.caIssuers = res.group(1),
reg = re_compile(
'CRL Distribution Points:([ \n]*Full Name:[ \n]*URI:[^\n]+)+'
)
res = reg.search(ddata)
if res is not None:
crls = list()
_str = res.group(0)
reg = re_compile('URI:([^\n]+)')
for res in reg.finditer(_str):
crls.append(res.group(1))
self.crlDistributionPoints = tuple(crls)
def set_pem_cert(self, netloc, port):
"""
set pem cert from openssl
:param netloc: target netloc
:param port: target port
:return:
"""
with sp_Popen([self.echo_cmd], stdout=SP_PIPE) as proc_echo:
cmd = self.ssl_client_cmd[:]
cmd.append('{}:{}'.format(shlex_quote(netloc), str(port)))
with sp_Popen(cmd,
stdin=proc_echo.stdout,
stdout=SP_PIPE,
stderr=SP_DEVNULL) as proc_ssl:
data = proc_ssl.stdout.read()
ddata = data.decode('utf-8')
if ddata is not None:
reg = re_compile(
'-+BEGIN CERTIFICATE-+[^-]+-+END CERTIFICATE-+\n')
res = reg.search(ddata)
if res is not None:
self.pem_cert = res.group(0)
def _set_subject_alt_name(self, netloc, port):
"""
set subject alt name dict from openssl (authorized DNS)
:param netloc: target netloc
:param port: target port
:return:
"""
with sp_Popen([self.echo_cmd], stdout=SP_PIPE) as proc_echo:
cmd = self.ssl_client_cmd[:]
cmd.append('{}:{}'.format(shlex_quote(netloc), str(port)))
with sp_Popen(cmd,
stdin=proc_echo.stdout,
stdout=SP_PIPE,
stderr=SP_DEVNULL) as proc_ssl:
cmd = self.ssl_x509_cmd[:]
cmd.append('-noout')
cmd.append('-text')
with sp_Popen(cmd,
stdin=proc_ssl.stdout,
stdout=SP_PIPE,
stderr=SP_DEVNULL) as proc_x509:
data = proc_x509.stdout.read()
ddata = data.decode('utf-8')
if ddata is not None:
san_dict = dict()
reg = re_compile('DNS:[^,\n]+')
idx = 0
for res in reg.finditer(ddata):
if idx > 0:
key = 'DNS{}{}'.format(self._separator,
str(idx))
else:
key = 'DNS'
san_dict[key] = res.group().split(':')[1]
idx += 1
self.subjectAltName = san_dict
def ssh((hostname, command, ssh_timeout, realtime, multiline)):
"""Takes a tuple of arguments as multiprocessing map only accepts one and
returns a result dictionary."""
result = {
"returncode": None,
"stdout": None,
"stderr": None,
"command": "%s %s %s" % (SSH_CMD % ssh_timeout, hostname, command),
"hostname": hostname,
"status": "pending",
}
# Run in a try/catch to check for SIGINT, this will kill the running process
# then set the global terminating event preventing subsequent workers from
# running.
try:
if terminating.is_set():
# If we are terminating, update the result set and return.
result["returncode"] = 2
result["status"] = "aborted"
else:
# Start subprocess for ssh. shell=True is nasty, but this is needed.
process = sp_Popen("%s %s %s" %
(SSH_CMD % ssh_timeout, hostname, command),
stdout=sp_PIPE,
stderr=sp_PIPE,
shell=True)
stdout, stderr = process.communicate()
if multiline:
result["stdout"] = stdout.strip("\n")
result["stderr"] = stderr.strip("\n")
else:
result["stdout"] = stdout.replace("\n", " ")
result["stderr"] = stderr.replace("\n", " ")
result["returncode"] = process.returncode
result["status"] = "done"
except KeyboardInterrupt:
# Catch keyboard interrupt, killing the current worker and setting the
# global terminating event.
result["returncode"] = 1
result["status"] = "killed"
terminating.set()
process.kill()
return result
def ssh((hostname, command, ssh_timeout, realtime, multiline)):
"""Takes a tuple of arguments as multiprocessing map only accepts one and
returns a result dictionary."""
result = {
"returncode": None,
"stdout": None,
"stderr": None,
"command": "%s %s %s" % (SSH_CMD % ssh_timeout, hostname, command),
"hostname": hostname,
"status": "pending",
}
# Run in a try/catch to check for SIGINT, this will kill the running process
# then set the global terminating event preventing subsequent workers from
# running.
try:
if terminating.is_set():
# If we are terminating, update the result set and return.
result["returncode"] = 2
result["status"] = "aborted"
else:
# Start subprocess for ssh. shell=True is nasty, but this is needed.
process = sp_Popen(
"%s %s %s" % (SSH_CMD % ssh_timeout, hostname, command), stdout=sp_PIPE, stderr=sp_PIPE, shell=True
)
stdout, stderr = process.communicate()
if multiline:
result["stdout"] = stdout.strip("\n")
result["stderr"] = stderr.strip("\n")
else:
result["stdout"] = stdout.replace("\n", " ")
result["stderr"] = stderr.replace("\n", " ")
result["returncode"] = process.returncode
result["status"] = "done"
except KeyboardInterrupt:
# Catch keyboard interrupt, killing the current worker and setting the
# global terminating event.
result["returncode"] = 1
result["status"] = "killed"
terminating.set()
process.kill()
return result
def check_revocation(self, netloc, port):
"""
check if one of the chain certs is revoked
:param netloc: target netloc
:param port: target port
:return:
"""
if self.crlDistributionPoints is not None and self.pem_cert is not None:
pid = os_getpid()
temp_cert_path = os_path_join(SQLITE_TEMP_DIR,
'temp_cert_{}.pem'.format(pid))
try:
temp_cert_file = open(
temp_cert_path, 'w') # file containing cert in PEM format
except OSError:
return None
else:
temp_cert_file.write(self.pem_cert)
temp_cert_file.close()
for crl_url in self.crlDistributionPoints: # for each crl given
_split = urlsplit(crl_url)
der_path = os_path_join(
SQLITE_TEMP_DIR,
'{}{}'.format(_split.netloc,
_split.path.replace('/', self._url_sep)))
pem_path = '{}_{}.pem'.format(der_path, pid)
if not os_path_exists(
der_path
): # create crl in DER format if does not exist
opener = build_opener(HTTPSHandler)
install_opener(opener)
try:
req = urlopen(url=crl_url)
except:
continue
data = req.read(10000000)
if data is None:
continue
try:
der_file = open(der_path, 'wb')
except OSError:
continue
else:
der_file.write(data)
der_file.close()
try: # PEM file which will contain crl + chain certs
pem_file = open(pem_path, 'w')
except OSError:
continue
else:
cmd = [
self.openssl_cmd, 'crl', '-inform', 'DER', '-in',
der_path, '-outform', 'PEM'
]
# print(' '.join(cmd)) # debug
sp_call(cmd, stdout=pem_file, stderr=SP_DEVNULL)
with sp_Popen([self.echo_cmd],
stdout=SP_PIPE,
stderr=SP_DEVNULL) as proc_echo:
cmd = [
self.openssl_cmd, 's_client', '-connect',
'{}:{}'.format(netloc, port), '-showcerts'
]
with sp_Popen(cmd,
stdin=proc_echo.stdout,
stdout=SP_PIPE,
stderr=SP_DEVNULL) as proc_certs:
data = proc_certs.stdout.read()
ddata = data.decode('utf-8')
if ddata is not None:
reg = re_compile(
'-+BEGIN CERTIFICATE-+[^-]+-+END CERTIFICATE-+\n'
)
for res in reg.finditer(ddata):
pem_file.write(res.group(0))
pem_file.close()
# finally verify if cert is revoked
cmd = [
self.openssl_cmd, 'verify', '-crl_check', '-CAfile',
pem_path, temp_cert_path
]
# print(' '.join(cmd)) # debug
with sp_Popen(cmd, stdout=SP_PIPE,
stderr=SP_DEVNULL) as proc_verify:
data = proc_verify.stdout.read()
ddata = data.decode('utf-8')
if ddata is not None:
reg = re_compile('lookup:([^\n]+)')
for res in reg.finditer(ddata):
if 'revoked' in res.group(1):
self.add_error(NameError(res.group(1)))
os_remove(pem_path)
os_remove(temp_cert_path)
def target():
self.process = sp_Popen(self.cmd, stdout=sp_PIPE, stderr=sp_PIPE, shell=True)
self.output = self.process.communicate()
def retrieve_fastas_by_acc(acc_dict, db_dir, local_fasta):
# Function downloads set of records from Genbank according to accessions passed to it.
# Downloaded FASTA file will be placed in 'db_dir' directory and named 'local_seq_set.fasta'
# :param acc_dict: dictionary comntaining accession data of hits;
# :type acc_dict: dict<str: tuple<str, str, int>>;
# :param db_dir: path to directory in which downloaded FASTA file will be placed;
# :type db_dir: str;
# :param local_fasta: path to file with reference sequences to be included in database;
# :type local_fasta: str;
# Path to file with current chunk (see below "100 accession numbers...")
tmp_fasta = os.path.join(db_dir, "tmp.fasta")
accessions = tuple(set(acc_dict.keys()))
if len(accessions) == 0: # just in case
return
# end if
# 100 accession numbers in order not to make too long URL
# Download genomes by chunks of 100 sequences.
max_accnum = 100
i = 0
accnum = len(accessions)
while i < accnum:
curr_accessions = accessions[i:i + max_accnum] # slice chunk
accs_del_comma = ','.join(
curr_accessions) # accessions must be separated by comma in url
# E-utilities provide a possibility to download records from Genbank by accessions.
retrieve_url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?\
db=nuccore&id={}&rettype=fasta&retmode=text".format(accs_del_comma)
log_info("Retrieve URL: `{}`".format(retrieve_url))
# GNU wget utility is safer, but there can be presence of absence of it :)
wget_util = "wget"
util_found = False
for d in os.environ["PATH"].split(os.pathsep):
if os.path.isdir(d) and wget_util in os.listdir(d):
util_found = True
break
# end if
# end for
print()
printlog_info("{} - Downloading {} reference sequences...".format(
getwt(), len(curr_accessions)))
if util_found:
# If we have wget -- just use it
wget_cmd = 'wget --no-check-certificate "{}" -O {}'.format(
retrieve_url, tmp_fasta)
pipe = sp_Popen(wget_cmd, shell=True)
pipe.communicate()
if pipe.returncode != 0:
printlog_error_time(
"Error occured while downloading reference sequences")
platf_depend_exit(pipe.returncode)
# end if
else:
# If there are no wget -- we will download sequences with Python disposal
stop_wait = Event(
) # a flag variable that will signal waiter-function to stop executing
def download_waiter(stop_wait):
"""
Function waits untill 'local_fasta' file is downloaded.
It prints size of downloaded data to console during downloading.
This function just waits -- it won't bring you the menu :).
"""
# Wait untill downloading starts
while not os.path.exists(tmp_fasta):
if not stop_wait.is_set():
return
# end if
sleep(1)
# end while
MB_size = 1024**2 # we will divide by it to get megabytes
while stop_wait.is_set():
# Get size of downloaded data
fsize = round(os.path.getsize(tmp_fasta) / MB_size,
1) # get megabytes
printn("\r{} - {} MB downloaded ".format(getwt(), fsize))
sleep(1) # instant updates are not necessary
# end while
# Print total size of downloaded file (it can be deleted by this time)
try:
fsize = round(os.path.getsize(tmp_fasta) / MB_size, 1)
except OSError:
# We can pass this ecxeption -- we do delete this file if downloading crushes
# And this function just waits :)
pass
# end try
printlog_info("\r{} - {} MB downloaded ".format(
getwt(), fsize))
# end def download_waiter
error = True
while error:
try:
waiter = Thread(target=download_waiter,
args=(stop_wait, )) # create thread
stop_wait.set() # raise the flag
waiter.start() # start waiting
urllib.request.urlretrieve(
retrieve_url, tmp_fasta) # retrieve FASTA file
except OSError as err:
printlog_error_time(
"Error occured while downloading fasta file.")
printlog_error(str(err))
printlog_error(
"`barapost-local.py` will try again in 30 seconds")
if os.path.exists(tmp_fasta):
os.unlink(tmp_fasta)
# end if
sleep(30)
else:
error = False
finally:
stop_wait.clear() # lower the flag
waiter.join(
) # main thread will wait until waiter function ends it's work
# end try
# end while
# end if
printlog_info_time("Downloading is completed")
# Write chunk to result fasta file
with open(tmp_fasta, 'r') as infile, open(local_fasta, 'a') as outfile:
outfile.write(infile.read())
# end with
# Remove temp chunk file
os.unlink(tmp_fasta)
i += max_accnum # go to next chunk
def target():
self.process = sp_Popen(self.cmd,
stdout=sp_PIPE,
stderr=sp_PIPE,
shell=True)
self.output = self.process.communicate()
