def _set_dates(self, netloc, port): """ set dates from openssl (notAfter and not Before) :param netloc: target netloc :param port: target port :return: """ with sp_Popen([self.echo_cmd], stdout=SP_PIPE) as proc_echo: cmd = self.ssl_client_cmd[:] cmd.append('{}:{}'.format(shlex_quote(netloc), str(port))) with sp_Popen(cmd, stdin=proc_echo.stdout, stdout=SP_PIPE, stderr=SP_DEVNULL) as proc_ssl: cmd = self.ssl_x509_cmd[:] cmd.append('-noout') cmd.append('-dates') with sp_Popen(cmd, stdin=proc_ssl.stdout, stdout=SP_PIPE, stderr=SP_DEVNULL) as proc_x509: data = proc_x509.stdout.read() ddata = data.decode('utf-8') if ddata is not None: reg = re_compile('notBefore=([^\n]+)') res = reg.search(ddata) if res is not None: self.notBefore = datetime.fromtimestamp( cert_time_to_seconds(res.group(1))) reg = re_compile('notAfter=([^\n]+)') res = reg.search(ddata) if res is not None: self.notAfter = datetime.fromtimestamp( cert_time_to_seconds(res.group(1)))
def _set_serial(self, netloc, port): """ set serial from openssl :param netloc: target netloc :param port: target port :return: """ with sp_Popen([self.echo_cmd], stdout=SP_PIPE) as proc_echo: cmd = self.ssl_client_cmd[:] cmd.append('{}:{}'.format(shlex_quote(netloc), str(port))) with sp_Popen(cmd, stdin=proc_echo.stdout, stdout=SP_PIPE, stderr=SP_DEVNULL) as proc_ssl: cmd = self.ssl_x509_cmd[:] cmd.append('-noout') cmd.append('-serial') with sp_Popen(cmd, stdin=proc_ssl.stdout, stdout=SP_PIPE, stderr=SP_DEVNULL) as proc_x509: data = proc_x509.stdout.read() ddata = data.decode('utf-8') if ddata is not None: reg = re_compile('serial=([^\n]+)') res = reg.search(ddata) if res is not None: self.serialNumber = res.group(1)
def _set_issuer(self, netloc, port): """ set issuer dict from openssl :param netloc: target netloc :param port: target port :return: """ with sp_Popen([self.echo_cmd], stdout=SP_PIPE) as proc_echo: cmd = self.ssl_client_cmd[:] cmd.append('{}:{}'.format(shlex_quote(netloc), str(port))) with sp_Popen(cmd, stdin=proc_echo.stdout, stdout=SP_PIPE, stderr=SP_DEVNULL) as proc_ssl: cmd = self.ssl_x509_cmd[:] cmd.append('-noout') cmd.append('-issuer') with sp_Popen(cmd, stdin=proc_ssl.stdout, stdout=SP_PIPE, stderr=SP_DEVNULL) as proc_x509: data = proc_x509.stdout.read() ddata = data.decode('utf-8') if ddata is not None: issuer_dict = dict() reg = re_compile('[A-Z]+=[^/\n]+') for res in reg.finditer(ddata): _split = res.group().split('=') if str(_split[0]) in self.keys: issuer_dict[self.keys[str( _split[0])]] = _split[1] self.issuer = issuer_dict
def _set_error_message(self, netloc, port): """ set basic error from openssl (certificate not valid) :param netloc: target netloc :param port: target port :return: """ with sp_Popen([self.echo_cmd], stdout=SP_PIPE) as proc_echo: cmd = self.ssl_client_cmd[:] cmd.append('{}:{}'.format(shlex_quote(netloc), str(port))) cmd.append('-CAfile') cmd.append(CA_CERTS) with sp_Popen(cmd, stdin=proc_echo.stdout, stdout=SP_DEVNULL, stderr=SP_PIPE) as proc_ssl: data = proc_ssl.stderr.read() ddata = data.decode('utf-8') if ddata is not None: reg = re_compile('error:num=[0-9]+:([^\n]+)') for res in reg.finditer(ddata): # expired checked at the end # print('error msg: {}'.format(res.group(0))) if 'expired' not in res.group(1): self.add_error(err=NameError(res.group(1)))
def _set_other_params(self, netloc, port): """ set other params from openssl (OCSP, CRL, caIssuers) :param netloc: target netloc :param port: target port :return: """ with sp_Popen([self.echo_cmd], stdout=SP_PIPE) as proc_echo: cmd = self.ssl_client_cmd[:] cmd.append('{}:{}'.format(shlex_quote(netloc), str(port))) with sp_Popen(cmd, stdin=proc_echo.stdout, stdout=SP_PIPE, stderr=SP_DEVNULL) as proc_ssl: cmd = self.ssl_x509_cmd[:] cmd.append('-noout') cmd.append('-text') with sp_Popen(cmd, stdin=proc_ssl.stdout, stdout=SP_PIPE, stderr=SP_DEVNULL) as proc_x509: data = proc_x509.stdout.read() ddata = data.decode('utf-8') if ddata is not None: reg = re_compile('Version: ([0-9]+)') res = reg.search(ddata) if res is not None: self.version = int(res.group(1)) reg = re_compile('OCSP - URI:([^\n]+)') res = reg.search(ddata) if res is not None: self.OCSP = res.group(1), reg = re_compile('CA Issuers - URI:([^\n]+)') res = reg.search(ddata) if res is not None: self.caIssuers = res.group(1), reg = re_compile( 'CRL Distribution Points:([ \n]*Full Name:[ \n]*URI:[^\n]+)+' ) res = reg.search(ddata) if res is not None: crls = list() _str = res.group(0) reg = re_compile('URI:([^\n]+)') for res in reg.finditer(_str): crls.append(res.group(1)) self.crlDistributionPoints = tuple(crls)
def set_pem_cert(self, netloc, port): """ set pem cert from openssl :param netloc: target netloc :param port: target port :return: """ with sp_Popen([self.echo_cmd], stdout=SP_PIPE) as proc_echo: cmd = self.ssl_client_cmd[:] cmd.append('{}:{}'.format(shlex_quote(netloc), str(port))) with sp_Popen(cmd, stdin=proc_echo.stdout, stdout=SP_PIPE, stderr=SP_DEVNULL) as proc_ssl: data = proc_ssl.stdout.read() ddata = data.decode('utf-8') if ddata is not None: reg = re_compile( '-+BEGIN CERTIFICATE-+[^-]+-+END CERTIFICATE-+\n') res = reg.search(ddata) if res is not None: self.pem_cert = res.group(0)
def _set_subject_alt_name(self, netloc, port): """ set subject alt name dict from openssl (authorized DNS) :param netloc: target netloc :param port: target port :return: """ with sp_Popen([self.echo_cmd], stdout=SP_PIPE) as proc_echo: cmd = self.ssl_client_cmd[:] cmd.append('{}:{}'.format(shlex_quote(netloc), str(port))) with sp_Popen(cmd, stdin=proc_echo.stdout, stdout=SP_PIPE, stderr=SP_DEVNULL) as proc_ssl: cmd = self.ssl_x509_cmd[:] cmd.append('-noout') cmd.append('-text') with sp_Popen(cmd, stdin=proc_ssl.stdout, stdout=SP_PIPE, stderr=SP_DEVNULL) as proc_x509: data = proc_x509.stdout.read() ddata = data.decode('utf-8') if ddata is not None: san_dict = dict() reg = re_compile('DNS:[^,\n]+') idx = 0 for res in reg.finditer(ddata): if idx > 0: key = 'DNS{}{}'.format(self._separator, str(idx)) else: key = 'DNS' san_dict[key] = res.group().split(':')[1] idx += 1 self.subjectAltName = san_dict
def ssh((hostname, command, ssh_timeout, realtime, multiline)): """Takes a tuple of arguments as multiprocessing map only accepts one and returns a result dictionary.""" result = { "returncode": None, "stdout": None, "stderr": None, "command": "%s %s %s" % (SSH_CMD % ssh_timeout, hostname, command), "hostname": hostname, "status": "pending", } # Run in a try/catch to check for SIGINT, this will kill the running process # then set the global terminating event preventing subsequent workers from # running. try: if terminating.is_set(): # If we are terminating, update the result set and return. result["returncode"] = 2 result["status"] = "aborted" else: # Start subprocess for ssh. shell=True is nasty, but this is needed. process = sp_Popen("%s %s %s" % (SSH_CMD % ssh_timeout, hostname, command), stdout=sp_PIPE, stderr=sp_PIPE, shell=True) stdout, stderr = process.communicate() if multiline: result["stdout"] = stdout.strip("\n") result["stderr"] = stderr.strip("\n") else: result["stdout"] = stdout.replace("\n", " ") result["stderr"] = stderr.replace("\n", " ") result["returncode"] = process.returncode result["status"] = "done" except KeyboardInterrupt: # Catch keyboard interrupt, killing the current worker and setting the # global terminating event. result["returncode"] = 1 result["status"] = "killed" terminating.set() process.kill() return result
def ssh((hostname, command, ssh_timeout, realtime, multiline)): """Takes a tuple of arguments as multiprocessing map only accepts one and returns a result dictionary.""" result = { "returncode": None, "stdout": None, "stderr": None, "command": "%s %s %s" % (SSH_CMD % ssh_timeout, hostname, command), "hostname": hostname, "status": "pending", } # Run in a try/catch to check for SIGINT, this will kill the running process # then set the global terminating event preventing subsequent workers from # running. try: if terminating.is_set(): # If we are terminating, update the result set and return. result["returncode"] = 2 result["status"] = "aborted" else: # Start subprocess for ssh. shell=True is nasty, but this is needed. process = sp_Popen( "%s %s %s" % (SSH_CMD % ssh_timeout, hostname, command), stdout=sp_PIPE, stderr=sp_PIPE, shell=True ) stdout, stderr = process.communicate() if multiline: result["stdout"] = stdout.strip("\n") result["stderr"] = stderr.strip("\n") else: result["stdout"] = stdout.replace("\n", " ") result["stderr"] = stderr.replace("\n", " ") result["returncode"] = process.returncode result["status"] = "done" except KeyboardInterrupt: # Catch keyboard interrupt, killing the current worker and setting the # global terminating event. result["returncode"] = 1 result["status"] = "killed" terminating.set() process.kill() return result
def check_revocation(self, netloc, port): """ check if one of the chain certs is revoked :param netloc: target netloc :param port: target port :return: """ if self.crlDistributionPoints is not None and self.pem_cert is not None: pid = os_getpid() temp_cert_path = os_path_join(SQLITE_TEMP_DIR, 'temp_cert_{}.pem'.format(pid)) try: temp_cert_file = open( temp_cert_path, 'w') # file containing cert in PEM format except OSError: return None else: temp_cert_file.write(self.pem_cert) temp_cert_file.close() for crl_url in self.crlDistributionPoints: # for each crl given _split = urlsplit(crl_url) der_path = os_path_join( SQLITE_TEMP_DIR, '{}{}'.format(_split.netloc, _split.path.replace('/', self._url_sep))) pem_path = '{}_{}.pem'.format(der_path, pid) if not os_path_exists( der_path ): # create crl in DER format if does not exist opener = build_opener(HTTPSHandler) install_opener(opener) try: req = urlopen(url=crl_url) except: continue data = req.read(10000000) if data is None: continue try: der_file = open(der_path, 'wb') except OSError: continue else: der_file.write(data) der_file.close() try: # PEM file which will contain crl + chain certs pem_file = open(pem_path, 'w') except OSError: continue else: cmd = [ self.openssl_cmd, 'crl', '-inform', 'DER', '-in', der_path, '-outform', 'PEM' ] # print(' '.join(cmd)) # debug sp_call(cmd, stdout=pem_file, stderr=SP_DEVNULL) with sp_Popen([self.echo_cmd], stdout=SP_PIPE, stderr=SP_DEVNULL) as proc_echo: cmd = [ self.openssl_cmd, 's_client', '-connect', '{}:{}'.format(netloc, port), '-showcerts' ] with sp_Popen(cmd, stdin=proc_echo.stdout, stdout=SP_PIPE, stderr=SP_DEVNULL) as proc_certs: data = proc_certs.stdout.read() ddata = data.decode('utf-8') if ddata is not None: reg = re_compile( '-+BEGIN CERTIFICATE-+[^-]+-+END CERTIFICATE-+\n' ) for res in reg.finditer(ddata): pem_file.write(res.group(0)) pem_file.close() # finally verify if cert is revoked cmd = [ self.openssl_cmd, 'verify', '-crl_check', '-CAfile', pem_path, temp_cert_path ] # print(' '.join(cmd)) # debug with sp_Popen(cmd, stdout=SP_PIPE, stderr=SP_DEVNULL) as proc_verify: data = proc_verify.stdout.read() ddata = data.decode('utf-8') if ddata is not None: reg = re_compile('lookup:([^\n]+)') for res in reg.finditer(ddata): if 'revoked' in res.group(1): self.add_error(NameError(res.group(1))) os_remove(pem_path) os_remove(temp_cert_path)
def target(): self.process = sp_Popen(self.cmd, stdout=sp_PIPE, stderr=sp_PIPE, shell=True) self.output = self.process.communicate()
def retrieve_fastas_by_acc(acc_dict, db_dir, local_fasta): # Function downloads set of records from Genbank according to accessions passed to it. # Downloaded FASTA file will be placed in 'db_dir' directory and named 'local_seq_set.fasta' # :param acc_dict: dictionary comntaining accession data of hits; # :type acc_dict: dict<str: tuple<str, str, int>>; # :param db_dir: path to directory in which downloaded FASTA file will be placed; # :type db_dir: str; # :param local_fasta: path to file with reference sequences to be included in database; # :type local_fasta: str; # Path to file with current chunk (see below "100 accession numbers...") tmp_fasta = os.path.join(db_dir, "tmp.fasta") accessions = tuple(set(acc_dict.keys())) if len(accessions) == 0: # just in case return # end if # 100 accession numbers in order not to make too long URL # Download genomes by chunks of 100 sequences. max_accnum = 100 i = 0 accnum = len(accessions) while i < accnum: curr_accessions = accessions[i:i + max_accnum] # slice chunk accs_del_comma = ','.join( curr_accessions) # accessions must be separated by comma in url # E-utilities provide a possibility to download records from Genbank by accessions. retrieve_url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?\ db=nuccore&id={}&rettype=fasta&retmode=text".format(accs_del_comma) log_info("Retrieve URL: `{}`".format(retrieve_url)) # GNU wget utility is safer, but there can be presence of absence of it :) wget_util = "wget" util_found = False for d in os.environ["PATH"].split(os.pathsep): if os.path.isdir(d) and wget_util in os.listdir(d): util_found = True break # end if # end for print() printlog_info("{} - Downloading {} reference sequences...".format( getwt(), len(curr_accessions))) if util_found: # If we have wget -- just use it wget_cmd = 'wget --no-check-certificate "{}" -O {}'.format( retrieve_url, tmp_fasta) pipe = sp_Popen(wget_cmd, shell=True) pipe.communicate() if pipe.returncode != 0: printlog_error_time( "Error occured while downloading reference sequences") platf_depend_exit(pipe.returncode) # end if else: # If there are no wget -- we will download sequences with Python disposal stop_wait = Event( ) # a flag variable that will signal waiter-function to stop executing def download_waiter(stop_wait): """ Function waits untill 'local_fasta' file is downloaded. It prints size of downloaded data to console during downloading. This function just waits -- it won't bring you the menu :). """ # Wait untill downloading starts while not os.path.exists(tmp_fasta): if not stop_wait.is_set(): return # end if sleep(1) # end while MB_size = 1024**2 # we will divide by it to get megabytes while stop_wait.is_set(): # Get size of downloaded data fsize = round(os.path.getsize(tmp_fasta) / MB_size, 1) # get megabytes printn("\r{} - {} MB downloaded ".format(getwt(), fsize)) sleep(1) # instant updates are not necessary # end while # Print total size of downloaded file (it can be deleted by this time) try: fsize = round(os.path.getsize(tmp_fasta) / MB_size, 1) except OSError: # We can pass this ecxeption -- we do delete this file if downloading crushes # And this function just waits :) pass # end try printlog_info("\r{} - {} MB downloaded ".format( getwt(), fsize)) # end def download_waiter error = True while error: try: waiter = Thread(target=download_waiter, args=(stop_wait, )) # create thread stop_wait.set() # raise the flag waiter.start() # start waiting urllib.request.urlretrieve( retrieve_url, tmp_fasta) # retrieve FASTA file except OSError as err: printlog_error_time( "Error occured while downloading fasta file.") printlog_error(str(err)) printlog_error( "`barapost-local.py` will try again in 30 seconds") if os.path.exists(tmp_fasta): os.unlink(tmp_fasta) # end if sleep(30) else: error = False finally: stop_wait.clear() # lower the flag waiter.join( ) # main thread will wait until waiter function ends it's work # end try # end while # end if printlog_info_time("Downloading is completed") # Write chunk to result fasta file with open(tmp_fasta, 'r') as infile, open(local_fasta, 'a') as outfile: outfile.write(infile.read()) # end with # Remove temp chunk file os.unlink(tmp_fasta) i += max_accnum # go to next chunk