def fetch_handler(): source_name = "CVESearch" colors.print_info("[-] Downloading %s" % source_name) source = sourcehelper.SourceHelper( "https://cve.circl.lu/static/circl-cve-search-expanded.json.gz") source_data = source.fetch() source_data = source_data.decode("utf8") source_data = source_data.replace("\n", ",\n") source_data = "[" + source_data source_data = source_data[:-2] + "]\n" # Reformat document # sourcehelper.write_source(source_name, source_data) # TODO: Reconvert data to Vulners JSON model source_data = json.dumps(json.loads(source_data)) colors.print_success("Saving source %s" % source_name) sourcehelper.write_source(source_name, source_data) sourcehelper.write_source_sig(source_name, sourcehelper.make_sig(source_data))
def source_update(src_name): # Need new connection for the new process phdb = prohacktivedb.ProHacktiveDB() src_sig = sourcehelper.read_source_sig(src_name).decode("utf8") src_dat = sourcehelper.read_file_bytes( sourcehelper.get_fetched_srcs_dir() + src_name + ".dat").decode("utf8") colors.print_info("[-] Inserting source %s signature %s" % (src_name, src_sig)) phdb.insert_src_sig(src_name, src_sig) colors.print_info("[-] Inserting source %s dat %s" % (src_name, src_dat)) phdb.insert_src_dat(src_name, src_dat) colors.print_info("[-] Inserting source %s" % src_name) src_data = json.loads(sourcehelper.read_source(src_name)) colors.print_info("[-] Erasing old vulnerabilities %s ..." % src_name) phdb.collections.drop_collection(src_name) colors.print_info("[-] Inserting vulnerabilities of %s ..." % src_name) phdb.insert_vulnerabilities(src_data, src_name) colors.print_success("[x] Updated %s" % src_name)
def main(): colors.print_info("[-] ProHacktive updating running...") srcs_name = SourcesManager().list_all() phdb = prohacktivedb.ProHacktiveDB() if len(srcs_name) == 0: colors.print_warn("[-] No sources to update!") else: colors.print_warn("[-] updating on host %s with port %s" % (phdb.host, phdb.port)) colors.print_info("[-] Updating sources") processes_list = list() # Prepare processes for each sources for src_name in srcs_name: processes_list.append( processes.CProcess(src_name, source_update, src_name)) process_limit_update = config.current_config.process_limit_update # Process sources updating processes.handle_processes(processes_list, process_limit_update, 0.01) colors.print_success( "[x] ProHacktive database has been updated successfully!")
def download(url, path): if os.path.exists(path): print_info("File %s already exists. Skipping download." % path) return u = urllib2.urlopen(url) make_dirs_if_needed(os.path.dirname(path)) f = open(path, 'wb') meta = u.info() file_size = int(meta.getheaders("Content-Length")[0]) print_info("Path: %s" % path) file_size_dl = 0 block_sz = 8192 while True: buffer = u.read(block_sz) if not buffer: break file_size_dl += len(buffer) f.write(buffer) percent = file_size_dl * 100. / file_size msg = "Downloading: [%s]" % sizeof_fmt(file_size) + " ... " + str( int(percent)) sys.stdout.write("\r%s%%" % msg) sys.stdout.flush() print f.close()
def reporthook(blocknum, blocksize, totalsize): readsofar = blocknum * blocksize percent = int((readsofar * 1e2 / totalsize) / 2) r_size = totalsize / 1024**2 d_size = readsofar / 1024**2 pgbar = '[{}{}] '.format('█' * percent, ' ' * (50 - percent)) + '[{0:.2f}/{1:.2f} MB]'.format(d_size, r_size) colors.print_info('Downloading: ', pgbar, start='\r', end='\r')
def source_update(src_name): # Need new connection for the new process phdb = prohacktivedb.ProHacktiveDB() # Read local sources signatures source_local_sig = sourcehelper.read_source_sig(src_name).decode("utf8") source_remote_sig = phdb.find_src_sig_from_name(src_name) if source_local_sig == source_remote_sig: colors.print_info("[-] Same file signature on %s (%s-%s), skipping" % (src_name, source_local_sig, source_remote_sig)) return # Get time from the top newest update update_date_remote = phdb.find_src_dat_from_name(src_name) update_date_remote = datetime.strptime(update_date_remote, "%Y-%m-%dT%H:%M:%S") # Find first the top newest updates on local # Read source data source_data = json.loads(sourcehelper.read_source(src_name).decode("utf8")) vulnerabilities_to_update = list() for vulnerability in source_data: vulnerability_lastseen_date = vulnerability["_source"]["lastseen"] vulnerability_published_date = vulnerability["_source"]["published"] vulnerability_modified_date = vulnerability["_source"]["modified"] # Get the max date between all those dates vulnerability_update_date = max(vulnerability_lastseen_date, vulnerability_modified_date, vulnerability_published_date) # If the date is higher than the last source fetching date on remote, # we append the vulnerabilities we need to update/insert vulnerability_date_local = datetime.strptime(vulnerability_update_date, "%Y-%m-%dT%H:%M:%S") if vulnerability_date_local > update_date_remote: vulnerabilities_to_update.append(vulnerability) if len(vulnerabilities_to_update) == 0: raise Exception( "File signature has changed but no vulnerabilities to update found" ) # Update all vulnerabilities into the list for vulnerability in vulnerabilities_to_update: phdb.update_vulnerability(vulnerability, src_name) phdb.update_src_sig(src_name, source_local_sig) colors.print_success("[x] Updated %s" % src_name)
def check_hash(hash): try: res = vtotal.file_report([hash]) data = res.get('json_resp') if (data.get('response_code') == 0): colors.print_header(' [~] No threat found') return positives = int(data.get("positives")) if (positives >= 3): colors.print_fail(f' [!] {positives} Threats found ') else: colors.print_warning(f' [!] {positives} Potential threats found') except Exception as err: colors.print_info(" [!] MAX REQUESTS EXCEEDED")
def fetch(self) -> bytes: try: response = requests.get(self.url) except requests.RequestException as e: colors.print_error("[!]" + e) return False # Get response data = response.content # Check if data is compressed if is_compressed(data): colors.print_info("[-] Decompressing %s" % self.url) # Write to temporary file the response if not os.path.exists(temp_dir): os.mkdir(temp_dir) temp_filename = temp_dir + "tempfile" # Sadly we need to write it to a file because pyunpack can't yet # decompress from binary data directly from memory temp_file = open(temp_filename, "wb") temp_file.write(data) temp_file.close() # Decompress filename = temp_filename archive_dir = temp_dir + "archive/" if not os.path.exists(archive_dir): os.mkdir(archive_dir) # Sometimes it's compressed multiple times while (True): arch = pyunpack.Archive(filename) arch.extractall(archive_dir) os.remove(filename) filename = archive_dir + os.listdir(archive_dir)[0] compressed = is_file_compressed(filename) if not compressed: break temp_file = open(filename, "rb") data = bytes(temp_file.read()) temp_file.close() os.remove(filename) return data
def connect_ssh(hostname, username, password): client = paramiko.SSHClient() try: client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) except: pass try: colors.print_warning( f"[ATTEMPTING CONNECTION] || {hostname} || {username}:{password}") client.connect(hostname=hostname, username=username, password=password, timeout=3) except socket.timeout as err: colors.print_fail(f"[!] Invalid Host: {hostname}") raise err except paramiko.AuthenticationException: return None except paramiko.SSHException as err: timeout_attempts += 1 if (timeout_attempts < 5): colors.print_info(f"Time Locked retrying... {timeout_attempts}/5") time.sleep(60) return connect_ssh(hostname, username, password) else: raise err except Exception as err: raise err colors.print_success("[+] CONNECTION ESTABLISHED:") print(f""" {colors.colors.ENDC}HOSTNAME: {colors.colors.HEADER}{hostname} {colors.colors.ENDC}USERNAME: {colors.colors.HEADER}{username} {colors.colors.ENDC}PASSWORD: {colors.colors.HEADER}{password} {colors.colors.ENDC} """) return client
def fetch_handler(): vulners_api = vulners.Vulners(vulners_api_key) colors.print_info("[-] Vulners API loading multiples collections:\n") count_for_newline = 0 print_new_line = False collections = vulners_api.collections() number_of_collections = len(collections) for collection_name in collections: if (count_for_newline > 0) and ((count_for_newline % 6) == 0): print("\n", end="") print_new_line = False if count_for_newline == (number_of_collections - 1): print(collection_name, end="") else: print(collection_name + ", ", end="") count_for_newline += 1 print_new_line = True if print_new_line: print("\n") colors.print_warn("[-] Downloading %i sub-sources..." % len(collections)) processes_list = list() for collection_name in collections: processes_list.append( processes.CProcess(collection_name, vulners_source_fetch, vulners_api_key, collection_name)) processes_count_limit = len(processes_list) processes.handle_processes(processes_list, processes_count_limit, 0.01)
def main(): colors.print_info("[-] ProHacktive fetching running...") srcsmanager = SourcesManager() if srcsmanager.fetch_all(): colors.print_info("[-] Sources generated signatures:") for source in srcsmanager.read_srcs_sigs(): colors.print_info(" Source Name: %s -> %s" % (source["_id"], source["sig"])) colors.print_success("[x] ProHacktive fetching done!") else: colors.print_error("[!] ProHacktive fetching failed")
def main(): phdb = prohacktivedb.ProHacktiveDB() if len(srcs_name) == 0: colors.print_warn("[-] No sources to update!") else: colors.print_warn( "[-] Full updating on host %s with port %s" % (phdb.host, phdb.port)) colors.print_info("[-] Erasing old signatures") phdb.collections.drop_collection(phdb.get_srcs_sigs_collection_name()) colors.print_info("[-] Erasing old data informations") phdb.collections.drop_collection(phdb.get_srcs_dat_collection_name()) colors.print_info("[-] Erasing old statistics") phdb.drop_remote_stats() colors.print_info("[-] Updating sources") processes_list = list() # Prepare processes for each sources for src_name in srcs_name: processes_list.append( processes.CProcess( src_name, source_update, src_name)) process_limit_update = config.current_config.process_limit_update # Process sources updating processes.handle_processes(processes_list, process_limit_update, 0.01) colors.print_success( "[x] ProHacktive database has been full updated successfully!")
def __del__(self): # Inserts stats at the end of connection colors.print_info("[-] Updating stats on %s:%i" % (self.host, self.port)) self.update_remote_stats()
import os import sys import json runPath = os.path.dirname(os.path.realpath(__file__ + "../")) sys.path.append(runPath + "/lib/") import config import colors import prohacktivedb import processes import sourcehelper from sourcesmanager import SourcesManager colors.print_info("[-] ProHacktive full updating running...") srcs_name = SourcesManager().list_all() def source_update(src_name): # Need new connection for the new process phdb = prohacktivedb.ProHacktiveDB() src_sig = sourcehelper.read_source_sig(src_name).decode("utf8") src_dat = sourcehelper.read_file_bytes( sourcehelper.get_fetched_srcs_dir() + src_name + ".dat").decode("utf8") colors.print_info("[-] Inserting source %s signature %s" % (src_name, src_sig)) phdb.insert_src_sig(src_name, src_sig)
def vulners_source_fetch(api_key, collection_name): source_name = "Vulners_" + collection_name source_file_data = source_name + ".dat" source_file_sig = source_name + ".sig" datetime_now = datetime.now() time_fmt = datetime_now.strftime("%Y-%m-%dT%H:%M:%S") important_files_exists = ( os.path.isfile(fetched_srcs + source_name) and os.path.isfile(fetched_srcs + source_file_sig) and os.path.isfile(fetched_srcs + source_file_data)) if important_files_exists: read_source_date = sourcehelper.read_file(fetched_srcs + source_file_data) from_date = str(read_source_date) to_date = time_fmt colors.print_info("[-] Downloading %s from date %s to date %s" % (collection_name, from_date, to_date)) vulners_api = vulners.Vulners(api_key) source_update = vulners_api.vulners_get_request( 'archive', { 'type': collection_name, 'datefrom': from_date }) # Decompress zip data with ZipFile(BytesIO(source_update)) as zip_file: if len(zip_file.namelist()) > 1: raise Exception("Unexpected file count in Vulners ZIP archive") file_name = zip_file.namelist()[0] source_update = bytes(zip_file.open(file_name).read()) source_update = json.loads(source_update.decode("utf8")) # No updates if len(source_update) == 0: colors.print_info("[-] No updates on %s, skipping" % source_name) sourcehelper.write_file(fetched_srcs + source_file_data, to_date) return source_data = sourcehelper.read_source(source_name).decode("utf8") source_data = json.loads(source_data) # Find every vulnerabilities that needs update into the file and update it for vulnerability_update in source_update: # By default it's not inserted updated_vulnerability = False # Check if it has been inserted already for vulnerability_index in range(len(source_data)): if source_data[vulnerability_index][ "_id"] == vulnerability_update["_id"]: source_data[vulnerability_index] = vulnerability_update updated_vulnerability = True # If the vulnerability isn't here, we append it into the file if not updated_vulnerability: source_data.append(vulnerability_update) colors.print_info("[-] Saving file signature %s" % (fetched_srcs + source_file_sig)) # Re-encode data source_data = bytes(json.dumps(source_data).encode("utf8")) # Write file signature sourcehelper.write_source_sig(source_name, sourcehelper.make_sig(source_data)) # Write file date (we might use the date directly from the os but to be sure) sourcehelper.write_file(fetched_srcs + source_file_data, to_date) colors.print_info("[-] Saving source %s" % (fetched_srcs + source_name)) sourcehelper.write_source(source_name, source_data) else: colors.print_info("[-] Downloading %s" % source_name) vulners_api = vulners.Vulners(api_key) collection = vulners_api.vulners_get_request('archive', {'type': collection_name}) # Decompress zip data with ZipFile(BytesIO(collection)) as zip_file: if len(zip_file.namelist()) > 1: raise Exception("Unexpected file count in Vulners ZIP archive") file_name = zip_file.namelist()[0] # Reformat document with a json load so it writes correct signature & convert it to bytes collection = bytes( json.dumps( json.loads(zip_file.open(file_name).read().decode( "utf8"))).encode("utf8")) src_sig = sourcehelper.make_sig(collection) # Write collection date, used for faster updates by using the API sourcehelper.write_file(fetched_srcs + source_file_data, time_fmt) colors.print_info("[-] Saving file signature %s" % (fetched_srcs + source_file_sig)) # Write file signature sourcehelper.write_source_sig(source_name, src_sig) colors.print_info("[-] Saving source %s" % (fetched_srcs + source_name)) sourcehelper.write_source(source_name, collection)
def count_dicts(json_obj): # If dictionary start directly from there if isinstance(json_obj, dict): _count_dicts(json_obj) # Otherwhise it must be a list else: for o in json_obj: _count_dicts(o) # Check if argument is passed sys.argv.append("tmp/fetched_srcs/Vulners_cve") if sys.argv[1]: # Read json data colors.print_info("[-] Loading %s" % sys.argv[1]) json_file = open(sys.argv[1], "r") json_data = json.loads(json_file.read()) json_file.close() colors.print_info("[-] Counting dictionaries...") dict_count = 0 # Count all dictionaries count_dicts(json_data) pbar = colors.print_progress_start(dict_count) colors.print_info("[-] Analyzing %i dictionaries..." % dict_count)
config_gen["DATABASE"] = { "db_host": self.db_host, "db_port": str(self.db_port), "db_name": self.db_name, "db_user": self.db_user, "db_pass": self.db_password} config_gen["VULNERS"] = {"vulners_api_key": self.vulners_api_key} config_gen["processes"] = { "process_limit_update": int(self.process_limit_update), "process_limit_fetch": int(self.process_limit_fetch)} return config_gen default_config = Config() current_config = default_config colors.print_info("[-] Parsing %s" % config_filename) config = configparser.ConfigParser() dataset = config.read(config_filename) if len(dataset) == 0: colors.print_error("[!] Couldn't read config file at %s" % config_filename) colors.print_warn("[!] Using default configuration") else: db = config["DATABASE"] vulners_api_key = config["VULNERS"]["vulners_api_key"] processes = config["PROCESSES"] current_config = Config( db["db_host"], db["db_port"], db["db_name"], db["db_user"], db["db_pass"],