def main(): """ Starts as many sslyze worker processes as specified in :mod:`settings`. """ signal.signal(signal.SIGINT, _sigint_handler) ########################## # PLUGINS INITIALIZATION # ########################## sslyze_plugins = PluginsFinder() available_plugins = sslyze_plugins.get_plugins() available_commands = sslyze_plugins.get_commands() ######################## # QUEUE INITIALIZATION # ######################## c = QueueClient() qm = c.queue_manager() ########################## # PROCESS INITIALIZATION # ########################## for _ in xrange(settings.NUMBER_PROCESSES): p = WorkerProcess(qm, available_commands) p.start() process_list.append(p) # Wait for all processes to terminate for p in process_list: p.join()
def post(self): user_id = hashlib.sha256(str(self.request)).hexdigest() domain = self.get_argument('domain') c = QueueClient() qm = c.queue_manager() qm.put_user(domain, user_id) r = qm.get_user_result(user_id) self.write(json.dumps(r))
def main(): # get instance of QueueClient c = QueueClient() # get appropriate queue from QueueClient qm = c.queue_manager() # get instance of Database mdb = Database() while True: result = qm.next_result() # get tld suffix. tld installed with 'pip install tld' # to update the tld-names run 'update-tld-names' as discribed in https://pypi.python.org/pypi/tld tld = get_tld('https://' + result["target"][0], as_object=True).suffix # fill cs_avail and cs_pref with the cipher suites of the result cs_avail = [] cs_pref = [] tls_ver = ['tlsv1_2', 'tlsv1_1', 'sslv3', 'sslv2', 'tlsv1'] for k in range(5): if 'preferredCipherSuite' in (result['result'][k]): for cs, value in ((result['result'][k])['preferredCipherSuite']).iteritems(): if value[2]: cs_pref.append('{ ' 'cs: ' + cs + ', tls_version: ' + tls_ver[k] + ', dh_param : ' + value[2]['GroupSize'] + ' }') else: cs_pref.append('{ cs: ' + cs + ', tls_version: ' + tls_ver[k] + ' }') for cs, value in ((result['result'][k])['acceptedCipherSuites']).iteritems(): if value[2]: cs_avail.append('{ cs: ' + cs + ', tls_version: ' + tls_ver[k] + ', dh_param : ' + value[2]['GroupSize'] + ' }') else: cs_avail.append('{ cs: ' + cs + ', tls_version: ' + tls_ver[k] + ' }') update_query = { 'domain:': result["target"][0], 'tld': tld, } # check whether a preferred cs is available. Otherwise the host was not accessible if cs_pref: update_add = { "$addToSet": {"scans": { 'date': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), 'source': result["source"], 'cs_pref': cs_pref, 'cs_avail': cs_avail, }} } mdb.insert_result(update_query, update_add)
def main(): ########################## # PLUGINS INITIALIZATION # ########################## sslyze_plugins = PluginsFinder() available_plugins = sslyze_plugins.get_plugins() available_commands = sslyze_plugins.get_commands() ######################## # QUEUE INITIALIZATION # ######################## # TODO use queue from remote queue manager, this is just for testing purposes c = QueueClient() host_queue = c.host_queue() result_queue = c.result_queue() # host_queue = JoinableQueue() # result_queue = JoinableQueue() # host_queue.put("google.com") ########################## # PROCESS INITIALIZATION # ########################## process_list = [] for _ in xrange(settings.NUMBER_PROCESSES): p = WorkerProcess(host_queue, result_queue, available_commands) p.start() process_list.append(p) # Wait for all processes to terminate for p in process_list: p.join()
from queue_manager import QueueClient __author__ = 'david' if __name__ == "__main__": # get instance of QueueClient c = QueueClient() # get appropriate queue from QueueClient host_queue = c.host_queue() for i in range(20): # put hostname to host_queue host_queue.put("google.com")
from queue_manager import QueueClient if __name__ == "__main__": # get instance of QueueClient c = QueueClient() # get appropriate queue from QueueClient result_queue = c.result_queue() while True: result = result_queue.get() # do something with result ... print(result)
splits.remove(splits[0]) f.close() return r # downloads and unzips AlexaCSV and return the date def getAlexaCSV(): #downloads from url r = requests.get("http://s3.amazonaws.com/alexa-static/top-1m.csv.zip") #unzips z = zipfile.ZipFile(StringIO.StringIO(r.content)) z.extractall() #moves file from "crawler" to "crawler/data" shutil.move("top-1m.csv", "data/top-1m.csv") #get date as "dd.mm.yyyy" a = time.strftime("%d.%m.%Y") return a if __name__ == "__main__": # get instance of QueueClient c = QueueClient() # get appropriate queue from QueueClient #host_queue = c.host_queue() queue_manager = c.queue_manager() d = getAlexaCSV() #host_queue.put(datedAlexaCSV(d, alexa_splits)) queue_manager.put_new_list(queue_manager, datedAlexaCSV(d, alexa_splits))