def insert(domain_data): db = init() data = vars(domain_data) logger.yellow('Adding %s to %s' % (logger.YELLOW(str(data)), logger.YELLOW(db_name))) try: db.insert(data) except Exception as e: logger.red('Got [%s] whilst adding %s to %s' % (logger.RED(str(e)), logger.RED(data), logger.RED(db_name))) return None # this return code wont be checked anywhere, i just dont like leaving unclosed functions :)
def validate(email): try: # The password here doesnt really matter as the o365 link just requires anything, but its worthwhile having a common password in order to check for access at the same time password = '******' url = 'https://outlook.office365.com/Microsoft-Server-ActiveSync' headers = {"MS-ASProtocolVersion": "14.0"} auth = (email, password) try: logger.verbose('Attempting to validate %s' % logger.YELLOW(email)) r = requests.options(url, headers=headers, auth=auth) status = r.status_code except: logger.verbose('Unable to connect to [%s]' % logger.RED(url)) quit() if status == 401: logger.verbose('Successfully validated %s' % logger.GREEN(email)) return True elif status == 404: logger.verbose('Could not validate %s' % logger.RED(email)) return False elif status == 403: logger.green('Found credentials: %s:%s (2FA)' % (logger.GREEN(email), logger.GREEN(password))) return [True, password] elif status == 200: logger.green('Found credentials: %s:%s' % (logger.GREEN(email), logger.GREEN(password))) return [True, password] else: logger.verbose( 'Got HTTP Status Response %s. Unexpected, skipping.' % logger.RED(str(status))) return None except KeyboardInterrupt: logger.yellow('Keyboard interrupt detected!') quit()
def validate(email): try: password = '******' url = 'https://outlook.office365.com/Microsoft-Server-ActiveSync' headers = {"MS-ASProtocolVersion": "14.0"} auth = (email, password) try: r = requests.options(url, headers=headers, auth=auth) status = r.status_code except: logger.red('Unable to connect to [%s]' % logger.RED(url)) quit() if status == 401: logger.green('Successfully validated %s' % logger.GREEN(email)) return True elif status == 404: if r.headers.get("X-CasErrorCode") == "emailNotFound": logger.red('Could not validate %s' % logger.RED(email)) return False elif status == 403: logger.green('Found credentials: %s:%s (2FA)' % (logger.GREEN(email), logger.GREEN(password))) return [True, password] elif status == 200: logger.green('Found credentials: %s:%s' % (logger.GREEN(email), logger.GREEN(password))) return [True, password] else: logger.red('Got HTTP Status Response %s. Unexected, skipping.') return None except KeyboardInterrupt: logger.yellow('Keyboard interrupt detected!') quit()
def go(): logger.yellow('Checking %s' % 'crt.sh') do_crtsh() logger.yellow('Checking %s' % 'bufferover.run') do_bufferoverrun() logger.yellow('Checking %s' % 'certspotter') do_certspotter()
logger.red('Unknown validation type: ' + logger.RED(args.validate)) logger.red('Please specify either %s or %s' % (logger.RED('o365'), logger.RED('hunter'))) quit() else: validation = None api_key = None data = data_structure.Data(cookie, company_id, email_format, domain, filename, keyword, validation, api_key) logger.debug(str(vars(data))) for k, v in vars(data).items(): if v != None: logger.yellow('%s set to %s' % (k, logger.YELLOW(v))) else: logger.debug('%s set to None' % k) print() try: logger.debug('Running core.run()') users = core.run(data) except KeyboardInterrupt: logger.yellow('Keyboard interrupt detected!') quit() logger.green('Done!') finish_time = datetime.datetime.now().replace(microsecond=0)
api_key = None elif args.validate.lower() == 'hunter': if args.api == None: logger.red( 'If validating through Hunter, the API Key is required (%s).' % logger.RED('--api')) quit() else: api_key = args.api logger.blue('Validating users via %s' % logger.BLUE('Hunter')) validation = 'hunter' else: logger.red('Unknown validation type: ' + logger.RED(args.validate)) logger.red('Please specify either %s or %s' % (logger.RED('o365'), logger.RED('hunter'))) quit() else: validation = None api_key = None connection_data = [cookie, company_id, email_format] try: users = user_enum.run(connection_data, domain, filename, keyword, validation, api_key) except KeyboardInterrupt: logger.yellow('Keyboard interrupt detected!') quit() logger.green('Done!')
def run(data): cookie = data.cookie company_id = data.company_id email_format = data.email_format keyword = data.keyword domain = data.domain validation = data.validation api_key = data.api_key filename = data.filename valid_emails_only = data.valid_emails_only threads = int(data.threads) logger.debug(str(vars(data))) profiles = linkedin_scraper.company_profile(cookie, company_id, keyword) if profiles == None: logger.red('Unable to extract data from LinkedIn') quit() company_profile_json = json.loads(profiles) try: total_employees = company_profile_json['elements'][0]['total'] except: logger.red('Failed to extract users, try generalising the keywords') logger.red('If this problem persists, create a issue on GitHub!') quit() per_page = 40 # Each response contains 40 profiles per page. pages = int( total_employees / per_page ) # Divide the amount of users by 40, this will give you the amount of pages logger.debug('Per page: %s' % per_page) if total_employees < per_page: logger.debug('Setting per_page to 1') pages = 1 logger.blue('Identified %s page(s)' % logger.BLUE(pages)) logger.blue('Identified %s result(s)' % logger.BLUE(total_employees)) if pages == 0: logger.red('Could not identify pages') quit() if total_employees > 1000: logger.red('This method of enumeration can only extract 1000 users') print() sleep(3) users = linkedin_scraper.get_users(data, pages, total_employees, keyword) print() if validation is not None: if o365_validation.verify_o365(domain)[domain]: logger.yellow('Running through email addresses...') users = linkedin_scraper.do_validation(users, threads, validation) print() job_role_count = role_occurrence.count(users, total_employees) if valid_emails_only: logger.valid_emails_only(users, filename) else: logger.dump(users, validation) logger.write_out(users, data, job_role_count, filename) return users
def parse_users(data,userdata_per_page,total_employees): cookie = data.cookie company_id = data.company_id email_format = data.email_format keyword = data.keyword domain = data.domain validation = data.validation api_key = data.api_key validation_count = 0 logger.debug(str(vars(data))) # For every page, do some parsing. if domain.startswith('@'): domain=domain else: domain='@'+domain users = [] if validation: print() logger.yellow('Starting Validation') for user_data in userdata_per_page: for d in user_data['elements'][0]['elements']: #This goes one user at a time validation_count += 1 for user_data in userdata_per_page: for d in user_data['elements'][0]['elements']: #This goes one user at a time if 'com.linkedin.voyager.search.SearchProfile' in d['hitInfo'] and d['hitInfo']['com.linkedin.voyager.search.SearchProfile']['headless'] == False: try: industry = d['hitInfo']['com.linkedin.voyager.search.SearchProfile']['industry'] logger.debug(industry) except: industry = "" raw_firstname = d['hitInfo']['com.linkedin.voyager.search.SearchProfile']['miniProfile']['firstName'] raw_surname = d['hitInfo']['com.linkedin.voyager.search.SearchProfile']['miniProfile']['lastName'] profile_url = "https://www.linkedin.com/in/%s" % d['hitInfo']['com.linkedin.voyager.search.SearchProfile']['miniProfile']['publicIdentifier'] occupation = d['hitInfo']['com.linkedin.voyager.search.SearchProfile']['miniProfile']['occupation'] location = d['hitInfo']['com.linkedin.voyager.search.SearchProfile']['location'] try: role_data=d['hitInfo']['com.linkedin.voyager.search.SearchProfile']['snippets'][0]['heading']['text'] try: current_role=role_data.split(' at ')[0] current_company=role_data.split(' at ')[1] except: current_company=None current_role=occupation except: try: current_company=occupation.split(' at ')[1] current_role=occupation.split(' at ')[0] except: current_company=None current_role=occupation name_data=[raw_firstname,raw_surname] logger.debug(str(name_data)) name_scheme=naming_scheme.names(name_data) firstname=name_scheme[0] middlename=name_scheme[1] surname=name_scheme[2] fullname=name_scheme[3] name_data=[firstname,middlename,surname] email_scheme=naming_scheme.emails(name_data,email_format,domain) email = email_scheme try: datapoint_1=d['hitInfo']['com.linkedin.voyager.search.SearchProfile']['miniProfile']['picture']['com.linkedin.common.VectorImage']['rootUrl'] datapoint_2=d['hitInfo']['com.linkedin.voyager.search.SearchProfile']['miniProfile']['picture']['com.linkedin.common.VectorImage']['artifacts'][2]['fileIdentifyingUrlPathSegment'] picture=datapoint_1+datapoint_2 logger.debug(picture) except: picture = None if validation != None: validation_count-=1 if validation == 'o365': validated=o365_validation.validate(email) elif validation == 'hunter': validated=hunter_validation.validate(email,api_key) if validated == 429: logger.red('You have exceeded your hunter API Requests.') quit() elif validated == 401: logger.red('The API Key specified recieved an %s error.' % 'authentication') quit() else: validated=False else: validated = False if validation: logger.verbose('%s emails remaining...' % logger.YELLOW(validation_count)) user=user_structure.User(profile_url,picture,firstname,middlename,surname,fullname,email,validated,current_role,current_company) users.append(user) if validation: logger.yellow('Validation finished!') print() return users
if not args.single: try: while True: sleep(1) counter -= 1 logger.timer('Querying in %s seconds(s)' % logger.BLUE(counter)) if counter <= 0: runner.go() counter = interval except KeyboardInterrupt as e: logger.red('CTRL+C Detected!') quit() else: runner.go() subdomains, wildcards = db.query() if args.probe: logger.yellow('Running HTTP probe') sleep(3) probed = probe.do(subdomains, args.threads) if args.probe: log_results = db.log_results((subdomains, wildcards), probed) else: log_results = db.log_results((subdomains, wildcards), None) quit() else: log_results = db.log_results(db.query()) quit()
def init(): if not os.path.isfile(db_name): logger.yellow('Creating database [%s]' % logger.YELLOW(db_name)) db = TinyDB(db_name) return db