def dump_bucket(self, dry_run=False): """ Create download workers Fetch and organize a list of Objects to be downloaded pass to workers """ logger.info("Downloading bucket...") self._dry_run = dry_run _download_workers = [ multiprocessing.Process(target=self._download_file) for _ in range(self._max_workers) ] [w.start() for w in _download_workers] _objects = self._s3_client.list_objects() for _obj in _objects: logger.debug(f"Checking if '{_obj.key}' is downloaded") if not self._is_file_exists(_obj.key): self._download_queue.put(_obj.key) self._files_to_be_downloaded.release() # Telling workers to stop for _ in range(0, self._max_workers): self._download_queue.put(None) self._files_to_be_downloaded.release() # Joining workers [w.join() for w in _download_workers] logger.info("Bucket downloaded")
def reset_configs(self): logger.warning(f"Deleting config file {self._CONFIG_FILE}") if os.path.exists(self._CONFIG_FILE): os.remove(self._CONFIG_FILE) logger.info("Config file successfully removed") self._config = self._default_config return self._config
def run_rules(conf): data = rds.get_scan_data() exclusions = rds.get_exclusions() if not data: return for ip, values in data.items(): rules = rule_manager(role='attacker') if 'ports' in values and len(values['ports']) > 0: for port in values['ports']: logger.info('Attacking Asset: {} on port: {}'.format(ip, port)) for rule in rules.values(): """ Check if the target is in exclusions list, if it is, skip. """ if rule.rule in exclusions and ip in exclusions[rule.rule]: logger.debug('Skipping rule {} for target {}'.format( rule.rule, ip)) continue """ Only run rules that are in the allowed_aggressive config level. """ if conf['config']['allow_aggressive'] >= rule.intensity: thread = threading.Thread(target=rule.check_rule, args=(ip, port, values, conf)) thread.start()
def validate_unique(self): """ By default, the form will complain if a user already exists with that username. But that message is confusing for Site admins who don't have a User with that username on their particular site. This function changes the error message so it's non-confusng, but also vague enough not to suggest that other Sites exist. """ # Ensure our username is not taken in LDAP; we have to un-namespace it before doing the LDAP lookup. if self.cleaned_data.get('username').startswith(self.request.site.hostname + "-"): username = self.cleaned_data.get('username')[len(self.request.site.hostname)+1:] try: ldap_user = search_ldap_for_user(username) except ldap.LDAPError: # Not taken in LDAP logger.exception('usr.local.ldap-check.exception', target_user=username) if ldap_user: logger.info('usr.local.ldap.taken', target_user=username) self.add_error('username', forms.ValidationError(self.error_messages['duplicate_username_in_ldap'])) try: get_user_model().objects.get(username=self.cleaned_data.get('username')) except get_user_model().DoesNotExist: pass else: self.add_error('username', forms.ValidationError(self.error_messages['duplicate_username_local']))
def quickstart(): if request.method == 'POST': # In Quickstart, we only take the network provided by the user as input # The rest is as defined in config.py network = request.values.get('network') if network: scan = copy.deepcopy(config.DEFAULT_SCAN) scan['targets']['networks'].append(network) schema = SchemaParser(scan, request) vfd, msg, scan = schema.verify() if vfd: res, code, msg = register.scan(scan) if res: logger.info('A scan was initiated') flash('Assessment started.', 'success') return redirect('/qs') else: flash(msg, 'error') else: flash(msg, 'error') return render_template('quickstart.html')
def email(email, subject, body, html=None): """ Sends email """ logger.info(f'Sending email to {email}... ' f'Subject: {subject} ' f'Body: {body}') msg = Message(subject, to='%s <%s>' % (email, email), text=body, html=html) gmail.send(msg)
def get(self, request, format=None): data = { "status": "up", "build_version": settings.BUILD_VERSION, "build_date": settings.BUILD_DATE, } logger.info("Health check") return Response(data)
def get(self, request, format=None): data = { 'status': 'up', 'build_version': settings.BUILD_VERSION, 'build_date': settings.BUILD_DATE } logger.info('Health check') return Response(data)
def store_vuln(self, value): logger.info('Vulnerability detected') key = '{}{}{}{}'.format(value['ip'], value['port'], value['rule_id'], value['rule_details']) key_hash = 'vuln_' + self.utils.hash_sha1(key) if self.r.exists(key_hash): return False self.store_json(key_hash, value)
def parse_args_f(): logger.info(f"sys.argv: {sys.argv}") parser = argparse.ArgumentParser(description='description here') parser.add_argument('-v', '--version', action='version', version='%(prog)s 0.1') parser.add_argument('-a', '--arg1', required=True, help='this is for arg1') parser.add_argument('-b', '--arg2', required=True, help='this is for arg2') return parser.parse_args()
def get_sync_status(self, kwargs): files = self.syncer.scan(recursive=True) synced = len( list( filter(lambda f: len(f) > 2 and f[2] in ("NOT_UPLOADED", "NOT_SYNCED"), files) ) ) == 0 logger.info(f"Target path scanned. Sync status: {'all synced' if synced else 'not synced'}") return {"synced": synced}
def _upload_file(self, rel_file_path, md5sum): if self._dry_run: return logger.info(f"Uploading file... ({rel_file_path})") self._client.put_object( object_key=rel_file_path, file_path=self._get_abs_file_path(rel_file_path), metadata={'md5sum': md5sum}) self._set_last_synced(rel_file_path=rel_file_path) logger.info(f"File is uploaded! ({rel_file_path})")
def scan(self, kwargs): if not self.syncer.has_bucket_name(): self.syncer.set_bucket_name( self.config_manager.get("DEFAULT_BUCKET_NAME") ) path = self.config_manager.get("TARGET_PATH") \ if kwargs.get("path") is None \ else kwargs["path"] logger.info(f"Scanning directory: {path}") return self.syncer.scan(path, recursive=False)
def submit_webhook(self, webhook, cfg, data={}): logger.info('Sending the webhook...') try: data = {'status':'done', 'vulnerabilities':data, 'scan_config':cfg} requests.post(webhook, json=data, headers={'User-Agent':USER_AGENT, 'Content-Type':'application/json'}, verify=False) return True except Exception as e: logger.error('Could not submit webhook: {}'.format(e)) return
def redis_attack(self, ip, port, password): try: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((ip, port)) payload = 'AUTH {}\n'.format(password) s.sendall(payload.encode()) data = s.recv(1024) logger.info(data) if 'OK' in data.decode('utf-8'): return True except Exception as e: logger.error(e) return False finally: s.close() return False
def save(self, commit=True): """ If a Django User with this username already exists, pull it from the DB and add the specified Groups to it, instead of creating a new User object. """ user = super(LDAPUserCreateForm, self).save(commit=False) # Users can access django-admin iff they are a superuser. user.is_staff = user.is_superuser username = self.cleaned_data['username'] groups = Group.objects.filter(pk__in=self.cleaned_data['groups']).all() logger_extras = { 'target_user': username, 'target_user_superuser': user.is_superuser, 'groups': ", ".join(str(g) for g in groups) } try: existing_user = get_user_model().objects.get(username=username) except get_user_model().DoesNotExist: existing_user = False else: user = existing_user for group in groups: user.groups.add(group) if existing_user: logger_extras['target_user_id'] = user.id populate_user_from_ldap(user) if commit: user.save() if not existing_user: # Set a random password for the user, otherwise they end up with # '' as their password. If the LDAP user later gets removed # from LDAP (because they left, for instance), this # account reverts to being a local user and we don't want anyone # to be able to login as them. password = get_user_model().objects.make_random_password() user.set_password(password) # Only call save_m2m() if we're not updating an existing User. It'll try to overwrite the updated # user.groups list, AND it'll crash for some reason I haven't figured out. self.save_m2m() logger.info('user.ldap.create', **logger_extras) else: logger.info('user.ldap.update', **logger_extras) return user
def login(request): # Since we're not logged in yet, get_logger()'s logger will not be bound # with the username, so we add that here. if request.user.is_authenticated and request.user.has_perm( 'wagtailadmin.access_admin'): # User is already logged in. Just redirect them to wagtail home. return redirect('wagtailadmin_home') else: # This code is adapted from django.contrib.auth.login(), to allow us to log login successes and failures. username = request.POST.get(get_user_model().USERNAME_FIELD) redirect_to = request.POST.get( REDIRECT_FIELD_NAME, request.GET.get(REDIRECT_FIELD_NAME, '')) if request.method == 'POST': form = MultitenantLoginForm(request, data=request.POST) if form.is_valid(): # Ensure the user-originating redirection url is safe. if not is_safe_url(url=redirect_to, allowed_hosts=[request.get_host()]): redirect_to = resolve_url(settings.LOGIN_REDIRECT_URL) # Okay, security check complete. Log the user in. auth_login(request, form.get_user()) # Normally we wouldn't need to override the username here, because request.user is now a real User, but # because a LocalUser's username is prefixed with their Site's hostname, we log the username string # they actually logged in with. logger.info('auth.login.success', username=username) return HttpResponseRedirect(redirect_to) else: logger.warning('auth.login.failed', username=username) else: form = MultitenantLoginForm(request) current_site = get_current_site(request) context = { 'form': form, REDIRECT_FIELD_NAME: redirect_to, 'site': current_site, 'site_name': current_site.name, 'show_password_reset': password_reset_enabled(), 'username_field': get_user_model().USERNAME_FIELD, } return TemplateResponse(request, 'wagtailadmin/login.html', context)
def remove_ldap_user(request, user_id): if hasattr(request, 'site'): if request.method == 'GET': user = get_object_or_404(get_user_model(), pk=user_id) site_hostname = request.site.hostname for group_name in ['Admins', 'Editors']: g = Group.objects.get( name="{} {}".format(site_hostname, group_name)) g.user_set.remove(user) msg = "{} {} ({}) no longer has admin rights on {}.".format( user.first_name, user.last_name, user.username, site_hostname) messages.success(request, mark_safe(msg)) logger.info('user.ldap.disabled-for-site', target_user=user.username) else: msg = "You must be on a particular site to remove an LDAP user." messages.error(request, mark_safe(msg)) return redirect(reverse('wagtailusers_users:index'))
def save(self, commit=True): user = super(LocalUserCreateForm, self).save(commit=False) # Users can access django-admin iff they are a superuser. user.is_staff = user.is_superuser user.set_password(self.cleaned_data['password1']) if commit: user.save() # List the quoted group names for logging. group_names = [str(g) for g in Group.objects.filter(pk__in=self.cleaned_data['groups']).all()] if user.is_superuser: group_names.append('Superusers') self.save_m2m() logger.info( 'user.local.create', target_user=user.username, target_user_superuser=user.is_superuser, groups=", ".join(group_names) ) return user
def run_rules(conf): data = rds.get_scan_data() if not data: return for ip, values in data.items(): rules = rule_manager(role='attacker') if 'ports' in values and len(values['ports']) > 0: for port in values['ports']: logger.info('Attacking Asset: {} on port: {}'.format(ip, port)) for rule in rules.values(): if conf['config']['allow_aggressive'] >= rule.intensity: thread = threading.Thread(target=rule.check_rule, args=(ip, port, values, conf)) thread.name = 'rule_{}_{}_{}'.format( rule.rule, ip, port) thread.start()
def wrapper(*args, **kwargs): lock_name = kwargs.pop('lock_name', os.environ.get('SERVER_DOMAIN', 'oursites.com')) + f.__name__ timeout = kwargs.pop('timeout', 60 * 5) have_lock = False client = redis.Redis(host=os.environ.get('CACHE'), port=6379, db=2) lock = client.lock(lock_name, timeout=timeout) try: have_lock = lock.acquire(blocking=False) if have_lock: f(*args, **kwargs) else: logger.info('celery.task.lock.already_locked', task=f.__name__) finally: if have_lock: lock.release()
def _download_file(self): while self._files_to_be_downloaded.acquire(): object_key = self._download_queue.get() if object_key is None: break # A sentinel value to quit the loop if self._dry_run: return logger.info(f"Downloading file... ({object_key})") _file_path = os.path.abspath( os.path.join(self._target_path, object_key)) _dir_name = os.path.dirname(_file_path) os.makedirs(_dir_name, exist_ok=True) self._s3_client.download_object(object_key=object_key, file_path=_file_path) self._download_queue.task_done()
def save(self, commit=True): """ In case the data in LDAP has changed, or it failed to populate on the previous create/edit, we override save() to re-populate this User's personal info from LDAP. """ user = super(LDAPUserEditForm, self).save(commit=False) # Users can access django-admin iff they are a superuser. user.is_staff = user.is_superuser populate_user_from_ldap(user) if commit: user.save() self.save_m2m() if self.has_changed(): logger_extras = { 'target_user': user.username, 'target_user_superuser': user.is_superuser, } for field_name in self.changed_data: logger_extras[field_name] = self.cleaned_data[field_name] logger.info('user.ldap.update', **logger_extras) return user
def populate_user_from_ldap(user): """ Populate the given User object's personal info from LDAP. """ try: _, ldap_attrs = search_ldap_for_user(user.username) user.first_name = ldap_attrs['givenName'][0] user.last_name = ldap_attrs['sn'][0] user.email = ldap_attrs['CAPPrimaryEmail'][0] except (IndexError, KeyError, TypeError, ldap.LDAPError) as err: # If any of these attrs are missing, or anything goes wrong, fail gracefully instead of crashing. # These attrs are not essential, and will be updated if possible the next time this User is saved. logger.error('user.update_from_ldap.failed', target_user=user.username, reason="{}: {}".format(err.__class__.__name__, err)) else: logger.info('user.update_from_ldap.success', target_user=user.username, first_name=user.first_name, last_name=user.last_name, email=user.email)
def submit_slack(self, hook, data={}): try: if not data: logger.info( 'Did not send slack notification, scan did not yield any result.' ) return fields = [] for _, value in data.items(): if value['rule_sev'] == 0: continue for k, v in value.items(): if not v: v = 'N/A' fields.append({'title': k, 'value': v, 'short': False}) slack_data = { "color": '#000000', "pretext": "<!channel> NERVE Notification", "author_name": ':warning: Notification', "title": 'NERVE Report', "fields": fields, } response = requests.post(hook, data=json.dumps(slack_data)) if response.status_code != 200: logger.error('Could not submit slack hook: {}'.format( response.text)) else: logger.info('Submitted slack hook') except Exception as e: logger.error('Could not submit slack hook: {}'.format(e)) return
def logout(request): """ This code is identical to wagtail.wagtailadmin.views.account.logout, except for the post-logout redirect. """ # The next_page argument is here just to make the function return faster. We don't be using this response. response = auth_views.LogoutView.as_view()(request, next_page='/') logger.info('auth.logout') messages.success(request, 'You have logged out.') # By default, logging out will generate a fresh sessionid cookie. We want to use the # absence of sessionid as an indication that front-end pages are being viewed by a # non-logged-in user and are therefore cacheable, so we forcibly delete the cookie here. response.delete_cookie(settings.SESSION_COOKIE_NAME, settings.SESSION_COOKIE_PATH, settings.SESSION_COOKIE_DOMAIN) # HACK: pretend that the session hasn't been modified, so that SessionMiddleware # won't override the above and write a new cookie. request.session.modified = False # Just in case something is messed up with the Site's settings, avoid a crash. try: alias_count = request.site.settings.aliases.count() except: alias_count = 0 # If we can get the referer, we use it as long as it's not an admin page (which would redirect back to the # login prompt). The user's already there, so we know it won't give a cert error. referer = request.META.get('HTTP_REFERER') if referer and '/admin/' not in referer: redirect_url = referer elif alias_count == 1: # If there is exactly one alias, redirect to the homepage of that domain. alias = request.site.settings.aliases.first() redirect_url = 'http://{}'.format(alias.domain) else: # We can't know which to pick among multiple aliases, so we fall back on the homepage of the canonical hostname. redirect_url = 'http://{}'.format(request.site.hostname) return HttpResponseRedirect(redirect_url)
def attacker(): count = 0 logger.info('Attacker process started') while True: conf = rds.get_scan_config() if not conf: time.sleep(10) continue run_rules(conf) count += 1 if count == conf['config']['scan_opts']['parallel_attack']: time.sleep(30) count = 0 if threading.active_count() > 50: logger.debug( 'Sleeping for 30 seconds to control threads (Threads: {})'. format(threading.active_count())) time.sleep(30)
def save(self, request): user = get_object_or_404(get_user_model(), username=self.cleaned_data['username']) kwargs = { 'uidb64': urlsafe_base64_encode(force_bytes(user.id)), 'token': default_token_generator.make_token(user) } reset_password_path = reverse('wagtailadmin_password_reset_confirm', kwargs=kwargs) domain = request.site.hostname context = { 'user': user, 'reset_password_url': 'https://{}{}'.format(domain, reset_password_path), 'domain': domain, } self.send_mail( 'wagtail_patches/users/reset_password_email_subject.txt', 'wagtail_patches/users/reset_password_email.txt', context, from_email=settings.SERVER_EMAIL, to_email=user.email ) logger.info( 'user.local.password_reset.admin', target_user=user.username, target_user_superuser=user.is_superuser, target_user_email=user.email )
def scanner(): utils = Utils() scanner = Scanner() logger.info('Scanner process started') while True: if not rds.is_session_active(): time.sleep(10) continue conf = rds.get_scan_config() if not conf: time.sleep(10) continue hosts = rds.get_ips_to_scan( limit=conf['config']['scan_opts']['parallel_scan']) if hosts: conf = rds.get_scan_config() scan_data = scanner.scan( hosts, max_ports=conf['config']['scan_opts']['max_ports'], custom_ports=conf['config']['scan_opts']['custom_ports'], interface=conf['config']['scan_opts']['interface']) if scan_data: for host, values in scan_data.items(): if 'ports' in values and values['ports']: logger.info('Discovered Asset: {}'.format(host)) logger.debug('Host: {}, Open Ports: {}'.format( host, values['ports'])) rds.store_topology(host) rds.store_sca(host, values) rds.store_inv(host, values) else: if values['status_reason'] == 'echo-reply': logger.info('Discovered Asset: {}'.format(host)) rds.store_topology(host)
def scanner(): scanner = Scanner() logger.info('Scanner process started') while True: if not rds.is_session_active(): time.sleep(10) continue conf = rds.get_scan_config() if not conf: time.sleep(10) continue c = ConfParser(conf) hosts = rds.get_ips_to_scan(limit=c.get_cfg_scan_threads()) if hosts: conf = rds.get_scan_config() scan_data = scanner.scan(hosts, max_ports=c.get_cfg_max_ports(), custom_ports=c.get_cfg_custom_ports(), interface=c.get_cfg_netinterface()) if scan_data: for host, values in scan_data.items(): if 'ports' in values and values['ports']: logger.info('Discovered Asset: {}'.format(host)) logger.debug('Host: {}, Open Ports: {}'.format( host, values['ports'])) rds.store_topology(host) rds.store_sca(host, values) rds.store_inv(host, values) else: if values['status_reason'] == 'echo-reply': logger.info('Discovered Asset: {}'.format(host)) rds.store_topology(host)