예제 #1
0
파일: tasks.py 프로젝트: sdonepudy/RAPID
def ip_hosts(self, ip_address):

    current_time = datetime.datetime.utcnow()
    scraper = RobtexScraper()
    hosts = scraper.run(ip_address)
    ip_location = geolocate_ip(ip_address),
    source = "Robtex"

    if hosts:
        HostRecord.objects.bulk_create([
            HostRecord(domain_name=host,
                       ip_address=ip_address,
                       ip_location=ip_location,
                       resolution_date=current_time,
                       resolution_source=source,
                       query_keyword=ip_address,
                       query_date=current_time) for host in hosts
        ])
예제 #2
0
파일: tasks.py 프로젝트: armyninja/RAPID
def ip_hosts(self, ip_address):

    current_time = datetime.datetime.utcnow()
    scraper = RobtexScraper()
    hosts = scraper.run(ip_address)
    ip_location = geolocate_ip(ip_address)

    if type(hosts) == list:
        for host in hosts:
            try:
                record_entry = IndicatorRecord(record_type="HR",
                                               info_source="REX",
                                               info_date=current_time,
                                               info=OrderedDict({"geo_location": ip_location,
                                                                 "ip": ip_address, "domain": host}))
                record_entry.save()
            except Exception as e:
                print(e)
예제 #3
0
파일: tasks.py 프로젝트: rgaffar/RAPID
def ip_hosts(ip_address):
    scraper = RobtexScraper()
    hosts = scraper.run(ip_address)
    ip_location = geolocate_ip(ip_address)
    https_cert = lookup_ip_censys_https(ip_address)

    if type(hosts) == list:
        record_type = RecordType.HR
        record_source = RecordSource.REX
        for host in hosts:
            try:
                info = OrderedDict({"geo_location": ip_location,
                                    "https_cert": https_cert,
                                    "ip": ip_address, "domain": host})
                save_record(record_type,
                            record_source,
                            info)
            except Exception:
                logger.exception("Error saving %s (%s) record from %s",
                                 record_type.name,
                                 record_type.title,
                                 record_source.title)
예제 #4
0
GEOLOCATION_KEY = "geo_location"
"""The key used in JSON for the geo-location of an IP address"""

COUNTRY_KEY = "country"
"""
The key used in JSON for the location.  This key serves double duty, being used both within the geo-location for the
full country name (e.g. "United States") and within certificate monitors for the resolved country code (e.g. "US").
"""

IP_KEY = "ip"
"""The key used in JSON for an IP address"""

DOMAIN_KEY = "domain"
"""The key used in JSON for a domain name"""

__scraper = RobtexScraper()
"""The scraper used to resolve domain names for IP addresses"""


def get_domains_for_ip(ip):
    """
    Get the list of domains associated with an IP address.
    :param ip:
    :return:
    """
    return __scraper.run(ip)


def enclose_periods_in_braces(value):
    """
    Perform sanitization by enclosing any periods in square braces.
예제 #5
0
파일: tasks.py 프로젝트: rgaffar/RAPID
    def run(self, **kwargs):
        start_timestamp = datetime.datetime.utcnow()
        minute_timestamp = start_timestamp.strftime('%Y-%m-%d %H:%M')

        current_time = datetime.datetime.strptime(minute_timestamp,
                                                  '%Y-%m-%d %H:%M')
        desired_time = current_time + datetime.timedelta(minutes=1)

        # Check for overdue domain monitors
        overdue_domains = DomainMonitor.objects.filter(
            next_lookup__lt=current_time)

        for overdue_domain in overdue_domains:
            overdue_domain.next_lookup = current_time + datetime.timedelta(
                minutes=5)
            overdue_domain.save()

        # Check for overdue IP address monitors
        overdue_ips = IpMonitor.objects.filter(next_lookup__lt=current_time)

        for overdue_ip in overdue_ips:
            overdue_ip.next_lookup = current_time + datetime.timedelta(
                minutes=5)
            overdue_ip.save()

        # Compile list of domains to resolve based on lookup time
        domain_lookups = DomainMonitor.objects.filter(
            next_lookup__gte=current_time, next_lookup__lte=desired_time)

        # Compile list of IP addresses to resolve based on lookup time
        ip_lookups = IpMonitor.objects.filter(next_lookup__gte=current_time,
                                              next_lookup__lte=desired_time)

        # Lookup domain resolutions
        for domain_lookup in domain_lookups:
            owner = domain_lookup.owner
            last_hosts = domain_lookup.last_hosts
            domain_resolutions = resolve_domain(domain_lookup.domain_name)

            if type(domain_resolutions) == list:
                for host in domain_resolutions:

                    ip_location = geolocate_ip(host)

                    try:
                        record_entry = IndicatorRecord(
                            record_type="HR",
                            info_source="DNS",
                            info_date=current_time,
                            info={
                                "geo_location": ip_location,
                                "ip": host,
                                "domain": domain_lookup.domain_name
                            })
                        record_entry.save()
                    except:
                        pass

                if domain_resolutions and last_hosts:
                    # Check for new or missing hosts since last lookup
                    missing_hosts = list(
                        set(last_hosts).difference(domain_resolutions))
                    new_hosts = list(
                        set(domain_resolutions).difference(last_hosts))

                    # Sanitize domain name for safe email content
                    sanitized_domain = domain_lookup.domain_name.replace(
                        '.', '[.]')

                    email_recipient = [owner.email]

                    # Compose alert and email content for hosting changes
                    if missing_hosts and new_hosts:
                        sanitized_missing = [
                            host.replace('.', '[.]') for host in missing_hosts
                        ]
                        sanitized_new = [
                            host.replace('.', '[.]') for host in new_hosts
                        ]

                        alert_text = 'Removed hosts: %s' % ', '.join(
                            missing_hosts)
                        self.create_alert(domain_lookup.domain_name,
                                          alert_text, owner)

                        alert_text = 'New hosts: %s' % ', '.join(new_hosts)
                        self.create_alert(domain_lookup.domain_name,
                                          alert_text, owner)

                        email_subject = 'IP Address Changes for ' + sanitized_domain
                        email_body = """ DNS lookup performed at %s indicates that the tracked
                                         domain %s has dropped the following IP addresses: %s
                                         and has added the following IP addresses: %s
                                     """ % (str(current_time),
                                            sanitized_domain,
                                            sanitized_missing, sanitized_new)

                        deliver_email.delay(email_subject, email_body,
                                            email_recipient)

                    elif missing_hosts:
                        sanitized_missing = [
                            host.replace('.', '[.]') for host in missing_hosts
                        ]
                        alert_text = 'Removed hosts: %s' % ', '.join(
                            missing_hosts)
                        self.create_alert(domain_lookup.domain_name,
                                          alert_text, owner)

                        email_subject = 'IP Address Drops for ' + sanitized_domain
                        email_body = """ DNS lookup performed at %s indicates that the tracked
                                         domain %s has dropped the following IP addresses: %s
                                     """ % (str(current_time),
                                            sanitized_domain,
                                            sanitized_missing)

                        deliver_email.delay(email_subject, email_body,
                                            email_recipient)

                    elif new_hosts:
                        sanitized_new = [
                            host.replace('.', '[.]') for host in new_hosts
                        ]
                        alert_text = 'New hosts: %s' % ', '.join(new_hosts)
                        self.create_alert(domain_lookup.domain_name,
                                          alert_text, owner)

                        email_subject = 'IP Address Additions for ' + sanitized_domain
                        email_body = """ DNS lookup performed at %s indicates that the tracked
                                         domain %s has changed to the following IP addresses: %s
                                     """ % (str(current_time),
                                            sanitized_domain, sanitized_new)

                        deliver_email.delay(email_subject, email_body,
                                            email_recipient)
            else:
                alert_text = domain_resolutions
                self.create_alert(domain_lookup.domain_name, alert_text, owner)

            # Update entry information
            domain_lookup.last_hosts = domain_resolutions
            domain_lookup.next_lookup = current_time + datetime.timedelta(
                hours=domain_lookup.lookup_interval)
            domain_lookup.save()

        # Lookup IP address resolutions
        scraper = RobtexScraper()

        for ip_lookup in ip_lookups:
            owner = ip_lookup.owner
            last_hosts = ip_lookup.last_hosts
            ip_resolutions = scraper.run(ip_lookup.ip_address)
            ip_location = geolocate_ip(ip_lookup.ip_address)

            if type(ip_resolutions) == list:

                for host in ip_resolutions:
                    try:
                        record_entry = IndicatorRecord(
                            record_type="HR",
                            info_source="REX",
                            info_date=current_time,
                            info={
                                "geo_location": ip_location,
                                "ip": ip_lookup.ip_address,
                                "domain": host
                            })
                        record_entry.save()
                    except:
                        pass

                if ip_resolutions and last_hosts:
                    # Check for new or missing hosts since last lookup
                    missing_hosts = list(
                        set(last_hosts).difference(ip_resolutions))
                    new_hosts = list(
                        set(ip_resolutions).difference(last_hosts))

                    # Sanitize ip address for safe email content
                    sanitized_ip = ip_lookup.ip_address.replace('.', '[.]')

                    email_recipient = [owner.email]

                    # Compose alert and email content for hosting changes
                    if missing_hosts and new_hosts:
                        sanitized_missing = [
                            host.replace('.', '[.]') for host in missing_hosts
                        ]
                        sanitized_new = [
                            host.replace('.', '[.]') for host in new_hosts
                        ]

                        alert_text = 'Removed hosts: %s' % ', '.join(
                            missing_hosts)
                        self.create_alert(ip_lookup.ip_address, alert_text,
                                          owner)

                        alert_text = 'New hosts: %s' % ', '.join(new_hosts)
                        self.create_alert(ip_lookup.ip_address, alert_text,
                                          owner)

                        email_subject = 'Domain Changes for ' + sanitized_ip
                        email_body = """ IP lookup performed at %s indicates that the tracked
                                         IP address %s has dropped the following domains: %s
                                         and has added the following domains: %s
                                     """ % (str(current_time), sanitized_ip,
                                            sanitized_missing, sanitized_new)

                        deliver_email.delay(email_subject, email_body,
                                            email_recipient)

                    elif missing_hosts:
                        sanitized_missing = [
                            host.replace('.', '[.]') for host in missing_hosts
                        ]
                        alert_text = 'Removed hosts: %s' % ', '.join(
                            missing_hosts)
                        self.create_alert(ip_lookup.ip_address, alert_text,
                                          owner)

                        email_subject = 'Domain Drops for ' + sanitized_ip
                        email_body = """ IP lookup performed at %s indicates that the tracked
                                         IP address %s has dropped the following domains: %s
                                     """ % (str(current_time), sanitized_ip,
                                            sanitized_missing)

                        deliver_email.delay(email_subject, email_body,
                                            email_recipient)

                    elif new_hosts:
                        sanitized_new = [
                            host.replace('.', '[.]') for host in new_hosts
                        ]
                        alert_text = 'New hosts: %s' % ', '.join(new_hosts)
                        self.create_alert(ip_lookup.ip_address, alert_text,
                                          owner)

                        email_subject = 'Domain Additions for ' + sanitized_ip
                        email_body = """ IP lookup performed at %s indicates that the tracked
                                         IP address %s has added the following domains: %s
                                     """ % (str(current_time), sanitized_ip,
                                            sanitized_new)

                        deliver_email.delay(email_subject, email_body,
                                            email_recipient)
            else:
                alert_text = ip_resolutions
                self.create_alert(ip_lookup.ip_address, alert_text, owner)

            # Update entry information
            ip_lookup.last_hosts = ip_resolutions
            ip_lookup.next_lookup = current_time + datetime.timedelta(
                hours=ip_lookup.lookup_interval)
            ip_lookup.save()
예제 #6
0
파일: tasks.py 프로젝트: armyninja/RAPID
    def run(self, **kwargs):
        start_timestamp = datetime.datetime.utcnow()
        minute_timestamp = start_timestamp.strftime('%Y-%m-%d %H:%M')

        current_time = datetime.datetime.strptime(minute_timestamp, '%Y-%m-%d %H:%M')
        desired_time = current_time + datetime.timedelta(minutes=1)

        # Check for overdue domain monitors
        overdue_domains = DomainMonitor.objects.filter(next_lookup__lt=current_time)

        for overdue_domain in overdue_domains:
            overdue_domain.next_lookup = current_time + datetime.timedelta(minutes=5)
            overdue_domain.save()

        # Check for overdue IP address monitors
        overdue_ips = IpMonitor.objects.filter(next_lookup__lt=current_time)

        for overdue_ip in overdue_ips:
            overdue_ip.next_lookup = current_time + datetime.timedelta(minutes=5)
            overdue_ip.save()

        # Compile list of domains to resolve based on lookup time
        domain_lookups = DomainMonitor.objects.filter(next_lookup__gte=current_time,
                                                      next_lookup__lte=desired_time)

        # Compile list of IP addresses to resolve based on lookup time
        ip_lookups = IpMonitor.objects.filter(next_lookup__gte=current_time,
                                              next_lookup__lte=desired_time)

        # Lookup domain resolutions
        for domain_lookup in domain_lookups:
            owner = domain_lookup.owner
            last_hosts = domain_lookup.last_hosts
            domain_resolutions = resolve_domain(domain_lookup.domain_name)

            if type(domain_resolutions) == list:
                for host in domain_resolutions:

                    ip_location = geolocate_ip(host)

                    try:
                        record_entry = IndicatorRecord(record_type="HR",
                                                       info_source="DNS",
                                                       info_date=current_time,
                                                       info={"geo_location": ip_location,
                                                             "ip": host, "domain": domain_lookup.domain_name})
                        record_entry.save()
                    except:
                        pass

                if domain_resolutions and last_hosts:
                    # Check for new or missing hosts since last lookup
                    missing_hosts = list(set(last_hosts).difference(domain_resolutions))
                    new_hosts = list(set(domain_resolutions).difference(last_hosts))

                    # Sanitize domain name for safe email content
                    sanitized_domain = domain_lookup.domain_name.replace('.', '[.]')

                    email_recipient = [owner.email]

                    # Compose alert and email content for hosting changes
                    if missing_hosts and new_hosts:
                        sanitized_missing = [host.replace('.', '[.]') for host in missing_hosts]
                        sanitized_new = [host.replace('.', '[.]') for host in new_hosts]

                        alert_text = 'Removed hosts: %s' % ', '.join(missing_hosts)
                        self.create_alert(domain_lookup.domain_name, alert_text, owner)

                        alert_text = 'New hosts: %s' % ', '.join(new_hosts)
                        self.create_alert(domain_lookup.domain_name, alert_text, owner)

                        email_subject = 'IP Address Changes for ' + sanitized_domain
                        email_body = """ DNS lookup performed at %s indicates that the tracked
                                         domain %s has dropped the following IP addresses: %s
                                         and has added the following IP addresses: %s
                                     """ % (str(current_time), sanitized_domain,
                                            sanitized_missing, sanitized_new)

                        deliver_email.delay(email_subject, email_body, email_recipient)

                    elif missing_hosts:
                        sanitized_missing = [host.replace('.', '[.]') for host in missing_hosts]
                        alert_text = 'Removed hosts: %s' % ', '.join(missing_hosts)
                        self.create_alert(domain_lookup.domain_name, alert_text, owner)

                        email_subject = 'IP Address Drops for ' + sanitized_domain
                        email_body = """ DNS lookup performed at %s indicates that the tracked
                                         domain %s has dropped the following IP addresses: %s
                                     """ % (str(current_time), sanitized_domain, sanitized_missing)

                        deliver_email.delay(email_subject, email_body, email_recipient)

                    elif new_hosts:
                        sanitized_new = [host.replace('.', '[.]') for host in new_hosts]
                        alert_text = 'New hosts: %s' % ', '.join(new_hosts)
                        self.create_alert(domain_lookup.domain_name, alert_text, owner)

                        email_subject = 'IP Address Additions for ' + sanitized_domain
                        email_body = """ DNS lookup performed at %s indicates that the tracked
                                         domain %s has changed to the following IP addresses: %s
                                     """ % (str(current_time), sanitized_domain, sanitized_new)

                        deliver_email.delay(email_subject, email_body, email_recipient)
            else:
                alert_text = domain_resolutions
                self.create_alert(domain_lookup.domain_name, alert_text, owner)

            # Update entry information
            domain_lookup.last_hosts = domain_resolutions
            domain_lookup.next_lookup = current_time + datetime.timedelta(hours=domain_lookup.lookup_interval)
            domain_lookup.save()

        # Lookup IP address resolutions
        scraper = RobtexScraper()

        for ip_lookup in ip_lookups:
            owner = ip_lookup.owner
            last_hosts = ip_lookup.last_hosts
            ip_resolutions = scraper.run(ip_lookup.ip_address)
            ip_location = geolocate_ip(ip_lookup.ip_address)

            if type(ip_resolutions) == list:

                for host in ip_resolutions:
                    try:
                        record_entry = IndicatorRecord(record_type="HR",
                                                       info_source="REX",
                                                       info_date=current_time,
                                                       info={"geo_location": ip_location,
                                                             "ip": ip_lookup.ip_address, "domain": host})
                        record_entry.save()
                    except:
                        pass

                if ip_resolutions and last_hosts:
                    # Check for new or missing hosts since last lookup
                    missing_hosts = list(set(last_hosts).difference(ip_resolutions))
                    new_hosts = list(set(ip_resolutions).difference(last_hosts))

                    # Sanitize ip address for safe email content
                    sanitized_ip = ip_lookup.ip_address.replace('.', '[.]')

                    email_recipient = [owner.email]

                    # Compose alert and email content for hosting changes
                    if missing_hosts and new_hosts:
                        sanitized_missing = [host.replace('.', '[.]') for host in missing_hosts]
                        sanitized_new = [host.replace('.', '[.]') for host in new_hosts]

                        alert_text = 'Removed hosts: %s' % ', '.join(missing_hosts)
                        self.create_alert(ip_lookup.ip_address, alert_text, owner)

                        alert_text = 'New hosts: %s' % ', '.join(new_hosts)
                        self.create_alert(ip_lookup.ip_address, alert_text, owner)

                        email_subject = 'Domain Changes for ' + sanitized_ip
                        email_body = """ IP lookup performed at %s indicates that the tracked
                                         IP address %s has dropped the following domains: %s
                                         and has added the following domains: %s
                                     """ % (str(current_time), sanitized_ip,
                                            sanitized_missing, sanitized_new)

                        deliver_email.delay(email_subject, email_body, email_recipient)

                    elif missing_hosts:
                        sanitized_missing = [host.replace('.', '[.]') for host in missing_hosts]
                        alert_text = 'Removed hosts: %s' % ', '.join(missing_hosts)
                        self.create_alert(ip_lookup.ip_address, alert_text, owner)

                        email_subject = 'Domain Drops for ' + sanitized_ip
                        email_body = """ IP lookup performed at %s indicates that the tracked
                                         IP address %s has dropped the following domains: %s
                                     """ % (str(current_time), sanitized_ip, sanitized_missing)

                        deliver_email.delay(email_subject, email_body, email_recipient)

                    elif new_hosts:
                        sanitized_new = [host.replace('.', '[.]') for host in new_hosts]
                        alert_text = 'New hosts: %s' % ', '.join(new_hosts)
                        self.create_alert(ip_lookup.ip_address, alert_text, owner)

                        email_subject = 'Domain Additions for ' + sanitized_ip
                        email_body = """ IP lookup performed at %s indicates that the tracked
                                         IP address %s has added the following domains: %s
                                     """ % (str(current_time), sanitized_ip, sanitized_new)

                        deliver_email.delay(email_subject, email_body, email_recipient)
            else:
                alert_text = ip_resolutions
                self.create_alert(ip_lookup.ip_address, alert_text, owner)

            # Update entry information
            ip_lookup.last_hosts = ip_resolutions
            ip_lookup.next_lookup = current_time + datetime.timedelta(hours=ip_lookup.lookup_interval)
            ip_lookup.save()
예제 #7
0
파일: tasks.py 프로젝트: sdonepudy/RAPID
    def run(self, **kwargs):
        start_timestamp = datetime.datetime.utcnow()
        minute_timestamp = start_timestamp.strftime('%Y-%m-%d %H:%M')

        self.current_time = datetime.datetime.strptime(minute_timestamp, '%Y-%m-%d %H:%M')
        self.desired_time = self.current_time + datetime.timedelta(minutes=1)

        # Check for any overdue lookups
        self.check_overdue()

        ip_lookups = IpMonitor.objects.filter(next_lookup__gte=self.current_time,
                                              next_lookup__lte=self.desired_time)

        scraper = RobtexScraper()

        for lookup in ip_lookups:
            last_hosts = lookup.last_hosts
            ip_resolutions = scraper.run(lookup.ip_address)

            if ip_resolutions:
                HostRecord.objects.bulk_create([
                    HostRecord(domain_name=host,
                               ip_address=lookup.ip_address,
                               resolution_date=self.current_time,
                               resolution_source="Robtex",
                               query_keyword=lookup.ip_address,
                               query_date=self.current_time) for host in ip_resolutions
                ])

            if ip_resolutions and last_hosts:
                # Check for new or missing hosts since last lookup
                missing_hosts = list(set(last_hosts).difference(ip_resolutions))
                new_hosts = list(set(ip_resolutions).difference(last_hosts))

                # Sanitize ip address for safe email content
                sanitized_ip = lookup.ip_address.replace('.', '[.]')

                # Compile list of email recipients for a given IP address indicator
                email_recipients = User.objects.filter(ip_list__contains=[lookup.ip_address],
                                                       alerts=True).values_list('email', flat=True)

                # Compose alert and email content for hosting changes
                if missing_hosts and new_hosts:
                    sanitized_missing = [host.replace('.', '[.]') for host in missing_hosts]
                    sanitized_new = [host.replace('.', '[.]') for host in new_hosts]

                    alert_text = 'Removed hosts: %s' % ', '.join(missing_hosts)
                    self.create_alert(lookup.ip_address, alert_text)

                    alert_text = 'New hosts: %s' % ', '.join(new_hosts)
                    self.create_alert(lookup.ip_address, alert_text)

                    email_subject = 'Domain Changes for ' + sanitized_ip
                    email_body = """ IP lookup performed at %s indicates that the tracked
                                     IP address %s has dropped the following domains: %s
                                     and has added the following domains: %s
                                 """ % (str(self.current_time), sanitized_ip,
                                        sanitized_missing, sanitized_new)

                    deliver_email.delay(email_subject, email_body, email_recipients)

                elif missing_hosts:
                    sanitized_missing = [host.replace('.', '[.]') for host in missing_hosts]
                    alert_text = 'Removed hosts: %s' % ', '.join(missing_hosts)
                    self.create_alert(lookup.ip_address, alert_text)

                    email_subject = 'Domain Drops for ' + sanitized_ip
                    email_body = """ IP lookup performed at %s indicates that the tracked
                                     IP address %s has dropped the following domains: %s
                                 """ % (str(self.current_time), sanitized_ip, sanitized_missing)

                    deliver_email.delay(email_subject, email_body, email_recipients)

                elif new_hosts:
                    sanitized_new = [host.replace('.', '[.]') for host in new_hosts]
                    alert_text = 'New hosts: %s' % ', '.join(new_hosts)
                    self.create_alert(lookup.ip_address, alert_text)

                    email_subject = 'Domain Additions for ' + sanitized_ip
                    email_body = """ IP lookup performed at %s indicates that the tracked
                                     IP address %s has added the following domains: %s
                                 """ % (str(self.current_time), sanitized_ip, sanitized_new)

                    deliver_email.delay(email_subject, email_body, email_recipients)

            # Update entry information
            lookup.last_hosts = ip_resolutions
            lookup.last_lookup = self.current_time
            lookup.next_lookup = self.current_time + datetime.timedelta(hours=lookup.lookup_interval)
            lookup.save()