def instance_edit(request, hostname):
if not validators.domain(hostname) and not validators.ipv4(hostname):
return redirect('config:instances')
vinfo = viewinfo.prepare(request, "Instance edit " + hostname)
try:
instance = Instance.objects.get(hostname=hostname)
except Instance.DoesNotExist:
return redirect('config:instances')
if len(request.POST) > 0:
error = False
instance.alias = request.POST.get('alias').strip()
instance.hostname = request.POST.get('hostname').strip()
instance.port = request.POST.get('port').strip()
try:
port_number = int(instance.port)
except:
port_number = 0
instance.key = request.POST.get('key').strip()
if 'https' in request.POST:
instance.https = True
else:
instance.https = False
if not validator_letters_numbers(instance.alias):
error = True
vinfo.msg.add_error("Alias need to be letters and numbers only")
if not validators.domain(instance.hostname) and not validators.ipv4(
instance.hostname):
error = True
vinfo.msg.add_error("Hostname need to be only domain or ipv4")
if not validator_letters_numbers(instance.key):
error = True
vinfo.msg.add_error("Key need to be letters and numbers only")
if not validators.between(port_number, 1, 65535):
error = True
vinfo.msg.add_error("Port need to be between 1 and 65535")
if error == False:
instance.save()
context = viewinfo.context(vinfo)
localcontext = {
'instance': instance,
}
context.update(localcontext)
return render(request, 'config/instance_edit.html', context)
def process_iocs(results):
"""Return data formatted for Splunk from Malshare."""
if results != None:
provided_iocs = [y for x in results for y in x.values()]
else:
provided_iocs = sys.argv[1:]
session = commons.create_session()
api_key = commons.get_apikey("malshare")
splunk_table = []
for provided_ioc in set(provided_iocs):
provided_ioc = commons.deobfuscate_string(provided_ioc)
provided_ioc = provided_ioc.lower()
if validators.ipv4(provided_ioc) or validators.domain(provided_ioc) or \
re.match("^[a-f\d]{32}$", provided_ioc) or re.match("^[a-f\d]{64}$", provided_ioc):
pass
else:
splunk_table.append({"invalid": provided_ioc})
continue
ioc_dicts = query_malshare(provided_ioc, api_key, session)
for ioc_dict in ioc_dicts:
splunk_table.append(ioc_dict)
session.close()
return splunk_table
def isValidIP(ip: str):
"""
* Validate IP address\n
* @param string ip\n
* @return boolean
"""
return bool(validators.ipv4(ip))
def run(self, params={}):
threat_identifier = params.get(Input.THREAT_IDENTIFIER)
threats = self.connection.client.search_threats([threat_identifier])
if len(threats) > 1:
self.logger.info(
f"Multiple threats found that matched the query: {threat_identifier}."
"We will only act upon the first match")
payload = {"threat_id": threats[0].get('sha256')}
if params.get(Input.QUARANTINE_STATE):
payload["event"] = "Quarantine"
else:
payload["event"] = "Waive"
# If IPv4, attempt to find its ID
agent = params.get(Input.AGENT)
if validators.ipv4(agent):
agent = find_agent_by_ip(self.connection, agent)
errors = self.connection.client.update_agent_threat(
self.connection.client.get_agent_details(agent).get('id'), payload)
if len(errors) != 0:
raise PluginException(
cause=
'The response from the CylancePROTECT API was not in the correct format.',
assistance='Contact support for help. See log for more details',
data=errors)
return {Output.SUCCESS: True}
def search(self):
mod.display(self.module_name, "", "INFO", "Searching...")
url = "https://openphish.com/"
paths = ["feed.txt"]
for path in paths:
content = Cache(self.module_name, url, path,
self.search_method).content
for line in content.split("\n"):
try:
midle = line.split("//")[-1].split("/")[0]
except:
midle = None
if self.type == "URL":
if self.ioc in line:
mod.display(self.module_name, self.ioc, "FOUND",
"%s%s" % (url, path))
return
elif self.type == "IPv4" and validators.ipv4(midle):
if self.ioc == midle:
mod.display(self.module_name, self.ioc, "FOUND",
"%s%s" % (url, path))
return
elif self.type == "domain" and validators.domain(midle):
if midle == self.ioc:
mod.display(self.module_name, self.ioc, "FOUND",
"%s%s" % (url, path))
return
def run(self, params={}):
whitelist = params.get(Input.WHITELIST, None)
agent = params.get(Input.AGENT)
if validators.ipv4(agent):
found_agent = find_agent_by_ip(self.connection, agent)
if found_agent != agent:
agent = found_agent
else:
raise PluginException(
cause="Agent not found.",
assistance=f"Unable to find an agent with IP: {agent},"
f" please ensure that the IP address is correct.",
)
device_obj = self.connection.client.get_agent_details(agent)
if whitelist:
matches = find_in_whitelist(device_obj, whitelist)
if matches:
raise PluginException(
cause="Agent found in the whitelist.",
assistance=f"If you would like to block this host, remove {str(matches)[1:-1]} from the whitelist.",
)
return {Output.LOCKDOWN_DETAILS: self.connection.client.device_lockdown(device_obj.get("id"))}
def process_iocs(results):
"""Return data formatted for Splunk from CyberCrime Tracker."""
if results != None:
provided_iocs = [y for x in results for y in x.values()]
else:
provided_iocs = sys.argv[1:]
session = commons.create_session()
splunk_table = []
for provided_ioc in set(provided_iocs):
provided_ioc = provided_ioc.replace("[.]", ".")
provided_ioc = provided_ioc.replace("[d]", ".")
provided_ioc = provided_ioc.replace("[D]", ".")
if validators.domain(provided_ioc) or validators.ipv4(provided_ioc):
cct_dicts = query_cct(provided_ioc, session)
else:
splunk_table.append({"invalid": provided_ioc})
continue
for cct_dict in cct_dicts:
splunk_table.append(cct_dict)
session.close()
return splunk_table
def checkType(self, argument):
"""
Identify observable type
"""
if not argument or len(argument.strip()) == 0:
return None
elif argument[0] is '#':
return None
elif validators.url(argument):
return "URL"
elif validators.md5(argument):
return "MD5"
elif validators.sha1(argument):
return "SHA1"
elif validators.sha256(argument):
return "SHA256"
elif validators.sha512(argument):
return "SHA512"
elif validators.ipv4(argument):
return "IPv4"
elif validators.ipv6(argument):
return "IPv6"
elif validators.domain(argument):
return "domain"
else:
return None
def is_mavlink_endpoint(cls: Type["Endpoint"], values: Any) -> Any:
connection_type, place, argument = (values.get("connection_type"),
values.get("place"),
values.get("argument"))
if connection_type in [
EndpointType.UDPServer,
EndpointType.UDPClient,
EndpointType.TCPServer,
EndpointType.TCPClient,
]:
if not (validators.domain(place) or validators.ipv4(place)
or validators.ipv6(place)):
raise ValueError(f"Invalid network address: {place}")
if not argument in range(1, 65536):
raise ValueError(
f"Ports must be in the range 1:65535. Received {argument}."
)
return values
if connection_type == EndpointType.Serial.value:
if not place.startswith("/") or place.endswith("/"):
raise ValueError(
f"Bad serial address: {place}. Make sure to use an absolute path."
)
if not argument in VALID_SERIAL_BAUDRATES:
raise ValueError(
f"Invalid serial baudrate: {argument}. Valid option are {VALID_SERIAL_BAUDRATES}."
)
return values
raise ValueError(
f"Invalid connection_type: {connection_type}. Valid types are: {[e.value for e in EndpointType]}."
)
def create(lab_name, allowed_drivers=None, is_mgm_only=False, is_interactive=False):
import validators
from lab.laboratory import Laboratory
from lab.mercury.nodes import MercuryMgm
if not validators.ipv4(lab_name):
if lab_name not in WithMercury.KNOWN_PODS_DIC:
raise ValueError('"{}" unknown, possible names are {}'.format(lab_name, WithMercury.KNOWN_PODS_DIC.keys()))
else:
ip = WithMercury.KNOWN_PODS_DIC[lab_name]['mgm_ip']
else:
ip = lab_name
mgm, status, setup_data_dic, release_tag, gerrit_tag = MercuryMgm.create_from_actual(ip=ip, password=WithMercury.COMMON_PASSWORD)
if not is_interactive and '| ORCHESTRATION | Success |' not in status:
raise RuntimeError('{} is not properly installed: {}'.format(lab_name, status))
driver = setup_data_dic['MECHANISM_DRIVERS']
if allowed_drivers:
assert driver.lower() in allowed_drivers, 'driver {} not in {}'.format(driver, allowed_drivers)
pod = Laboratory(name=lab_name,
driver=driver,
release_tag=release_tag,
gerrit_tag=gerrit_tag,
setup_data_dic=setup_data_dic)
if is_mgm_only:
return mgm
WithMercury.create_from_setup_data(pod=pod, mgm=mgm, is_interactive=is_interactive)
if pod.driver == WithMercury.VTS:
pod.driver_version = pod.vtc.r_vtc_get_version()
else:
pod.driver_version = 'vpp XXXX'
return pod
def is_valid(hostname):
"""
Check if a hostname is a valid IP address
"""
if hostname is None or len(hostname) == 0:
return False
return validators.ipv4(hostname)
def read_iocs(cb, file=sys.stdin):
iocs = defaultdict(list)
report_id = hashlib.md5()
report_id.update(str(time.time()).encode("utf-8"))
for idx, line in enumerate(sys.stdin):
line = line.rstrip("\r\n")
report_id.update(line.encode("utf-8"))
if validators.md5(line):
iocs["md5"].append(line)
elif validators.sha256(line):
eprint("line {}: sha256 provided but not yet supported by backend".format(idx + 1))
iocs["sha256"].append(line)
elif validators.ipv4(line):
iocs["ipv4"].append(line)
elif validators.ipv6(line):
iocs["ipv6"].append(line)
elif validators.domain(line):
iocs["dns"].append(line)
else:
if cb.validate_query(line):
query_ioc = {"search_query": line}
iocs["query"].append(query_ioc)
else:
eprint("line {}: invalid query".format(idx + 1))
return (report_id.hexdigest(), dict(iocs))
def getAnonymizedIpOrDomain(value):
if validators.ipv4(value) or validators.ipv6(value):
return "<IPADDRESS:%s>" % hashlib.md5(value).hexdigest()
elif validators.domain(value):
return "<DOMAIN:%s>" % hashlib.md5(value).hexdigest()
else:
return "<NOTIPADDRESSORDOMAIN:%s>" % hashlib.md5(value).hexdigest()
def start(lab, log, args):
import validators
from lab import fi
is_show_details = args.get('is-show-details', False)
name_or_ip = args.get('name_or_ip', 'from_lab')
if validators.ipv4(name_or_ip):
ucsm_ip = name_or_ip
ucsm_username = args['username']
ucsm_password = args['password']
fi = fi.FI(name='NotDefined', ip=ucsm_ip, username=ucsm_username, password=ucsm_password, lab=lab, hostname='NoDefined')
else:
fi = lab.get_nodes_by_class(fi.FI)[0]
vlan_profiles = sorted([x.split()[0] for x in fi.list_vlans()])
log.info('n_vlans={0} {1}'.format(len(vlan_profiles), 'details={0}'.format('+'.join(vlan_profiles)) if is_show_details else ''))
if is_show_details:
user_sessions = fi.list_user_sessions()
log.info('n_sessions={0} details={1}'.format(len(user_sessions), '+'.join(user_sessions)))
service_profiles = [x.split()[0] for x in fi.list_service_profiles()]
for sp in service_profiles:
if 'control' in sp or 'compute' in sp:
vlans_on_profile = [x.split(':')[-1].strip() for x in fi.list_allowed_vlans(sp, 'eth0')]
log.info('profile=eth0@{sp} n_vlans_on_profile={n} details={vlans}'.format(sp=sp, n=len(vlans_on_profile), vlans='+'.join(vlans_on_profile)))
def validate(cls, resource):
temp_username = f"{resource}@thethe.com"
try:
if validators.ipv4(resource):
return ResourceType.IPv4
elif validators.domain(resource):
return ResourceType.DOMAIN
elif validators.url(resource):
return ResourceType.URL
elif validators.email(resource):
return ResourceType.EMAIL
elif (HashType.hash_detection(resource) in HashType and
not HashType.hash_detection(resource) == HashType.UNKNOWN):
return ResourceType.HASH
# EYES HERE! We check a username as a contrived email
# See temp_username
elif validators.email(temp_username):
return ResourceType.USERNAME
print(
"[!] No resource type has found for your resource: {resource}")
return ResourceType.UNKNOWN
except Exception as e:
raise Exception()
def add(request):
msg = vmsg.msg()
all_instances = perms.instance_getall_by_group(request)
hostname_default = perms.get_hostname_prefered(request)
is_superuser = perms.get_is_superuser(request.user)
if 'server' in request.POST:
if validators.ipv6(request.POST['server'].strip()) or validators.ipv4(
request.POST['server'].strip()) or validators.domain(
request.POST['server'].strip()):
v = vapi.set_ntp(hostname_default, request.POST['server'].strip())
if v.success == False:
msg.add_error("NTP server add fail - " + v.reason)
else:
msg.add_success("NTP server added")
else:
msg.add_error(
"ntp server add fail - insert only domains or IPv4 or IPv6")
context = {
'instances': all_instances,
'hostname_default': hostname_default,
'is_superuser': is_superuser,
'username': request.user,
'msg': msg.get_all(),
}
return render(request, 'ntp/add.html', context)
def validate(self, val):
try:
check_type = val["check_type"]
except KeyError:
return val
# disk checks
# make sure no duplicate diskchecks exist for an agent/policy
if check_type == "diskspace" and not self.instance: # only on create
checks = (Check.objects.filter(**self.context).filter(
check_type="diskspace").exclude(managed_by_policy=True))
if checks:
for check in checks:
if val["disk"] in check.disk:
raise serializers.ValidationError(
f"A disk check for Drive {val['disk']} already exists!"
)
# ping checks
if check_type == "ping":
if (not _v.ipv4(val["ip"]) and not _v.ipv6(val["ip"])
and not _v.domain(val["ip"])):
raise serializers.ValidationError(
"Please enter a valid IP address or domain name")
return val
def run(self, params={}):
# If IPv4, attempt to find its ID first
agent = params.get(Input.AGENT)
if validators.ipv4(agent):
agent = find_agent_by_ip(self.connection, agent)
return {Output.AGENTS: self.connection.client.search_agents_all(agent)}
def checkType(self, argument):
"""
Identify observable type
"""
if len(argument.strip()) == 0:
return None
elif argument[0] is '#':
return None
elif validators.url(argument):
return "URL"
elif validators.md5(argument):
return "MD5"
elif validators.sha1(argument):
return "SHA1"
elif validators.sha256(argument):
return "SHA256"
elif validators.sha512(argument):
return "SHA512"
elif validators.ipv4(argument):
return "IPv4"
elif validators.ipv6(argument):
return "IPv6"
elif validators.domain(argument):
return "domain"
else:
mod.display("MAIN", argument, "ERROR", "Unable to retrieve observable type")
return None
def run(self, params={}):
# If IPv4, attempt to find its ID first
agent = params.get(Input.AGENT)
if validators.ipv4(agent):
agent = find_agent_by_ip(self.connection, agent)
agent = self.connection.client.get_agent_details(agent)
policy = params.get(Input.POLICY)
if policy == "":
policy = self._find_default_policy_id()
errors = self.connection.client.update_agent(
agent.get("id"),
{
"add_zone_ids": params.get(Input.ADD_ZONES, None),
"name": agent.get("name"),
"policy_id": policy,
"remove_zone_ids": params.get(Input.REMOVE_ZONES, None),
},
)
if len(errors) != 0:
raise PluginException(
cause=
"The response from the CylancePROTECT API was not in the correct format.",
assistance="Contact support for help. See log for more details",
data=errors,
)
return {Output.SUCCESS: True}
def verify_config(owner, sample_config, config, current_key=None):
"""Verify that config corresponds to sample_config"""
import validators
def raise_exception(message):
raise ValueError('in {} config {}\nsample: {}\nprovided: {}'.format(owner, message, sorted(sample_config.items()), sorted(config.items())))
if isinstance(sample_config, list):
if not len(config):
raise_exception('empty_list')
for element in config:
verify_config(owner=owner, sample_config=sample_config[0], config=element, current_key=current_key)
elif isinstance(sample_config, dict):
for sample_key, sample_value in sample_config.items():
if sample_key not in config:
raise_exception('key "{}" is not provided'.format(sample_key))
if config[sample_key] is None:
raise_exception('Value of "{}" is empty'.format(sample_key))
verify_config(owner=owner, sample_config=sample_value, config=config[sample_key], current_key=sample_key)
else:
# from this point config and sample_config start to be simple values
if type(sample_config) is str:
if sample_config.startswith('http') and validators.url(config) is not True:
raise_exception('Key "{}" do not contain valid url: {}'.format(current_key, config))
elif sample_config.startswith('email') and not validators.email(config):
raise_exception('Key "{}" do not contain valid email: {}'.format(current_key, config))
elif sample_config.startswith('ipv4') and not validators.ipv4(config):
raise_exception('Key "{}" do not contain valid IPv4: {}'.format(current_key, config))
elif sample_config.startswith('int'):
try:
int(config)
except ValueError:
raise_exception('Key "{}" do not contain valid int number: {}'.format(current_key, config))
elif type(sample_config) is bool and type(config) is not bool:
raise_exception('Key "{}" must be bool: {}'.format(current_key, config))
def geo_lookup(ip_addr):
"""Resolve an IP address to it's hosting country.
Params:
- ip_addr: (type: string) IP address.
Returns:
- result: (type: string) two-letter country code.
"""
if validators.ipv4(ip_addr):
try:
if ip_addr == '255.255.255.255':
return 'ZZ'
with geoip2.database.Reader(BASECONFIG.geolite_db) as reader:
response = reader.city(ip_addr)
if response is not None:
if response.country.iso_code is not None:
return response.country.iso_code
if response.continent.code is not None:
return response.continent.code
except Exception as e:
LOGGING.warning(
'Failed to perform GeoLookup for address {0}: {1}'.format(
ip_addr, str(e)))
else:
LOGGING.warning('Invalid IP address: {0}'.format(ip_addr))
return 'ZZ'
def read_iocs(cb, file=sys.stdin):
iocs = defaultdict(list)
report_id = hashlib.md5()
report_id.update(str(time.time()).encode("utf-8"))
for idx, line in enumerate(sys.stdin):
line = line.rstrip("\r\n")
report_id.update(line.encode("utf-8"))
if validators.md5(line):
iocs["md5"].append(line)
elif validators.sha256(line):
eprint("line {}: sha256 provided but not yet supported by backend".
format(idx + 1))
iocs["sha256"].append(line)
elif validators.ipv4(line):
iocs["ipv4"].append(line)
elif validators.ipv6(line):
iocs["ipv6"].append(line)
elif validators.domain(line):
iocs["dns"].append(line)
else:
if cb.validate_query(line):
query_ioc = {"search_query": line}
iocs["query"].append(query_ioc)
else:
eprint("line {}: invalid query".format(idx + 1))
return (report_id.hexdigest(), dict(iocs))
def url_or_ip(self, url):
if validators.domain(url) is True or validators.url(url) is True:
return 110 # URL
elif validators.ipv4(url) is True:
return 111 # IP
else:
return 112 # Error
def test_public_ip(agent):
error = []
if not validators.ipv4(agent.get_public_ip()) and not validators.ipv6(
agent.get_public_ip()):
error.append("not ipv4 or ipv6")
assert not error
def process_iocs(results):
"""Return data formatted for Splunk from crt.sh."""
if results != None:
provided_iocs = [y for x in results for y in x.values()]
elif sys.argv[1] != "subdomain" and sys.argv[1] != "wildcard":
if len(sys.argv) > 1:
provided_iocs = sys.argv[1:]
elif sys.argv[1] == "subdomain" or sys.argv[1] == "wildcard":
if len(sys.argv) > 2:
provided_iocs = sys.argv[2:]
session = commons.create_session()
splunk_table = []
for provided_ioc in set(provided_iocs):
provided_ioc = commons.deobfuscate_url(provided_ioc)
if validators.domain(provided_ioc) or validators.ipv4(provided_ioc):
crt_dicts = query_crtsh(provided_ioc, session)
else:
splunk_table.append({"invalid": provided_ioc})
continue
for crt_dict in crt_dicts:
splunk_table.append(crt_dict)
session.close()
return splunk_table
def process_master(results):
"""Return dictionary containing data returned from the (unofficial)
CryberCrime Tracker API."""
splunk_dict = []
if results != None:
provided_iocs = [y for x in results for y in x.values()]
else:
provided_iocs = sys.argv[1:]
for provided_ioc in set(provided_iocs):
if validators.ipv4(provided_ioc) or validators.domain(provided_ioc):
threatcrowd_dicts = threatcrowd.process_host(provided_ioc)
elif validators.email(provided_ioc):
threatcrowd_dicts = threatcrowd.process_email(provided_ioc)
else:
splunk_dict.append({"Invalid": provided_ioc})
continue
if len(threatcrowd_dicts) == 0:
splunk_dict.append({"Invalid": provided_ioc})
continue
for threatcrowd_dict in threatcrowd_dicts:
splunk_dict.append(threatcrowd_dict)
if len(provided_iocs) > 1:
sleep(10)
return splunk_dict
def process_host(provided_ioc):
""" """
if validators.ipv4(provided_ioc):
ioc_type = "ip"
else:
ioc_type = "domain"
ioc_dict = []
resp = requests.get(api.format(ioc_type, ioc_type, provided_ioc),
headers={"User-Agent": useragent})
if resp.status_code == 200:
for key in resp.json().keys():
if key == "votes" or key == "permalink" or key == "response_code":
pass
elif key == "resolutions":
if len(resp.json()[key]) == 0:
ioc_dict.append({key: ""})
elif len(resp.json()[key]) > 0:
for res in resp.json()[key]:
ioc_dict.append(res)
else:
if len(resp.json()[key]) == 0:
ioc_dict.append({key: ""})
else:
for value in resp.json()[key]:
ioc_dict.append({key: value})
return ioc_dict
def asn_lookup(ip_addr):
"""Resolve the ASN of an IP address.
Params:
- ip_addr: (type: string) IP address.
Returns:
- result: (type: string) ASN number and organisation.
"""
if validators.ipv4(ip_addr):
try:
if ip_addr == '255.255.255.255':
return 'AS0000 Unknown'
with geoip2.database.Reader(BASECONFIG.asn_db) as reader:
response = reader.asn(ip_addr)
if response is not None:
asn_number = response.autonomous_system_number
asn_org = response.autonomous_system_organization
return 'AS{0} {1}'.format(asn_number, asn_org)
except Exception as e:
LOGGING.warning(
'Failed to perform ASN lookup for address {0}: {1}'.format(
ip_addr, str(e)))
else:
LOGGING.warning('Invalid IP address: {0}'.format(ip_addr))
return 'AS0000 Unknown'
def start(lab, log, args):
import validators
name_or_ip = args.get('name_or_ip', 'from_lab')
is_show_details = args.get('is_show_details', False)
if validators.ipv4(name_or_ip):
n9k_ip = name_or_ip
n9k_username = args['username']
n9k_password = args['password']
else:
n9k_ip, _, n9k_username, n9k_password = lab.n9k_creds()
nx = lab.nodes.n9.Nexus(n9k_ip, n9k_username, n9k_password)
port_channels = nx.show_port_channel_summary()
# Vlans
vlans = nx.n9_show_vlans()
log.info('ip={ip} n_vlans={n} {det}'.format(ip=n9k_ip, n=len(vlans),
det='list={0}'.format(vlans) if is_show_details else ''))
# Allowed vlans
for port_channel in port_channels:
allowed_vlans = nx.show_interface_switchport(name=port_channel)
log.info('ip={ip} service={srv} n_vlans={n} {det}'.format(ip=n9k_ip, srv=port_channel, n=len(allowed_vlans),
det='list={0}'.format(allowed_vlans) if is_show_details else ''))
# User sessions
if is_show_details:
users = nx.n9_show_users()
log.info('ip={ip} n_user_sessions={n} list={l}'.format(ip=n9k_ip, n=len(users), l=users))
def process_iocs(results):
"""Return data formatted for Splunk from urlscan.io."""
if results != None:
provided_iocs = [y for x in results for y in x.values()]
else:
provided_iocs = sys.argv[1:]
session = commons.create_session()
splunk_table = []
for provided_ioc in set(provided_iocs):
provided_ioc = commons.deobfuscate_url(provided_ioc)
if provided_ioc == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855":
splunk_table.append({"no data": provided_ioc})
continue
if validators.domain(provided_ioc) or validators.ipv4(provided_ioc) or \
validators.sha256(provided_ioc) or "certstream-suspicious" in provided_ioc:
ioc_dicts = query_urlscan(session, provided_ioc)
else:
splunk_table.append({"invalid": provided_ioc})
continue
for ioc_dict in ioc_dicts:
splunk_table.append(ioc_dict)
session.close()
return splunk_table
def __do_convert_url_pattern(self, result: Result, s: str,
is_allow: bool) -> None:
t = s
is_domain_suffix = True
if t.startswith("||"):
t = t[2:]
is_domain_suffix = True
if t.startswith("|"):
t = t[1:]
is_domain_suffix = False
if t.startswith("."):
t = t[1:]
is_domain_suffix = True
t = re.sub(r"^https?://", "", t)
t = t.lstrip("*.").rstrip("^/*")
ur = urllib.parse.urlparse("http://" + t)
if ur.fragment:
return
if ur.path and not self.ignore_path:
return
t = ur.netloc
t = re.sub(r":\d{2,5}$", "", t)
if validators.ipv4(t) or validators.ipv4_cidr(t):
result.add(result.ipv4_cidr, t, remove=is_allow)
elif validators.ipv6(t) or validators.ipv6_cidr(t):
result.add(result.ipv6_cidr, t, remove=is_allow)
else:
if validators.domain(t) and tld.get_fld(
t, fail_silently=True, fix_protocol=True):
if is_domain_suffix:
result.add(result.domain_suffix, t, remove=is_allow)
else:
result.add(result.domain, t, remove=is_allow)
def getAnonymizedIpOrDomain(value):
if validators.ipv4(value) or validators.ipv6(value):
return "<IPADDRESS:%s>" % hashlib.md5(value).hexdigest()
elif validators.domain(value):
return "<DOMAIN:%s>" % hashlib.md5(value).hexdigest()
else:
return "<NOTIPADDRESSORDOMAIN:%s>" % hashlib.md5(value).hexdigest()
def add_nic(self, nic_name, mac_or_pattern, ip_or_index, net, on_wires):
import validators
from lab.network import Nic
ip_or_index = ip_or_index or self._assign_default_ip_index(net)
try:
index = int(ip_or_index) # this is shift in the network
if index in [0, 1, 2, 3, -1]:
raise ValueError('{}: index={} is not possible since 0 => network address [1,2,3] => GW addresses -1 => broadcast address'.format(self.get_id(), index))
try:
ip = net[index]
except (IndexError, ValueError):
raise ValueError('{}: index {} is out of bound of {}'.format(self.get_id(), index, net))
except ValueError:
if validators.ipv4(str(ip_or_index)):
try:
index, ip = {x: str(net[x]) for x in range(net.size) if str(ip_or_index) in str(net[x])}.items()[0]
except IndexError:
raise ValueError('{}: ip {} is out of bound of {}'.format(self.get_id(), ip_or_index, net))
else:
raise ValueError('{}: specified value "{}" is neither ip nor index in network'.format(self.get_id(), ip_or_index))
self.lab().make_sure_that_object_is_unique(obj=ip, node_id=self.get_id())
mac = mac_or_pattern if validators.mac_address(str(mac_or_pattern)) else self.form_mac(mac_or_pattern)
self.lab().make_sure_that_object_is_unique(obj=mac, node_id=self.get_id())
nic = Nic(name=nic_name, mac=mac, node=self, net=net, net_index=index, on_wires=on_wires)
self._nics[nic_name] = nic
return nic
def get_public_ip(self):
try:
ifconfig = requests.get("https://ifconfig.co/ip", timeout=5).text.strip()
if not validators.ipv4(ifconfig) and not validators.ipv6(ifconfig):
icanhaz = requests.get("https://icanhazip.com", timeout=7).text.strip()
if not validators.ipv4(icanhaz) and not validators.ipv6(icanhaz):
return "error"
else:
return icanhaz
else:
return ifconfig
except Exception as e:
self.logger.debug(e)
return "error"
def check_endpoint(self, endpoint):
is_ipv4 = validators.ipv4(endpoint)
is_ipv6 = validators.ipv6(endpoint)
if is_ipv4 or is_ipv6:
return False
if validators.domain(endpoint):
return True
raise CheckThirdpartEndpointFailed(msg="invalid endpoint")
def get_domain(self, var, default=NOTSET):
# type: (str, Union[str, NotSetType]) -> str
value = self.get(var, default=default)
if value == default:
return value
if not (validators.ipv4(value) or validators.ipv6(value) or validators.domain(value)):
raise VariableTypeError('Environment variable {var} with value {val} is not a valid hostname.'
.format(var=var, val=value))
return value
def setup_worker(self):
import validators
from lab.nodes.lab_server import LabServer
self._cmd = self._kwargs['cmd']
if validators.ipv4(self._ip):
self._node = LabServer(node_id='Server{}'.format(self._ip), role='vtc', lab=None, hostname='NoDefined')
self._node.set_ssh_creds(ip=self._ip, username=self._username, password=self._password)
else:
lab = self._cloud.mediator.lab()
self._node = lab.get_nodes_by_id(self._ip)
def get_ip(self, var, default=NOTSET):
# type: (str, Union[str, NotSetType]) -> str
value = self.get(var, default=default)
if value == default:
return value
if not (validators.ipv4(value) or validators.ipv6(value)):
raise VariableTypeError('Environment variable {var} with value {val} could not be converted into ipv4/ipv6.'
.format(var=var, val=value))
return value
def __init__(self, cloud, **kwargs):
import validators
self._kwargs = kwargs
self._cloud = cloud
self._ip = kwargs.get('ip')
if self._ip:
if validators.ipv4(self._ip):
try:
self._username, self._password = kwargs['username'], kwargs['password']
except KeyError:
raise ValueError('"username" and/or "password" are not provided'.format())
else:
raise ValueError('Provided invalid ip address: "{0}"'.format(self._ip))
def construct_settings(self, warn_only, connection_attempts=100):
import validators
from lab import with_config
ssh_ip, ssh_username, ssh_password = self.get_ssh()
ssh_ip = ssh_ip if validators.ipv4(ssh_ip) else self.get_oob()[0]
kwargs = {'host_string': '{user}@{ip}'.format(user=ssh_username, ip=ssh_ip),
'connection_attempts': connection_attempts,
'warn_only': warn_only}
if ssh_password == 'ssh_key':
kwargs['key_filename'] = with_config.KEY_PRIVATE_PATH
else:
kwargs['password'] = ssh_password
return kwargs
def verify_config(sample_config, config, exception, current_key=None):
"""Verify that config corresponds to sample_config"""
import validators
if isinstance(sample_config, list):
if not len(config):
exception.message = 'empty list'
raise exception
for element in config:
verify_config(sample_config=sample_config[0], config=element, exception=exception, current_key=current_key)
elif isinstance(sample_config, dict):
for sample_key, sample_value in sample_config.iteritems():
if sample_key not in config:
exception.message = 'Key "{0}" not in config'.format(sample_key)
raise exception
if config[sample_key] is None:
exception.message = 'Value of "{0}" is empty'.format(sample_key)
raise exception
verify_config(sample_config=sample_value, config=config[sample_key], exception=exception, current_key=sample_key)
else:
# from this point config and sample_config start to be simple values
if type(sample_config) is str:
if sample_config.startswith('http') and validators.url(config) is not True:
exception.message = 'Key "{0}" do not contain valid url: {1}'.format(current_key, config)
raise exception
elif sample_config.startswith('email') and not validators.email(config):
exception.message = 'Key "{0}" do not contain valid email: {1}'.format(current_key, config)
raise exception
elif sample_config.startswith('ipv4') and not validators.ipv4(config):
exception.message = 'Key "{0}" do not contain valid IPv4: {1}'.format(current_key, config)
raise exception
elif sample_config.startswith('int'):
try:
int(config)
except ValueError:
exception.message = 'Key "{0}" do not contain valid int number: {1}'.format(current_key, config)
raise exception
elif type(sample_config) is bool and type(config) is not bool:
exception.message = 'Key "{0}" must be bool: {1}'.format(current_key, config)
raise exception
def start(context, log, args):
import socket
import validators
name_or_ip = args['name_or_ip']
port = args.get('port', 22)
if validators.ipv4(name_or_ip):
ip = name_or_ip
else:
ip = str(context.get_node_by_id(name_or_ip).ip)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(1)
try:
s.connect((ip, port))
res = 1
except (socket.timeout, socket.error):
res = 0
finally:
s.close()
log.info('service={name}:{port} ip={ip} status={status}'.format(name=name_or_ip, port=port, ip=ip, status=res))
def netmask(self, interface_name, netmask):
"""
Checking interface first, if interface name found in Get().interfaces()
validating Ipv4. After that applied ip address to interace
interface_name = Applied Interface
netmask = New netmask ip address
"""
interface_check = Get().interfaces
valid_ipv4 = validators.ipv4(netmask)
if not interface_name in interface_check:
raise WrongInterfaceName("Wrong Interface Name %s" % interface_name)
elif not valid_ipv4 is True:
raise NotValidIPv4Address("Not Valid IPv4 Address %s" % netmask)
else:
prefix_len = self.get_net_size(netmask.split('.'))
ifname = interface_name.encode(encoding='UTF-8')
netmask = ctypes.c_uint32(~((2 ** (32 - prefix_len)) - 1)).value
nmbytes = socket.htonl(netmask)
ifreq = struct.pack('16sH2sI8s', ifname, AF_INET, b'\x00'*2, nmbytes, b'\x00'*8)
fcntl.ioctl(self.sock, SIOCSIFNETMASK, ifreq)
def ip(self, interface_name, newip):
"""
This function automatically will change netmask address.
Checking interface first, if interface name found in Get().interfaces()
validating Ipv4. After that applied ip address to interface
interface_name: Applied Interface
newip: New Ip Address
"""
interface_check = Get().interfaces
valid_ipv4 = validators.ipv4(newip)
if not interface_name in interface_check:
raise WrongInterfaceName("Wrong Interface Name %s" % interface_name)
elif not valid_ipv4 is True:
raise NotValidIPv4Address("Not Valid IPv4 Address %s" % newip)
else:
ifname = interface_name.encode(encoding='UTF-8')
ipbytes = socket.inet_aton(newip)
ifreq = struct.pack('16sH2s4s8s', ifname, AF_INET, b'\x00'*2, ipbytes, b'\x00'*8)
fcntl.ioctl(self.sock, SIOCSIFADDR, ifreq)
def isipv4(value):
try:
validators.ipv4(value)
except validators.ValidationFailure:
return False
return True
def is_ipv4(ip):
if ip is not None and validators.ipv4(ip):
return ip
else:
raise Exception('{} is not valid ipv4'.format(ip))
def cobbler_configure_for(self, node):
import validators
from lab.time_func import time_as_string
self.log('{}: (Re)creating cobbler profile for {}'.format(self, node))
system_name = '{}-{}'.format(self.lab(), node.get_id())
comment = 'This system is created by {0} for LAB {1} at {2}'.format(__file__, self.lab(), time_as_string())
network_commands = []
gateway = None
for master_name, nic in node.get_nics().items():
ip, mask = nic.get_ip_and_mask()
ip_mask_part = '--ip-address={} --netmask={} --static 1'.format(ip, mask) if not nic.is_pxe() and validators.ipv4(str(ip)) else ''
macs = nic.get_macs()
names = nic.get_names()
if nic.is_ssh():
gateway = nic.get_net().get_gw()
if len(names) > 1:
bond_mode = 'mode=802.3ad lacp_rate=1' if nic.is_ssh() else 'mode=balance-xor'
for slv_name, slv_mac in zip(names, macs):
network_commands.append('--interface={} --mac={} --interface-type=bond_slave --interface-master={}'.format(slv_name, slv_mac, master_name))
network_commands.append('--interface={} --interface-type=bond --bonding-opts="{} miimon=50 xmit_hash_policy=1 updelay=0 downdelay=0 " {}'.format(master_name, bond_mode, ip_mask_part))
else:
network_commands.append('--interface={} --mac={} {}'.format(master_name, macs[0], ip_mask_part))
ans = self.cmd('cobbler system list | grep {}'.format(system_name))
if system_name in ans:
self.cmd('cobbler system remove --name={}'.format(system_name))
self.cmd('cobbler system add --name={} --profile=RHEL7.2-x86_64 --kickstart=/var/lib/cobbler/kickstarts/sqe --comment="{}"'.format(system_name, comment))
self.cmd('cobbler system edit --name={} --hostname={} --gateway={}'.format(system_name, node.get_hostname(), gateway))
for cmd in network_commands:
self.cmd('cobbler system edit --name={} {}'.format(system_name, cmd))
ipmi_ip, ipmi_username, ipmi_password = node.get_oob()
self.cmd('cobbler system edit --name={} --power-type=ipmilan --power-address={} --power-user={} --power-pass={}'.format(system_name, ipmi_ip, ipmi_username, ipmi_password))
return system_name
def cobbler_configure_for(self, node):
import validators
from lab.time_func import time_as_string
from lab.logger import lab_logger
lab_logger.info('{}: (Re)creating cobbler profile for {}'.format(self, node))
system_name = '{}-{}'.format(self.lab(), node.get_id())
comment = 'This system is created by {0} for LAB {1} at {2}'.format(__file__, self.lab(), time_as_string())
network_commands = []
gateway = None
for nic in node.get_nics().values():
ip, mask = nic.get_ip_and_mask()
ip_mask_part = '--ip-address={} --netmask={} --static 1'.format(ip, mask) if validators.ipv4(str(ip)) else ''
mac = nic.get_mac()
name = nic.get_name()
if nic.is_ssh():
gateway = nic.get_net()[0]
if nic.is_bond():
for name_slave, mac_port in nic.get_slave_nics().items():
mac = mac_port['mac']
network_commands.append('--interface={} --mac={} --interface-type=bond_slave --interface-master={}'.format(name_slave, mac, name))
network_commands.append('--interface={} --interface-type=bond --bonding-opts="miimon=100 mode=1" {}'.format(name, ip_mask_part))
else:
network_commands.append('--interface={} --mac={} {}'.format(name, mac, ip_mask_part))
systems = self.run('cobbler system list')
if system_name in systems:
self.run('cobbler system remove --name={}'.format(system_name))
self.run('cobbler system add --name={} --profile=RHEL7.2-x86_64 --kickstart=/var/lib/cobbler/kickstarts/sqe --comment="{}"'.format(system_name, comment))
self.run('cobbler system edit --name={} --hostname={} --gateway={}'.format(system_name, node.hostname(), gateway))
for cmd in network_commands:
self.run('cobbler system edit --name={} {}'.format(system_name, cmd))
ipmi_ip, ipmi_username, ipmi_password = node.get_oob()
self.run('cobbler system edit --name={} --power-type=ipmilan --power-address={} --power-user={} --power-pass={}'.format(system_name, ipmi_ip, ipmi_username, ipmi_password))
return system_name
def test_returns_failed_validation_on_invalid_ipv4_address(address):
assert isinstance(ipv4(address), ValidationFailure)
def test_returns_true_on_valid_ipv4_address(address):
assert ipv4(address)
def process(self):
ret={}
data=""
pelib=self._getLibrary(PEFileModule().getName())
if(pelib==None):data=self.sample.getBinary()
else:
for section in pelib.sections:
data=data+section.get_data()
regexp='[A-Za-z0-9/\-:.,_$&@=?%()[\]<> ]{4,}'
strings=re.findall(regexp,data)
aux={}
for s in strings:
aux[repr(s).lower()]=True
unique_strings=[]
for k in aux:
unique_strings.append(k)
mdc=self._getLibrary(MetaDataModule().getName())
if(mdc==None):return ret
searchUsed={}
imports=self.sample.getLastValue("particular_header.imports")
if(imports!=None ):
for i in imports:
searchUsed[i["lib"]]=True
for f in i["functions"]:
searchUsed[f]=True
exports=self.sample.getLastValue("particular_header.exports.symbols")
if(exports!=None ):
#print("No exports")
for i in exports:
searchUsed[i["name"]]=True
if(hasattr(i,"forwarder_dll") and hasattr(i,"forwarder_function")):
searchUsed[i["forwarder_dll"]]=True
searchUsed[i["forwarder_function"]]=True
version_p=self.sample.getLastValue("particular_header.version.string_file_info")
if(version_p!=None ):
for k in version_p.keys():
searchUsed["'"+str(version_p[k])+"'"]=True
raw=[]
hidden=[]
email=[]
url=[]
ip_l=[]
dll=[]
domain=[]
interesting=[]
registry=[]
for s in unique_strings:
#viendo si declara el import o no
#print(s)
#print(searchUsed.get(repr(s).lower()))
#raw_input()
if(searchUsed.get(s)==True): continue
raw.append(s)
#buscando si es un import o no
r=mdc.searchImportByName(s)
if(r!=None):
hidden.append(s)
continue
evaluado=eval(s)
#buscando dll
r=mdc.searchDllByName(s)
if(r!=None):
dll.append(s)
continue
#buscando cosas nombres de archivos
types=["exe","dll","bat","sys","htm","html","js","jar","jpg",
"png","vb","scr","pif","chm","zip","rar","cab","pdf",
"doc","docx","ppt","pptx","xls","xlsx","swf","gif","pdb","cpp"]
salir=False
for pat in types:
if(s.find("."+pat)!=-1):
interesting.append(s)
salir=True
break
if salir: continue
#buscando email
if(validators.email(evaluado)):
email.append(s)
continue
#buscando url
if(validators.url(evaluado)):
url.append(s)
continue
#buscando ip
if(validators.ipv4(evaluado)): #or validators.ipv6(evaluado)):
ip_l.append(s)
continue
#buscando registry
if(s.find("HKLM\\")!=-1 or s.find("HKCU\\")!=-1 ):
registry.append(s)
continue
#buscando dominios
if(validators.domain(evaluado)):
domain.append(s)
continue
ret["raw_strings"]=sorted(raw)
if(len(hidden)>0):ret["hidden_imports"]=sorted(hidden)
if(len(email)>0):ret["emails"]=sorted(email)
if(len(url)>0):ret["urls"]=sorted(url)
if(len(ip_l)>0):ret["ips"]=sorted(ip_l)
if(len(dll)>0):ret["hidden_dll"]=sorted(dll)
if(len(domain)>0):ret["domains"]=sorted(domain)
if(len(interesting)>0):ret["interesting"]=sorted(interesting)
if(len(registry)>0):ret["registry"]=sorted(registry)
return ret
def arg_validator(arg, vlist=None):
"""
检查数据，可对同一个数据进行多种检查
arg ： 字符串，要验证的参数
vlist ： 列表，验证列表，每个元素包含两项.
第一个项是检查类型（字符串），第二项是可选参数字典,类似：
[
("检查类型",{"可选参数1":参数值,"可选参数2":参数值...})，
("检查类型",{"可选参数1":参数值,"可选参数2":参数值...})，
...
]
返回： 双元组,第一项为True 或 False,第二项为验证失败的类型(第一项为True的话第二项就留空)
注意：
vlist 列表的第一项可以是字符串 "required"，用于表示是必填项：
如果第一项不是，而且要验证的 arg 为空,会直接返回 True,不是的继续验证。
如果第一项是，就完全按照 vlist[1:] 的要求验证
vlist 的元素如果是验证整数/小数/email等不需要附加参数的可以直接传入验证类型字符串即可
用例（使用args_validator函数的，看这里vdict每个键值对的形式）：
vdict = {
"token": ["required", "uuid"],
"username": ["required", ("length", {"min": 4, "max": 30}), "safe_str"],
"password": ["required", ("length", {"min": 4, "max": 20}), "safe_str"],
"captcha": ["required", ("length", {"min": 4, "max": 8}), "safe_str"],
}
form = args_validator(self.request.arguments, vdict)
"""
if not any((isinstance(vlist, list), isinstance(vlist, tuple))):
einfo = "不支持的数据类型！应使用列表或元组,但输入的是{}".format(type(vlist))
raise ValueError(einfo)
if vlist[0] == "required":
# 第一项不是 required 的,把第一项的 "required" 去掉
vlist = vlist[1:]
else:
# 第一项不是 required 的，如果 arg 是空的，直接返回 True,不是的继续验证
if not arg:
return True, None
# 待返回的验证结果
verification = None
failed_type = None # 验证失败的类型
# 开始检查，有一个不通过就返回 False
for i in vlist:
local_verification = None
if isinstance(i, str): # 如果是字符串的话就改为元组
i = (i, {})
if len(i) == 1: # 只有一个元素的，添加一个空字典
i = (i[0], {})
vtype = i[0] # 检查类型是第一项
vdict = i[1] # 检查类型所需的可选参数字典
# 在 validators 之外添加的
# 没有空白
if vtype == "no_space":
if not re.search(r"\s", arg):
local_verification = True
# 安全字符串，只包含 0-9a-zA-Z-空格和下划线
elif vtype == "safe_str":
if re.match(r"^[0-9a-zA-Z-_ ]+$", arg, flags=re.U):
local_verification = True
# 是否包含
elif vtype == "in":
# 迭代 vdict 的键值(所以键名无所谓)
for v in vdict.values():
if arg not in v:
local_verification = False
break
elif vtype == "not_in":
# 迭代 vdict 的键值(所以键名无所谓)
for v in vdict.values():
if arg in v:
local_verification = False
break
# 字符串形式的数字
elif vtype == "str_number":
if re.match(r"[+-]?\d+$", arg, flags=re.U) or \
re.match(r"[+-]?\d+\.\d+$", arg, flags=re.U):
local_verification = True
elif vtype == "str_int":
if re.match(r"[+-]?\d+$", arg, flags=re.U):
local_verification = True
elif vtype == "str_float":
if re.match(r"[+-]?\d+\.\d+$", arg, flags=re.U):
local_verification = True
# 数字
elif vtype == "number": # 整数或浮点数都可以
local_verification = isinstance(arg, int) or isinstance(arg, float)
elif vtype == "int":
local_verification = isinstance(arg, int)
elif vtype == "float":
local_verification = isinstance(arg, float)
# 直接调用 validators的
elif vtype == "length":
local_verification = validators.length(arg, **vdict)
elif vtype == "url":
local_verification = validators.url(arg, **vdict)
elif vtype == "email":
local_verification = validators.email(arg, **vdict)
elif vtype == "ip": # ipv4 或 ipv6都可以
local_verification = any((validators.ipv4(arg, **vdict),
validators.ipv6(arg, **vdict)))
elif vtype == "between":
local_verification = validators.between(arg, **vdict)
elif vtype == "uuid":
local_verification = validators.uuid(arg, **vdict)
elif vtype == "ipv4":
local_verification = validators.ipv4(arg, **vdict)
elif vtype == "ipv6":
local_verification = validators.ipv6(arg, **vdict)
elif vtype == "mac_address":
local_verification = validators.mac_address(arg, **vdict)
elif vtype == "iban":
local_verification = validators.iban(arg, **vdict)
elif vtype == "slug":
local_verification = validators.slug(arg, **vdict)
elif vtype == "truthy":
local_verification = validators.truthy(arg, **vdict)
# 对于验证不为真或没有验证的
# 不为真的时候返回的是: ValidationFailure(......)
if not local_verification:
verification = False
failed_type = vtype
break # 有一条不为 True， 直接返回 False
else:
verification = True
# 处理返回值
if verification not in(False, True):
verification = False
if not verification:
return verification, failed_type
else:
return True, None