class ActivityFeedSchema(AutoSchema):
_id = fields.Integer(dump_only=True, attribute='id')
itime = fields.Method(serialize='get_itime', deserialize='load_itime', required=True, attribute='start_date')
sum_created_vulnerabilities = fields.Method(serialize='get_sum_created_vulnerabilities', allow_none=True)
sum_created_hosts = fields.Method(serialize='get_sum_created_hosts', allow_none=True)
sum_created_services = fields.Method(serialize='get_sum_created_services', allow_none=True)
sum_created_vulnerability_critical = fields.Method(serialize='get_sum_created_vulnerability_critical',
allow_none=True)
workspace = PrimaryKeyRelatedField('name', dump_only=True)
creator = PrimaryKeyRelatedField('username', dump_only=True)
def load_itime(self, value):
return datetime.fromtimestamp(value)
def get_itime(self, obj):
return time.mktime(obj.start_date.utctimetuple()) * 1000
def get_sum_created_vulnerabilities(self, obj):
return obj.sum_created_vulnerabilities
def get_sum_created_hosts(self, obj):
return obj.sum_created_hosts
def get_sum_created_services(self, obj):
return obj.sum_created_services
def get_sum_created_vulnerability_critical(self, obj):
return obj.sum_created_vulnerability_critical
class Meta:
model = Command
fields = ('_id', 'command', 'ip', 'hostname',
'params', 'user', 'workspace', 'tool',
'import_source', 'itime', 'sum_created_vulnerabilities',
'sum_created_hosts', 'sum_created_services', 'sum_created_vulnerability_critical', 'creator')
class HostSchema(AutoSchema):
_id = fields.Integer(dump_only=True, attribute='id')
id = fields.Integer()
_rev = fields.String(default='', dump_only=True)
ip = fields.String(default='')
description = fields.String(required=True) # Explicitly set required=True
default_gateway = NullToBlankString(attribute="default_gateway_ip",
required=False)
name = fields.String(dump_only=True, attribute='ip', default='')
os = fields.String(default='')
owned = fields.Boolean(default=False)
owner = PrimaryKeyRelatedField('username',
attribute='creator',
dump_only=True)
services = fields.Integer(attribute='open_service_count', dump_only=True)
vulns = fields.Integer(attribute='vulnerability_count', dump_only=True)
credentials = fields.Integer(attribute='credentials_count', dump_only=True)
hostnames = MutableField(
PrimaryKeyRelatedField('name',
many=True,
attribute="hostnames",
dump_only=True,
default=[]), fields.List(fields.String))
metadata = SelfNestedField(MetadataSchema())
type = fields.Function(lambda obj: 'Host', dump_only=True)
service_summaries = fields.Method('get_service_summaries', dump_only=True)
versions = fields.Method('get_service_version', dump_only=True)
important = fields.Boolean(default=False)
severity_counts = SelfNestedField(HostCountSchema(), dump_only=True)
class Meta:
model = Host
fields = ('id', '_id', '_rev', 'ip', 'description', 'mac',
'credentials', 'default_gateway', 'metadata', 'name', 'os',
'owned', 'owner', 'services', 'vulns', 'hostnames', 'type',
'service_summaries', 'versions', 'important',
'severity_counts')
def get_service_summaries(self, obj):
return [
service.summary for service in obj.services
if service.status == 'open'
]
def get_service_version(self, obj):
return [
service.version for service in obj.services
if service.status == 'open'
]
class WorkspaceSchema(AutoSchema):
name = fields.String(required=True,
validate=validate.Regexp(
r"^[a-z0-9][a-z0-9\_\$\(\)\+\-\/]*$", 0,
"ERORROROR"))
stats = SelfNestedField(WorkspaceSummarySchema())
duration = SelfNestedField(WorkspaceDurationSchema())
_id = fields.Integer(dump_only=True, attribute='id')
scope = MutableField(
PrimaryKeyRelatedField('name', many=True, dump_only=True),
fields.List(fields.String))
active = fields.Boolean(dump_only=True)
create_date = fields.DateTime(attribute='create_date', dump_only=True)
update_date = fields.DateTime(attribute='update_date', dump_only=True)
class Meta:
model = Workspace
fields = ('_id', 'id', 'customer', 'description', 'active', 'duration',
'name', 'public', 'scope', 'stats', 'create_date',
'update_date', 'readonly')
@post_load
def post_load_duration(self, data):
# Unflatten duration (move data[duration][*] to data[*])
duration = data.pop('duration', None)
if duration:
data.update(duration)
if 'start_date' in data and 'end_date' in data and data[
'start_date'] and data['end_date']:
if data['start_date'] > data['end_date']:
raise ValidationError("start_date is bigger than end_date.")
return data
class AgentSchema(AutoSchema):
_id = fields.Integer(dump_only=True, attribute='id')
status = fields.String(dump_only=True)
creator = PrimaryKeyRelatedField('username', dump_only=True, attribute='creator')
token = fields.String(dump_only=True)
create_date = fields.DateTime(dump_only=True)
update_date = fields.DateTime(dump_only=True)
is_online = fields.Boolean(dump_only=True)
executors = fields.Nested(ExecutorSchema(), dump_only=True, many=True)
last_run = fields.DateTime(dump_only=True)
class Meta:
model = Agent
fields = (
'id',
'name',
'status',
'active',
'create_date',
'update_date',
'creator',
'token',
'is_online',
'active',
'executors',
'last_run'
)
class CommandSchema(AutoSchema):
_id = fields.Integer(dump_only=True, attribute='id')
itime = fields.Method(serialize='get_itime',
deserialize='load_itime',
required=True,
attribute='start_date')
duration = MutableField(fields.Method(serialize='get_duration'),
fields.Integer(),
allow_none=True)
workspace = PrimaryKeyRelatedField('name', dump_only=True)
creator = PrimaryKeyRelatedField('username', dump_only=True)
metadata = SelfNestedField(MetadataSchema())
def load_itime(self, value):
try:
return datetime.datetime.utcfromtimestamp(value)
except ValueError:
raise ValidationError('Invalid Itime Value')
def get_itime(self, obj):
return obj.start_date.replace(tzinfo=pytz.utc).timestamp() * 1000
def get_duration(self, obj):
# obj.start_date can't be None
if obj.end_date:
return (obj.end_date - obj.start_date).seconds + (
(obj.end_date - obj.start_date).microseconds / 1000000.0)
else:
if (datetime.datetime.utcnow() - obj.start_date
).total_seconds() > 86400: # 86400 is 1d TODO BY CONFIG
return 'Timeout'
return 'In progress'
@post_load
def post_load_set_end_date_with_duration(self, data, **kwargs):
# there is a potential bug when updating, the start_date can be changed.
duration = data.pop('duration', None)
if duration:
data['end_date'] = data['start_date'] + datetime.timedelta(
seconds=duration)
return data
class Meta:
model = Command
fields = ('_id', 'command', 'duration', 'itime', 'ip', 'hostname',
'params', 'user', 'creator', 'workspace', 'tool',
'import_source', 'metadata')
class VulnerabilityTemplateSchema(AutoSchema):
_id = fields.Integer(dump_only=True, attribute='id')
id = fields.Integer(dump_only=True, attribute='id')
_rev = fields.String(default='', dump_only=True)
cwe = fields.String(dump_only=True, default='') # deprecated field, the legacy data is added to refs on import
exploitation = SeverityField(attribute='severity', required=True)
references = fields.Method('get_references', deserialize='load_references', required=True)
refs = fields.List(fields.String(), dump_only=True, attribute='references')
desc = fields.String(dump_only=True, attribute='description')
data = fields.String(attribute='data')
impact = SelfNestedField(ImpactSchema())
easeofresolution = fields.String(
attribute='ease_of_resolution',
validate=OneOf(Vulnerability.EASE_OF_RESOLUTIONS),
allow_none=True)
policyviolations = fields.List(fields.String,
attribute='policy_violations')
creator = PrimaryKeyRelatedField('username', dump_only=True, attribute='creator')
create_at = fields.DateTime(attribute='create_date',
dump_only=True)
# Here we use vulnerability instead of vulnerability_template to avoid duplicate row
# in the custom_fields_schema table.
# All validation will be against vulnerability table.
customfields = FaradayCustomField(table_name='vulnerability', attribute='custom_fields')
external_id = fields.String(allow_none=True)
class Meta:
model = VulnerabilityTemplate
fields = ('id', '_id', '_rev', 'cwe', 'description', 'desc',
'exploitation', 'name', 'references', 'refs', 'resolution',
'impact', 'easeofresolution', 'policyviolations', 'data',
'customfields', 'external_id', 'creator', 'create_at')
def get_references(self, obj):
return ', '.join(map(lambda ref_tmpl: ref_tmpl.name, obj.reference_template_instances))
def load_references(self, value):
if isinstance(value, list):
references = value
elif isinstance(value, (unicode, str)):
if len(value) == 0:
# Required because "".split(",") == [""]
return []
references = [ref.strip() for ref in value.split(',')]
else:
raise ValidationError('references must be a either a string '
'or a list')
if any(len(ref) == 0 for ref in references):
raise ValidationError('Empty name detected in reference')
return references
@post_load
def post_load_impact(self, data):
# Unflatten impact (move data[impact][*] to data[*])
impact = data.pop('impact', None)
if impact:
data.update(impact)
return data
class CommandSchema(AutoSchema):
_id = fields.Integer(dump_only=True, attribute='id')
itime = fields.Method(serialize='get_itime',
deserialize='load_itime',
required=True,
attribute='start_date')
duration = fields.Method(serialize='get_duration', allow_none=True)
workspace = PrimaryKeyRelatedField('name', dump_only=True)
creator = PrimaryKeyRelatedField('username', dump_only=True)
def load_itime(self, value):
return datetime.datetime.fromtimestamp(value)
def get_itime(self, obj):
return time.mktime(obj.start_date.utctimetuple()) * 1000
def get_duration(self, obj):
# obj.start_date can't be None
if obj.end_date:
return (obj.end_date - obj.start_date).seconds + (
(obj.end_date - obj.start_date).microseconds / 1000000.0)
else:
if (datetime.datetime.now() - obj.start_date
).total_seconds() > 86400: # 86400 is 1d TODO BY CONFIG
return 'Timeout'
return 'In progress'
@post_load
def post_load_set_end_date_with_duration(self, data):
# there is a potential bug when updating, the start_date can be changed.
duration = data.pop('duration', None)
if duration:
data['end_date'] = data['start_date'] + datetime.timedelta(
seconds=duration)
class Meta:
model = Command
fields = ('_id', 'command', 'duration', 'itime', 'ip', 'hostname',
'params', 'user', 'creator', 'workspace', 'tool',
'import_source')
class WorkspaceSchema(AutoSchema):
name = fields.String(required=True, validate=validate_workspace_name)
stats = SelfNestedField(WorkspaceSummarySchema())
duration = SelfNestedField(WorkspaceDurationSchema())
_id = fields.Integer(dump_only=True, attribute='id')
scope = MutableField(
PrimaryKeyRelatedField('name', many=True, dump_only=True),
fields.List(fields.String))
active = fields.Boolean()
create_date = fields.DateTime(attribute='create_date', dump_only=True)
update_date = fields.DateTime(attribute='update_date', dump_only=True)
active_agents_count = fields.Integer(dump_only=True)
last_run_agent_date = fields.DateTime(dump_only=True,
attribute='last_run_agent_date')
histogram = fields.Nested(HistogramSchema(many=True))
class Meta:
model = Workspace
fields = ('_id', 'id', 'customer', 'description', 'active', 'duration',
'name', 'public', 'scope', 'stats', 'create_date',
'update_date', 'readonly', 'active_agents_count',
'last_run_agent_date', 'histogram')
@post_load
def post_load_duration(self, data, **kwargs):
# Unflatten duration (move data[duration][*] to data[*])
duration = data.pop('duration', None)
if duration:
data.update(duration)
if 'start_date' in data and 'end_date' in data and data[
'start_date'] and data['end_date']:
if data['start_date'] > data['end_date']:
raise ValidationError("start_date is bigger than end_date.")
return data
class VulnerabilitySchema(AutoSchema):
_id = fields.Integer(dump_only=True, attribute='id')
_rev = fields.String(dump_only=True, default='')
_attachments = fields.Method(serialize='get_attachments',
deserialize='load_attachments',
default=[])
owned = fields.Boolean(dump_only=True, default=False)
owner = PrimaryKeyRelatedField('username',
dump_only=True,
attribute='creator')
impact = SelfNestedField(ImpactSchema())
desc = fields.String(attribute='description')
description = fields.String(dump_only=True)
policyviolations = fields.List(fields.String,
attribute='policy_violations')
refs = fields.List(fields.String(), attribute='references')
issuetracker = fields.Method(serialize='get_issuetracker', dump_only=True)
parent = fields.Method(serialize='get_parent',
deserialize='load_parent',
required=True)
parent_type = MutableField(fields.Method('get_parent_type'),
fields.String(),
required=True)
tags = PrimaryKeyRelatedField('name', dump_only=True, many=True)
easeofresolution = fields.String(attribute='ease_of_resolution',
validate=OneOf(
Vulnerability.EASE_OF_RESOLUTIONS),
allow_none=True)
hostnames = PrimaryKeyRelatedField('name', many=True, dump_only=True)
service = fields.Nested(ServiceSchema(only=[
'_id', 'ports', 'status', 'protocol', 'name', 'version', 'summary'
]),
dump_only=True)
host = fields.Integer(dump_only=True, attribute='host_id')
severity = SeverityField(required=True)
status = fields.Method(serialize='get_status',
validate=OneOf(Vulnerability.STATUSES + ['opened']),
deserialize='load_status')
type = fields.Method(serialize='get_type',
deserialize='load_type',
required=True)
obj_id = fields.String(dump_only=True, attribute='id')
target = fields.String(dump_only=True, attribute='target_host_ip')
host_os = fields.String(dump_only=True, attribute='target_host_os')
metadata = SelfNestedField(CustomMetadataSchema())
date = fields.DateTime(attribute='create_date',
dump_only=True) # This is only used for sorting
custom_fields = FaradayCustomField(table_name='vulnerability',
attribute='custom_fields')
external_id = fields.String(allow_none=True)
class Meta:
model = Vulnerability
fields = ('_id', 'status', 'issuetracker', 'description', 'parent',
'parent_type', 'tags', 'severity', '_rev',
'easeofresolution', 'owned', 'hostnames', 'owner', 'date',
'data', 'refs', 'desc', 'impact', 'confirmed', 'name',
'service', 'obj_id', 'type', 'policyviolations',
'_attachments', 'target', 'host_os', 'resolution',
'metadata', 'custom_fields', 'external_id')
def get_type(self, obj):
return obj.__class__.__name__
def get_attachments(self, obj):
res = {}
for file_obj in obj.evidence:
try:
ret, errors = EvidenceSchema().dump(file_obj)
if errors:
raise ValidationError(errors, data=ret)
res[file_obj.filename] = ret
except IOError:
logger.warning("File not found. Did you move your server?")
return res
def load_attachments(self, value):
return value
def get_parent(self, obj):
return obj.service_id or obj.host_id
def get_parent_type(self, obj):
assert obj.service_id is not None or obj.host_id is not None
return 'Service' if obj.service_id is not None else 'Host'
def get_status(self, obj):
if obj.status == 'open':
return 'opened'
return obj.status
def get_issuetracker(self, obj):
return {}
def load_status(self, value):
if value == 'opened':
return 'open'
return value
def load_type(self, value):
if value == 'Vulnerability':
return 'vulnerability'
if value == 'VulnerabilityWeb':
return 'vulnerability_web'
else:
raise ValidationError('Invalid vulnerability type.')
def load_parent(self, value):
try:
# sometimes api requests send str or unicode.
value = int(value)
except ValueError:
raise ValidationError("Invalid parent type")
return value
@post_load
def post_load_impact(self, data):
# Unflatten impact (move data[impact][*] to data[*])
impact = data.pop('impact', None)
if impact:
data.update(impact)
return data
@post_load
def post_load_parent(self, data):
# schema guarantees that parent_type exists.
parent_class = None
parent_type = data.pop('parent_type', None)
parent_id = data.pop('parent', None)
if not (parent_type and parent_id):
# Probably a partial load, since they are required
return
if parent_type == 'Host':
parent_class = Host
parent_field = 'host_id'
if parent_type == 'Service':
parent_class = Service
parent_field = 'service_id'
if not parent_class:
raise ValidationError('Unknown parent type')
if parent_type == 'Host' and data['type'] == 'vulnerability_web':
raise ValidationError(
'Trying to set a host for a vulnerability web')
try:
parent = db.session.query(parent_class).join(Workspace).filter(
Workspace.name == self.context['workspace_name'],
parent_class.id == parent_id).one()
except NoResultFound:
raise ValidationError('Parent id not found: {}'.format(parent_id))
data[parent_field] = parent.id
# TODO migration: check what happens when updating the parent from
# service to host or viceverse
return data
class ProfileSchema(Schema):
user = PrimaryKeyRelatedField('username')
first_name = fields.String()
class UserSchema(Schema):
username = fields.String()
blogposts = PrimaryKeyRelatedField(many=True)
class UserSchemaWithTitle(UserSchema):
blogposts = PrimaryKeyRelatedField('title', many=True)
class ServiceSchema(AutoSchema):
_id = fields.Integer(attribute='id', dump_only=True)
_rev = fields.String(default='', dump_only=True)
owned = fields.Boolean(default=False)
owner = PrimaryKeyRelatedField('username', dump_only=True,
attribute='creator')
port = fields.Integer(dump_only=True, required=True,
validate=[Range(min=0, error="The value must be greater than or equal to 0")]) # Port is loaded via ports
ports = MutableField(fields.Integer(required=True,
validate=[Range(min=0, error="The value must be greater than or equal to 0")]),
fields.Method(deserialize='load_ports'),
required=True,
attribute='port')
status = fields.String(missing='open', validate=OneOf(Service.STATUSES),
allow_none=False)
parent = fields.Integer(attribute='host_id') # parent is not required for updates
host_id = fields.Integer(attribute='host_id', dump_only=True)
vulns = fields.Integer(attribute='vulnerability_count', dump_only=True)
credentials = fields.Integer(attribute='credentials_count', dump_only=True)
metadata = SelfNestedField(MetadataSchema())
type = fields.Function(lambda obj: 'Service', dump_only=True)
summary = fields.String(dump_only=True)
def load_ports(self, value):
if not isinstance(value, list):
raise ValidationError('ports must be a list')
if len(value) != 1:
raise ValidationError('ports must be a list with exactly one'
'element')
port = value.pop()
if isinstance(port, str):
try:
port = int(port)
except ValueError:
raise ValidationError('The value must be a number')
if port > 65535 or port < 1:
raise ValidationError('The value must be in the range [1-65535]')
return str(port)
@post_load
def post_load_parent(self, data, **kwargs):
"""Gets the host_id from parent attribute. Pops it and tries to
get a Host with that id in the corresponding workspace.
"""
host_id = data.pop('host_id', None)
if self.context['updating']:
if host_id is None:
# Partial update?
return data
if host_id != self.context['object'].parent.id:
raise ValidationError('Can\'t change service parent.')
else:
if not host_id:
raise ValidationError('Parent id is required when creating a service.')
try:
data['host'] = Host.query.join(Workspace).filter(
Workspace.name == self.context['workspace_name'],
Host.id == host_id
).one()
except NoResultFound:
raise ValidationError(f'Host with id {host_id} not found')
return data
class Meta:
model = Service
fields = ('id', '_id', 'status', 'parent', 'type',
'protocol', 'description', '_rev',
'owned', 'owner', 'credentials', 'vulns',
'name', 'version', '_id', 'port', 'ports',
'metadata', 'summary', 'host_id')