class GithubItemsByTime(jsl.Document):
class Options(object):
definition_id = "github_issue&prs_with_time_duration"
description = "Details of Github issues + prs yearly or monthly or any given date-range"
opened = jsl.IntField(required=True)
closed = jsl.IntField(required=True)
class GithubLastYearCommits(jsl.Document):
class Options(object):
definition_id = "github_last_year_commits_details"
description = "Details of last year Github commits"
sum = jsl.IntField(required=True)
weekly = jsl.ArrayField(jsl.IntField(), required=True)
class PortBinding(jsl.Document):
external = jsl.IntField(description="External (host) port number.",
minimum=1,
maximum=65536)
internal = jsl.IntField(description="Internal (container) port number.",
minimum=1,
maximum=65536)
class GithubDetail(jsl.Document):
"""JSL schema for Github worker results details."""
class Options(object):
"""JSL schema for Github worker results details."""
definition_id = "github_extracted_details"
description = "Details of Github inspection"
# we don't mandate any of these fields, because they may not be present
forks_count = jsl.IntField()
last_year_commits = jsl.DocumentField(GithubLastYearCommits, as_ref=True)
open_issues_count = jsl.IntField()
stargazers_count = jsl.IntField()
subscribers_count = jsl.IntField()
with removed_in(ROLE_v2_0_0) as until_v2_0_0:
until_v2_0_0.updated_issues = jsl.DocumentField(GithubUpdatedIssues,
as_ref=True)
until_v2_0_0.updated_pull_requests = jsl.DocumentField(
GithubUpdatedPullRequests, as_ref=True)
with added_in(ROLE_v1_0_2) as since_v1_0_2:
since_v1_0_2.contributors_count = jsl.IntField()
with jsl.Scope(ROLE_v1_0_3) as v1_0_3:
v1_0_3.topics = jsl.ArrayField(jsl.StringField(), required=True)
with added_in(ROLE_v1_0_4) as since_v1_0_4:
since_v1_0_4.topics = jsl.ArrayField(jsl.StringField())
with added_in(ROLE_v2_0_1) as since_v2_0_1:
since_v2_0_1.license = jsl.DictField()
with added_in(ROLE_v2_0_2) as since_v2_0_2:
since_v2_0_2.updated_on = jsl.StringField(required=True)
class DHCPService(jsl.Document):
leasetime = jsl.StringField(
description="Duration of client leases, e.g. 2h.", pattern="\d+[dhms]")
limit = jsl.IntField(
description="Size of address range beginning at start value.",
minimum=1)
start = jsl.IntField(description="Starting offset for address assignment.",
minimum=3)
class DiffDefinition(jsl.Document):
class Options(object):
definition_id = "diff"
description = "Information about changed files and lines"
files = jsl.IntField(required=True)
lines = jsl.IntField(required=True)
changes = jsl.ArrayField(jsl.DocumentField(ChangeDefinition, as_ref=True), required=True)
class RSyncInfo(jsl.Document):
path = jsl.ArrayField([
jsl.IntField(),
jsl.StringField()
])
version = jsl.ArrayField([
jsl.IntField(),
jsl.StringField()
])
class LinguistOutput(jsl.Document):
class Options(object):
definition_id = "linguist_output"
description = "Linguist output for one file"
lines = jsl.IntField(required=True)
sloc = jsl.IntField(required=True)
type = jsl.StringField(required=True)
language = jsl.StringField(required=True)
mime = jsl.StringField(required=True)
def dataclass_field_to_jsl_field(prop: dataclasses.Field,
nullable=False) -> jsl.BaseField:
t = dataclass_check_type(prop, date)
if t:
return jsl.DateTimeField(name=prop.name, required=t['required'])
t = dataclass_check_type(prop, datetime)
if t:
return jsl.DateTimeField(name=prop.name, required=t['required'])
t = dataclass_check_type(prop, str)
if t:
return jsl.StringField(name=prop.name, required=t['required'])
t = dataclass_check_type(prop, int)
if t:
return jsl.IntField(name=prop.name, required=t['required'])
t = dataclass_check_type(prop, float)
if t:
return jsl.NumberField(name=prop.name, required=t['required'])
t = dataclass_check_type(prop, bool)
if t:
return jsl.BooleanField(name=prop.name, required=t['required'])
t = dataclass_check_type(prop, dict)
if t:
return jsl.DictField(name=prop.name, required=t['required'])
t = dataclass_check_type(prop, ISchema)
if t:
subtype = jsonobject_to_jsl(t['schema'], nullable=nullable)
return jsl.DocumentField(name=prop.name,
document_cls=subtype,
required=t['required'])
t = dataclass_check_type(prop, list)
if t:
return jsl.ArrayField(name=prop.name, required=t['required'])
t = dataclass_check_type(prop, typing.List)
if t:
if 'schema' not in t.keys():
return jsl.ArrayField(name=prop.name, required=t['required'])
if issubclass(t['schema'], ISchema):
subtype = jsl.DocumentField(
document_cls=jsonobject_to_jsl(t['schema'], nullable=nullable))
elif t['schema'] == str:
subtype = jsl.StringField(name=prop.name)
elif t['schema'] == int:
subtype = jsl.IntField(name=prop.name)
elif t['schema'] == float:
subtype = jsl.NumberField(name=prop.name)
elif t['schema'] == dict:
subtype = jsl.DictField(name=prop.name)
else:
raise KeyError(t['schema'])
return jsl.ArrayField(items=subtype, required=t['required'])
raise KeyError(prop)
class GithubUpdatedPullRequests(GithubUpdatedIssues):
class Options(object):
definition_id = "github_pull_requests_details"
description = "Details of updated Github pull requests"
with jsl.Scope(ROLE_v1_0_0) as v1_0_0:
v1_0_0.open = jsl.IntField(required=True)
v1_0_0.closed = jsl.IntField(required=True)
with added_in(ROLE_v1_0_1) as since_v1_0_1:
since_v1_0_1.year = jsl.DocumentField(GithubItemsByTime, as_ref=True)
since_v1_0_1.month = jsl.DocumentField(GithubItemsByTime, as_ref=True)
class MODecimalField(MOBaseField):
type = jsl.StringField(enum=['boolean'],
required=True,
description='Data type identifier')
precision = jsl.IntField(
minimum=1,
required=True,
description='Total number of digits. E.g. 123.45 has a precision of 5')
scale = jsl.IntField(
minimum=1,
required=True,
description='Total number of digits representing numbers less than one. '
'E.g. 123.45 has a scale of 2')
class ApiSchema78(BaseApiSchema):
"""Schema for siem rule in API format."""
STACK_VERSION = "7.8"
RULE_TYPES = [MACHINE_LEARNING, SAVED_QUERY, QUERY]
actions = jsl.ArrayField(required=False)
description = jsl.StringField(required=True)
# api defaults to false if blank
enabled = jsl.BooleanField(default=False, required=False)
# _ required since `from` is a reserved word in python
from_ = jsl.StringField(required=False, default='now-6m', name='from')
false_positives = jsl.ArrayField(jsl.StringField(), required=False)
filters = jsl.ArrayField(jsl.DocumentField(Filters))
interval = jsl.StringField(pattern=INTERVAL_PATTERN, default='5m', required=False)
max_signals = jsl.IntField(minimum=1, required=False, default=100) # cap a max?
meta = jsl.DictField(required=False)
name = jsl.StringField(required=True)
note = MarkdownField(required=False)
# output_index =jsl.StringField(required=False) # this is NOT allowed!
references = jsl.ArrayField(jsl.StringField(), required=False)
risk_score = jsl.IntField(minimum=0, maximum=100, required=True, default=21)
severity = jsl.StringField(enum=['low', 'medium', 'high', 'critical'], default='low', required=True)
tags = jsl.ArrayField(jsl.StringField(), required=False)
throttle = jsl.StringField(required=False)
timeline_id = jsl.StringField(required=False)
timeline_title = jsl.StringField(required=False)
to = jsl.StringField(required=False, default='now')
type = jsl.StringField(enum=[MACHINE_LEARNING, QUERY, SAVED_QUERY], required=True)
threat = jsl.ArrayField(jsl.DocumentField(Threat), required=False, min_items=1)
with jsl.Scope(MACHINE_LEARNING) as ml_scope:
ml_scope.anomaly_threshold = jsl.IntField(required=True, minimum=0)
ml_scope.machine_learning_job_id = jsl.StringField(required=True)
ml_scope.type = jsl.StringField(enum=[MACHINE_LEARNING], required=True, default=MACHINE_LEARNING)
with jsl.Scope(SAVED_QUERY) as saved_id_scope:
saved_id_scope.index = jsl.ArrayField(jsl.StringField(), required=False)
saved_id_scope.saved_id = jsl.StringField(required=True)
saved_id_scope.type = jsl.StringField(enum=[SAVED_QUERY], required=True, default=SAVED_QUERY)
with jsl.Scope(QUERY) as query_scope:
query_scope.index = jsl.ArrayField(jsl.StringField(), required=False)
# this is not required per the API but we will enforce it here
query_scope.language = jsl.StringField(enum=['kuery', 'lucene'], required=True, default='kuery')
query_scope.query = jsl.StringField(required=True)
query_scope.type = jsl.StringField(enum=[QUERY], required=True, default=QUERY)
with jsl.Scope(jsl.DEFAULT_ROLE) as default_scope:
default_scope.type = type
class GithubUpdatedIssues(jsl.Document):
"""JSL schema for Details of updated Github issues."""
class Options(object):
"""JSL schema for Details of updated Github issues."""
definition_id = "github_issues_details"
description = "Details of updated Github issues"
with jsl.Scope(ROLE_v1_0_0) as v1_0_0:
v1_0_0.open = jsl.IntField(required=True)
v1_0_0.closed = jsl.IntField(required=True)
with added_in(ROLE_v1_0_1) as since_v1_0_1:
since_v1_0_1.year = jsl.DocumentField(GithubItemsByTime, as_ref=True)
since_v1_0_1.month = jsl.DocumentField(GithubItemsByTime, as_ref=True)
def jsonobject_property_to_jsl_field(prop: jsonobject.JsonProperty,
nullable=False) -> jsl.BaseField:
if isinstance(prop, jsonobject.DateProperty):
return jsl.DateTimeField(name=prop.name, required=prop.required)
if isinstance(prop, jsonobject.DateTimeProperty):
return jsl.DateTimeField(name=prop.name, required=prop.required)
if isinstance(prop, jsonobject.StringProperty):
return jsl.StringField(name=prop.name, required=prop.required)
if isinstance(prop, jsonobject.IntegerProperty):
return jsl.IntField(name=prop.name, required=prop.required)
if isinstance(prop, jsonobject.FloatProperty):
return jsl.NumberField(name=prop.name, required=prop.required)
if isinstance(prop, jsonobject.BooleanProperty):
return jsl.BooleanField(name=prop.name, required=prop.required)
if isinstance(prop, jsonobject.DictProperty):
if prop.item_wrapper:
subtype = jsonobject_to_jsl(prop.item_wrapper.item_type,
nullable=nullable)
return jsl.DocumentField(name=prop.name,
document_cls=subtype,
required=prop.required)
return jsl.DictField(name=prop.name, required=prop.required)
if isinstance(prop, jsonobject.ListProperty):
if prop.item_wrapper:
if isinstance(prop.item_wrapper, jsonobject.ObjectProperty):
if issubclass(prop.item_wrapper.item_type,
jsonobject.JsonObject):
subtype = jsl.DocumentField(document_cls=jsonobject_to_jsl(
prop.item_wrapper.item_type),
nullable=nullable)
elif isinstance(prop.item_wrapper.item_type,
jsonobject.JsonProperty):
subtype = jsonobject_property_to_jsl_field(
prop.item_wrapper.item_type)
else:
raise KeyError(prop.item_wrapper.item_type)
elif isinstance(prop.item_wrapper, jsonobject.StringProperty):
subtype = jsl.StringField(name=prop.name)
elif isinstance(prop.item_wrapper, jsonobject.IntegerProperty):
subtype = jsl.IntField(name=prop.name)
elif isinstance(prop.item_wrapper, jsonobject.FloatProperty):
subtype = jsl.NumberField(name=prop.name)
elif isinstance(prop.item_wrapper, jsonobject.DictProperty):
subtype = jsl.DictField(name=prop.name)
else:
raise KeyError(prop.item_wrapper)
return jsl.ArrayField(items=subtype, required=prop.required)
return jsl.ArrayField(name=prop.name, required=prop.required)
raise KeyError(prop)
class WirelessOptions(jsl.Document):
ssid = jsl.StringField(description="ESSID to broadcast.", max_length=32)
key = jsl.StringField(description="Wireless network password.",
min_length=8)
nasid = jsl.StringField(description="NAS identifier for RADIUS.")
acct_server = jsl.StringField(description="RADIUS accounting server.")
acct_secret = jsl.StringField(description="RADIUS accounting secret.")
acct_interval = jsl.IntField(
description="RADIUS accounting update interval (seconds).", minimum=1)
hidden = jsl.BooleanField(
description="Disable broadcasting the ESSID in beacons.")
isolate = jsl.BooleanField(
description="Disable forwarding traffic between connected clients.")
maxassoc = jsl.IntField(
description="Maximum number of associated clients.", minimum=0)
class ObjectSchema(CRUDSchema):
id = jsl.IntField(required=False)
uuid = jsl.StringField(required=False)
body = jsl.StringField(required=True, default='')
created_flag = jsl.BooleanField(required=False, default=False)
updated_flag = jsl.BooleanField(required=False, default=False)
class MOStringField(MOBaseField):
type = jsl.StringField(enum=['string'],
required=True,
description='Data type identifier')
size = jsl.IntField(minimum=1,
required=True,
description='Maximum number of code points in string')
class CortexExpSchemaJSLBase(jsl.Document):
"""class defining json schema for a database record. See top of file"""
timestamp = jsl.StringField(format="date-time", required=True)
monkey = jsl.StringField(enum=monkeylist, required=True)
session_number = jsl.IntField(minimum=1, maximum=999, required=True)
code_repo = jsl.DocumentField(schemautil.GitRepoRef, required=True)
experiment_name = jsl.StringField(
required=True, pattern=schemautil.StringPatterns.relativePathPattern)
timing_file_name = jsl.StringField(
pattern=schemautil.StringPatterns.strictFilenameLowerPattern('tm'),
required=True)
condition_file_name = jsl.StringField(
pattern=schemautil.StringPatterns.strictFilenameLowerPattern('cnd'),
required=True)
item_file_name = jsl.StringField(
pattern=schemautil.StringPatterns.strictFilenameLowerPattern('itm'),
required=True)
parameter_file_name = jsl.StringField(
pattern=schemautil.StringPatterns.strictFilenameLowerPattern('par'),
required=True)
set_file_name = jsl.StringField(
pattern=schemautil.StringPatterns.strictFilenameLowerPattern('set'),
required=True)
recorded_files = jsl.DocumentField(
schemautil.filetransfer.FileTransferSiteAndFileListRemote,
required=True)
additional_parameters = jsl.DictField(required=True)
notes = jsl.StringField(required=True)
class MappingCount(jsl.Document):
"""Mapping count schema."""
count = jsl.IntField(minimum=0, required=True)
rta_name = jsl.StringField(pattern=r'[a-zA-Z-_]+', required=True)
rule_name = jsl.StringField(required=True)
sources = jsl.ArrayField(jsl.StringField(), min_items=1)
class LicenseScanDetails(jsl.Document):
class Options(object):
definition_id = "license_scan_details"
additional_properties = True
with removed_in(ROLE_v3_0_0) as removed_in_v3_0_0:
removed_in_v3_0_0.files = jsl.ArrayField(
jsl.DocumentField(FileDetails, as_ref=True))
removed_in_v3_0_0.license_stats = jsl.ArrayField(
jsl.DocumentField(LicenseDetailsPre30, as_ref=True))
removed_in_v3_0_0.oslc_stats = jsl.DocumentField(OSLCStats,
as_ref=True)
with added_in(ROLE_v3_0_0) as added_in_v3_0_0:
added_in_v3_0_0.files_count = jsl.IntField(required=True)
added_in_v3_0_0.licenses = jsl.DictField(pattern_properties=jsl.Var({
'role': {
'*': jsl.DocumentField(LicenseDetails,
as_ref=True,
required=True),
}
}),
required=True)
added_in_v3_0_0.scancode_notice = jsl.StringField(required=True)
added_in_v3_0_0.scancode_version = jsl.StringField(required=True)
class TodoSchema(jsl.Document):
"""
A Todo schema
Attributes:
id (int):
A unique id for the todo.
description (str):
A text description of the todo.
items (array):
An array of sub-todos of this todo.
skip_if (array):
An array of conditions to skip this todo. If any of the
condition is true, the todo is skipped. Each condition is a
dictionary of attributes and predicates which get ANDed together.
remove_if (array):
An array of conditions to remove this todo. If any of the
condition is true, the todo is removed. Each condition is a
dictionary of attributes and predicates which get ANDed together.
"""
id = jsl.IntField(required=True)
description = jsl.StringField(required=True)
items = jsl.ArrayField(jsl.DocumentField('TodoSchema'))
skip_if = jsl.ArrayField(
jsl.DictField(
pattern_properties={'.*': jsl.DocumentField('PredicateSchema')}))
remove_if = jsl.ArrayField(
jsl.DictField(
pattern_properties={'.*': jsl.DocumentField('PredicateSchema')}))
class Endpoint(OwnedObject):
class Options(object):
definition_id = 'endpoint'
# TODO: Maybe do jsl.OneOf(jsl.IPv4Field, jsl.StringField) ?>??
address = jsl.StringField()
port = jsl.IntField(minimum=1, maximum=65535)
class Dependents(jsl.Document):
"""DSL schema for dependents count."""
class Options(object):
"""DSL schema for dependents count."""
definition_id = "libraries_io_dependents"
count = jsl.IntField(required=True)
class Releases(jsl.Document):
"""DSL schema for releases."""
class Options(object):
"""DSL schema for releases."""
definition_id = "libraries_io_releases"
count = jsl.IntField(required=True)
recent = jsl.ArrayField(jsl.DocumentField(RecentRelease, as_ref=True))
class DependenciesCounts(jsl.Document):
"""JSL schema for Counts of various types of dependencies."""
class Options(object):
"""JSL schema for Counts of various types of dependencies."""
definition_id = 'dependencies_counts'
description = 'Counts of various types of dependencies'
runtime = jsl.IntField()
class Person(jsl.Document):
# class Options:
# definition_id = 'PERSON'
# title = 'Person'
# description = 'A person'
fname = jsl.StringField(required=True)
lname = jsl.StringField(required=True)
age = jsl.IntField(required=True)
class CortexExpSortedSchemaJSL(jsl.Document):
schema_revision = jsl.IntField(enum=[1], required=True) # the version of schema, in case we have drastic change
cortex_exp_ref = jsl.StringField(format=schemautil.StringPatterns.bsonObjectIdPattern, required=True)
files_to_sort = jsl.DocumentField(schemautil.filetransfer.FileTransferSiteAndFileListRemote, required=True)
sorted_files = jsl.DocumentField(schemautil.filetransfer.FileTransferSiteAndFileListRemoteAuto, required=True)
sort_method = jsl.StringField(enum=sort_methods, required=True)
sort_config = jsl.DictField(required=True) # arbitrary dict to save the parameters for this sort.
sort_person = jsl.StringField(enum=sort_people, required=True) # who sorted.
timestamp = jsl.StringField(format="date-time", required=True)
notes = jsl.StringField(required=True)
class CortexExpSchemaJSLR2(CortexExpSchemaJSLBase):
schema_revision = jsl.IntField(
enum=[2],
required=True) # the version of schema, in case we have drastic change
lut_file_name = jsl.StringField(
pattern=schemautil.StringPatterns.strictFilenameLowerPattern('lut'),
required=True)
blocking_file_name = jsl.StringField(
pattern=schemautil.StringPatterns.strictFilenameLowerPattern('blk'),
required=True)
class MOIntegerField(MOBaseField):
type = jsl.StringField(enum=['integer'],
required=True,
description='Data type identifier')
bytes = jsl.IntField(
minimum=1,
maximum=8,
required=True,
description=
'Number of bytes needed to store value. Not the number of bytes/characters of text'
)
class MOBooleanField(MOBaseField):
type = jsl.StringField(enum=['boolean'],
required=True,
description='Data type identifier')
size = jsl.IntField(
minimum=1,
required=True,
description='Number of code points represented in text')
true_value = jsl.StringField(
required=True, description='The textual representation of TRUE')
false_value = jsl.StringField(
required=True, description='The textual representation of FALSE')
