def test_types(self):
with pytest.raises(fields.MarshallingError):
class WrongType:
pass
x = WrongType()
field1 = fields.Wildcard(WrongType) # noqa
field2 = fields.Wildcard(x) # noqa
def test_marshal_wildcard_with_skip_none(self):
wild = fields.Wildcard(fields.String)
model = OrderedDict([('foo', fields.Raw), ('*', wild)])
marshal_dict = OrderedDict([('foo', None), ('bat', None),
('baz', 'biz'), ('bar', None)])
output = marshal(marshal_dict, model, skip_none=True)
assert output == {'baz': 'biz'}
class DetectorDto:
api = Namespace("detector", description="Detect objects using Detectron2.")
detect_success = api.model(
"Detect success response",
{
"status": fields.Boolean,
"message": fields.String,
"job_id": fields.String,
},
)
detect_task_info = api.model(
"Task metadata",
{
"id": fields.String(allow_none=True),
"state": fields.String(allow_none=True),
},
)
detect_results = api.model(
"Detect results response (with info about job)",
{
"status": fields.Boolean,
"message": fields.String,
"processor_task": fields.Nested(detect_task_info),
"detector_chord": fields.Nested(detect_task_info),
"result": fields.List(fields.Wildcard(fields.String),
allow_none=True),
},
)
def test_marshal_wildcard_nested(self):
nest = fields.Nested(
OrderedDict([('thumbnail', fields.String),
('video', fields.String)]))
wild = fields.Wildcard(nest)
wildcard_fields = OrderedDict([('*', wild)])
model = OrderedDict([('preview', fields.Nested(wildcard_fields))])
sub_dict = OrderedDict([('9:16', {
'thumbnail': 24,
'video': 12
}), ('16:9', {
'thumbnail': 25,
'video': 11
}), ('1:1', {
'thumbnail': 26,
'video': 10
})])
marshal_dict = OrderedDict([('preview', sub_dict)])
output = marshal(marshal_dict, model)
assert output == {
'preview': {
'1:1': {
'thumbnail': '26',
'video': '10'
},
'16:9': {
'thumbnail': '25',
'video': '11'
},
'9:16': {
'thumbnail': '24',
'video': '12'
}
}
}
def test_marshal_wildcard_with_skip_none(self):
wild = fields.Wildcard(fields.String)
model = OrderedDict([("foo", fields.Raw), ("*", wild)])
marshal_dict = OrderedDict([("foo", None), ("bat", None),
("baz", "biz"), ("bar", None)])
output = marshal(marshal_dict, model, skip_none=True)
assert output == {"baz": "biz"}
def test_marshal_wildcard_with_envelope(self):
wild = fields.Wildcard(fields.String)
model = OrderedDict([("foo", fields.Raw), ("*", wild)])
marshal_dict = OrderedDict(
[("foo", {"bat": "baz"}), ("a", "toto"), ("b", "tata")]
)
output = marshal(marshal_dict, model, envelope="hey")
assert output == {"hey": {"a": "toto", "b": "tata", "foo": {"bat": "baz"}}}
def test_defaults(self):
field = fields.Wildcard(fields.String)
assert not field.required
assert field.__schema__ == {
"type": "object",
"additionalProperties": {
"type": "string"
},
}
def test_list_of_raw(self):
field = fields.Wildcard(fields.Raw)
data = [{'a': 1, 'b': 1}, {'a': 2, 'b': 1}, {'a': 3, 'b': 1}]
expected = [OrderedDict([('a', 1), ('b', 1)]),
OrderedDict([('a', 2), ('b', 1)]),
OrderedDict([('a', 3), ('b', 1)])]
self.assert_field(field, data, expected)
data = [1, 2, 'a']
self.assert_field(field, data, data)
def test_list_of_raw(self):
field = fields.Wildcard(fields.Raw)
data = [{"a": 1, "b": 1}, {"a": 2, "b": 1}, {"a": 3, "b": 1}]
expected = [
OrderedDict([("a", 1), ("b", 1)]),
OrderedDict([("a", 2), ("b", 1)]),
OrderedDict([("a", 3), ("b", 1)]),
]
self.assert_field(field, data, expected)
data = [1, 2, "a"]
self.assert_field(field, data, data)
def test_marshal_wildcard_list(self):
wild = fields.Wildcard(fields.List(fields.String))
wildcard_fields = OrderedDict([('*', wild)])
model = OrderedDict([('preview', fields.Nested(wildcard_fields))])
sub_dict = OrderedDict([
('1:1', [1, 2, 3]),
('16:9', [4, 5, 6]),
('9:16', [7, 8, 9])
])
marshal_dict = OrderedDict([('preview', sub_dict)])
output = marshal(marshal_dict, model)
assert output == {'preview': {'9:16': ['7', '8', '9'],
'16:9': ['4', '5', '6'],
'1:1': ['1', '2', '3']}}
def test_clone(self, api):
wild1 = fields.Wildcard(fields.String)
wild2 = wild1.clone()
wild_fields1 = api.model('cloneWildcard1', {'*': wild1})
wild_fields2 = api.model('cloneWildcard2', {'*': wild2})
data = {'John': 12, 'bob': 42, 'Jane': '68'}
expected1 = {'John': '12', 'bob': '42', 'Jane': '68'}
result1 = api.marshal(data, wild_fields1)
result2 = api.marshal(data, wild_fields2)
assert expected1 == result1
assert result2 == result1
def test_marshal_wildcard_list(self):
wild = fields.Wildcard(fields.List(fields.String))
wildcard_fields = OrderedDict([("*", wild)])
model = OrderedDict([("preview", fields.Nested(wildcard_fields))])
sub_dict = OrderedDict([("1:1", [1, 2, 3]), ("16:9", [4, 5, 6]),
("9:16", [7, 8, 9])])
marshal_dict = OrderedDict([("preview", sub_dict)])
output = marshal(marshal_dict, model)
assert output == {
"preview": {
"9:16": ["7", "8", "9"],
"16:9": ["4", "5", "6"],
"1:1": ["1", "2", "3"],
}
}
def test_clone(self, api):
wild1 = fields.Wildcard(fields.String)
wild2 = wild1.clone()
wild_fields1 = api.model("cloneWildcard1", {"*": wild1})
wild_fields2 = api.model("cloneWildcard2", {"*": wild2})
data = {"John": 12, "bob": 42, "Jane": "68"}
expected1 = {"John": "12", "bob": "42", "Jane": "68"}
result1 = api.marshal(data, wild_fields1)
result2 = api.marshal(data, wild_fields2)
assert expected1 == result1
assert result2 == result1
def test_with_scoped_attribute_on_dict_or_obj(self):
class Test(object):
def __init__(self, data):
self.data = data
class Nested(object):
def __init__(self, value):
self.value = value
nesteds = [Nested(i) for i in ['a', 'b', 'c']]
test_obj = Test(nesteds)
test_dict = {'data': [{'value': 'a'}, {'value': 'b'}, {'value': 'c'}]}
field = fields.Wildcard(fields.String(attribute='value'), attribute='data')
assert ['a' == 'b', 'c'], field.output('whatever', test_obj)
assert ['a' == 'b', 'c'], field.output('whatever', test_dict)
def test_marshal_wildcard_with_envelope(self):
wild = fields.Wildcard(fields.String)
model = OrderedDict([('foo', fields.Raw), ('*', wild)])
marshal_dict = OrderedDict([('foo', {
'bat': 'baz'
}), ('a', 'toto'), ('b', 'tata')])
output = marshal(marshal_dict, model, envelope='hey')
assert output == {
'hey': {
'a': 'toto',
'b': 'tata',
'foo': {
'bat': 'baz'
}
}
}
class TaskDto():
api = Namespace('tasks', description='Task api information')
model_content = api.model(
'content', {
'text': fields.String(required=True, description='text value'),
'created_on': fields.Date(description='YYYY-MM-DD'),
'updated_on': fields.Date(description='YYYY-MM-DD'),
})
model_nest = fields.Nested(model_content)
model_wild = fields.Wildcard(model_nest)
task = api.model('task', {'*': model_wild})
def test_with_scoped_attribute_on_dict_or_obj(self):
class Test(object):
def __init__(self, data):
self.data = data
class Nested(object):
def __init__(self, value):
self.value = value
nesteds = [Nested(i) for i in ["a", "b", "c"]]
test_obj = Test(nesteds)
test_dict = {"data": [{"value": "a"}, {"value": "b"}, {"value": "c"}]}
field = fields.Wildcard(fields.String(attribute="value"),
attribute="data")
assert ["a" == "b", "c"], field.output("whatever", test_obj)
assert ["a" == "b", "c"], field.output("whatever", test_dict)
def test_marshal_wildcard_nested(self):
nest = fields.Nested(
OrderedDict([("thumbnail", fields.String),
("video", fields.String)]))
wild = fields.Wildcard(nest)
wildcard_fields = OrderedDict([("*", wild)])
model = OrderedDict([("preview", fields.Nested(wildcard_fields))])
sub_dict = OrderedDict([
("9:16", {
"thumbnail": 24,
"video": 12
}),
("16:9", {
"thumbnail": 25,
"video": 11
}),
("1:1", {
"thumbnail": 26,
"video": 10
}),
])
marshal_dict = OrderedDict([("preview", sub_dict)])
output = marshal(marshal_dict, model)
assert output == {
"preview": {
"1:1": {
"thumbnail": "26",
"video": "10"
},
"16:9": {
"thumbnail": "25",
"video": "11"
},
"9:16": {
"thumbnail": "24",
"video": "12"
},
}
}
key_model = api.model('EntityKey', {'path': fields.Raw})
sync_state_model = api.model(
'SyncState',
{
'syncing': fields.Boolean(default=False),
'successful': fields.Boolean(default=False),
'error': fields.String,
'enqueued_at': fields.DateTime,
'started_at': fields.DateTime,
'updated_at': fields.DateTime,
},
)
_settings_wild = fields.Wildcard(
fields.Wildcard(fields.Wildcard(fields.String)))
settings_model = api.model(
'Settings',
{
'*': _settings_wild,
},
)
service_model = api.model(
'Service',
{
'credentials': CredentialsField(default=False),
'sync_enabled': fields.Boolean(default=False),
'sync_state': fields.Nested(sync_state_model),
'settings': fields.Nested(settings_model, skip_none=True),
},
from flask_restx import fields
from api.restplus import api
location = api.model('Resource Location', {
'location': fields.String(required=True)
})
message = api.model('Message', {
'message': fields.String(required=True)
})
bad_request = api.inherit('Bad request', message, {
'erorrs': fields.Wildcard(fields.String)
})
category_output = api.model('Category Output', {
'id': fields.Integer(required=True),
'name': fields.String(required=True, description='Category name')
})
category_input = api.model('Category Input', {
'name': fields.String(required=True, description='Category name')
})
generic_file_output = api.model('Generic File Output', {
'id': fields.Integer(required=True),
'file_name': fields.String(required=True),
'link': fields.String(required=True),
'created_at': fields.String(required=True)
example="SHOTDETECTION"),
'homepage':
fields.String(
description='Generator homepage',
required=True,
example="https://github.com/beeldengeluid/shot-detection-worker.git"
),
'type':
fields.String(description='Generator type',
required=True,
example="Software",
enum=["Organization", "Human", "Software"]),
})
_anyField = api.model('AnyField', {
'*': fields.Wildcard(fields.Raw),
})
_document = api.model(
'Document', {
'_id':
fields.String(description='DANE Assigned Document ID',
required=False,
example="KJfYfHQBqBJknIB4zrJL"),
'target':
fields.Nested(_target, description='Document target', required=True),
'creator':
fields.Nested(
_creator, description='Document creator/owner', required=True),
'created_at':
fields.String(description='Creation time',
fields.Nested(BlackboxIsolationForestApi),
"knearest_neighbors":
fields.Nested(BlackboxKNNApi),
"local_outlier_factor":
fields.Nested(BlackboxLOFApi),
},
)
BlackboxResponseApi = Model("BlackboxResponse", {"message": fields.String()})
BlackboxTrainResponseApi = BlackboxResponseApi.inherit(
"BlackboxTrainResponse", {"task_status": fields.String()})
BlackboxResponseErrorApi = Model(
"BlackboxResponseError",
{
"errors": fields.Wildcard(fields.String),
"message": fields.String
},
)
BlackboxResponseTaskApi = Model(
"BlackboxResponseTask",
{
"state": fields.String,
"current": fields.Float,
"total": fields.Integer,
"status": fields.String,
},
)
from mycodo.mycodo_client import DaemonControl
from mycodo.mycodo_flask.api import api
from mycodo.mycodo_flask.api import default_responses
from mycodo.mycodo_flask.api.sql_schema_fields import output_channel_fields
from mycodo.mycodo_flask.api.sql_schema_fields import output_fields
from mycodo.mycodo_flask.api.utils import get_from_db
from mycodo.mycodo_flask.api.utils import return_list_of_dictionaries
from mycodo.mycodo_flask.utils import utils_general
from mycodo.mycodo_flask.utils.utils_output import get_all_output_states
logger = logging.getLogger(__name__)
ns_output = api.namespace('outputs', description='Output operations')
MODEL_STATES_STATE = ns_output.model('states', {
'*': fields.Wildcard(fields.String(description='on, off, or a duty cycle'),)
})
MODEL_STATES_CHAN = ns_output.model('channels', {
'*': fields.Wildcard(fields.Nested(
MODEL_STATES_STATE,
description='Dictionary with channel as key and state data as value.'))
})
output_list_fields = ns_output.model('Output Fields List', {
'output devices': fields.List(fields.Nested(output_fields)),
'output channels': fields.List(fields.Nested(output_channel_fields)),
'output states': fields.Nested(
MODEL_STATES_CHAN,
description='Dictionary with ID as key and channel state data as value.')
})
output_unique_id_fields = ns_output.model('Output Device Fields List', {
})
project_resp_model = api.model('ProjectResponse', {
'id': fields.String(example='project1', description='Project ID'),
'name': fields.String(example='Project 1', description='Project name')
})
project_records_resp = fields.List(
fields.String, example=['bcd04c45-3cfc-456f-a31e-59e875aefabf.json'])
curated_mapping = fields.Raw(example={'12345': '23456'})
project_curation = fields.Wildcard(fields.Nested(curation_model), example={
'12345': {
'project_id': 'project1',
'statement_id': '83f5aec2-978b-4e01-a2c9-e231f90bfabd',
'update_type': 'discard_statement'
},
})
stmt_fields = fields.Raw(example={
"id": "acc6d47c-f622-41a4-8ae9-d7b0f3d24a2f",
"type": "Influence",
"subj": {"db_refs": {"WM": "wm/concept/causal_factor/environmental/meteorologic/precipitation/rainfall"}, "name": "rainfall"},
"obj": {"db_refs": {"WM": "wm/concept/causal_factor/crisis_and_disaster/environmental_disasters/natural_disaster/flooding"}, "name": "flood"},
"evidence": [{"text": "Rainfall causes flood", "source_api": "eidos"}]
}, description='INDRA Statement JSON')
stmts_model = api.model('Statements', {
'statements': fields.List(stmt_fields)
})
"DBInner",
{
"last_update":
fields.DateTime(description="Date time of last update",
example="2019-09-30T18:53:48"),
"size":
fields.Integer(
description="Amount of documents in collection",
example="570",
),
},
)
database_wild = fields.Wildcard(
fields.Nested(database_inner),
description="Database collection mapping id",
skip_none=True,
)
database_entry = api.model(
"documents",
{"*": database_wild},
)
PostBodyRequest = api.model(
"ApiPostRequest",
{
"retrieve":
fields.String(
description=
"Retrieve data from this collection, allowed options are 'capec', 'cpe', 'cves', 'cwe', 'via4'",
description="time spent reading from disk (in milliseconds)"),
"write_time":
fields.Integer(
description="time spent writing to disk (in milliseconds)"),
"busy_time":
fields.Integer(
description="time spent doing actual I/Os (in milliseconds)"),
"read_merged_count":
fields.Integer(description="number of merged reads"),
"write_merged_count":
fields.Integer(description="number of merged writes"),
},
)
nest = fields.Nested(io_counters_model)
wild = fields.Wildcard(nest)
io_counters_wild = disk.model("io_counters_", {
"*": wild,
})
disk_usage_model = disk.model(
"disk_usage",
{
"total": fields.Integer,
"used": fields.Integer,
"free": fields.Integer,
"percent": fields.Float,
},
)
disk_partitions_model = disk.model(
fields.Integer(description="total number of errors while receiving"),
"errout":
fields.Integer(description="total number of errors while sending"),
"dropin":
fields.Integer(
description="total number of incoming packets which were dropped"),
"dropout":
fields.Integer(
description=
"total number of outgoing packets which were dropped (always 0 on macOS and BSD)"
),
},
)
io_counters_nest = fields.Nested(net_io_counters_model)
io_counters_wild = fields.Wildcard(io_counters_nest)
net_io_counters_wild = network.model("net_io_counters_", {
"*": io_counters_wild,
})
net_if_stats_model = network.model(
"net_if_stats",
{
"isup":
fields.Boolean(
description="a bool indicating whether the NIC is up and running."
),
"duplex":
fields.String(
description=
"the duplex communication type; it can be either NIC_DUPLEX_FULL, NIC_DUPLEX_HALF or NIC_DUPLEX_UNKNOWN.",
def test_wildcard(self, api):
wild1 = fields.Wildcard(fields.String)
wild2 = fields.Wildcard(fields.Integer)
wild3 = fields.Wildcard(fields.String)
wild4 = fields.Wildcard(fields.String, default="x")
wild5 = fields.Wildcard(fields.String)
wild6 = fields.Wildcard(fields.Integer)
wild7 = fields.Wildcard(fields.String)
wild8 = fields.Wildcard(fields.String)
mod5 = OrderedDict()
mod5["toto"] = fields.Integer
mod5["bob"] = fields.Integer
mod5["*"] = wild5
wild_fields1 = api.model("WildcardModel1", {"*": wild1})
wild_fields2 = api.model("WildcardModel2", {"j*": wild2})
wild_fields3 = api.model("WildcardModel3", {"*": wild3})
wild_fields4 = api.model("WildcardModel4", {"*": wild4})
wild_fields5 = api.model("WildcardModel5", mod5)
wild_fields6 = api.model(
"WildcardModel6",
{
"nested": {
"f1": fields.String(default="12"),
"f2": fields.Integer(default=13),
},
"a*": wild6,
},
)
wild_fields7 = api.model("WildcardModel7", {"*": wild7})
wild_fields8 = api.model("WildcardModel8", {"*": wild8})
class Dummy(object):
john = 12
bob = "42"
alice = None
class Dummy2(object):
pass
class Dummy3(object):
a = None
b = None
data = {"John": 12, "bob": 42, "Jane": "68"}
data3 = Dummy()
data4 = Dummy2()
data5 = {"John": 12, "bob": 42, "Jane": "68", "toto": "72"}
data6 = {"nested": {"f1": 12, "f2": 13}, "alice": "14"}
data7 = Dummy3()
data8 = None
expected1 = {"John": "12", "bob": "42", "Jane": "68"}
expected2 = {"John": 12, "Jane": 68}
expected3 = {"john": "12", "bob": "42"}
expected4 = {"*": "x"}
expected5 = {"John": "12", "bob": 42, "Jane": "68", "toto": 72}
expected6 = {"nested": {"f1": "12", "f2": 13}, "alice": 14}
expected7 = {}
expected8 = {}
result1 = api.marshal(data, wild_fields1)
result2 = api.marshal(data, wild_fields2)
result3 = api.marshal(data3, wild_fields3, skip_none=True)
result4 = api.marshal(data4, wild_fields4)
result5 = api.marshal(data5, wild_fields5)
result6 = api.marshal(data6, wild_fields6)
result7 = api.marshal(data7, wild_fields7, skip_none=True)
result8 = api.marshal(data8, wild_fields8, skip_none=True)
assert expected1 == result1
assert expected2 == result2
assert expected3 == result3
assert expected4 == result4
assert expected5 == result5
assert expected6 == result6
assert expected7 == result7
assert expected8 == result8
config_model = NS.model('FTVConfig', {
'schemas': fields.List(fields.String, required=True, description='The schema URLs in the configuration file')
})
ftv_info_model = NS.model('FTVInfo', {
'version': fields.String(required=True, description = 'API Version'),
'config': fields.Nested(config_model, required=True, description = 'Public configuration bits')
})
########################
# The different models #
########################
SCHEMAS_NS = Namespace('schemas','Schemas being used for validation')
wci = fields.Wildcard(fields.Raw)
schema_error_model = SCHEMAS_NS.model('SchemaError', {
'*': wci
})
# Lesson learned: fields.Url is for self generated URIs
schema_info_model = SCHEMAS_NS.model('SchemaInfo', {
'source_urls': fields.List(fields.String,required=True, description = 'URL location of the JSON Schema'),
'fetched_at': fields.DateTime,
'errors': fields.List(fields.Nested(schema_error_model),required=False, description = 'The list of detected errors when the JSON Schema was initially processed'),
'schema_hash': fields.String(required=False, description = 'The SHA1 hash of the normalized JSON Schema, in hexadecimal representation'),
'schema_id': fields.String(required=False, description = 'The id of the schema'),
})
#schema_info_model_schema = SCHEMAS_NS.schema_model('SchemaInfo', {
def test_wildcard(self, api):
wild1 = fields.Wildcard(fields.String)
wild2 = fields.Wildcard(fields.Integer)
wild3 = fields.Wildcard(fields.String)
wild4 = fields.Wildcard(fields.String, default='x')
wild5 = fields.Wildcard(fields.String)
wild6 = fields.Wildcard(fields.Integer)
wild7 = fields.Wildcard(fields.String)
wild8 = fields.Wildcard(fields.String)
mod5 = OrderedDict()
mod5['toto'] = fields.Integer
mod5['bob'] = fields.Integer
mod5['*'] = wild5
wild_fields1 = api.model('WildcardModel1', {'*': wild1})
wild_fields2 = api.model('WildcardModel2', {'j*': wild2})
wild_fields3 = api.model('WildcardModel3', {'*': wild3})
wild_fields4 = api.model('WildcardModel4', {'*': wild4})
wild_fields5 = api.model('WildcardModel5', mod5)
wild_fields6 = api.model('WildcardModel6', {
'nested': {'f1': fields.String(default='12'), 'f2': fields.Integer(default=13)},
'a*': wild6
})
wild_fields7 = api.model('WildcardModel7', {'*': wild7})
wild_fields8 = api.model('WildcardModel8', {'*': wild8})
class Dummy(object):
john = 12
bob = '42'
alice = None
class Dummy2(object):
pass
class Dummy3(object):
a = None
b = None
data = {'John': 12, 'bob': 42, 'Jane': '68'}
data3 = Dummy()
data4 = Dummy2()
data5 = {'John': 12, 'bob': 42, 'Jane': '68', 'toto': '72'}
data6 = {'nested': {'f1': 12, 'f2': 13}, 'alice': '14'}
data7 = Dummy3()
data8 = None
expected1 = {'John': '12', 'bob': '42', 'Jane': '68'}
expected2 = {'John': 12, 'Jane': 68}
expected3 = {'john': '12', 'bob': '42'}
expected4 = {'*': 'x'}
expected5 = {'John': '12', 'bob': 42, 'Jane': '68', 'toto': 72}
expected6 = {'nested': {'f1': '12', 'f2': 13}, 'alice': 14}
expected7 = {}
expected8 = {}
result1 = api.marshal(data, wild_fields1)
result2 = api.marshal(data, wild_fields2)
result3 = api.marshal(data3, wild_fields3, skip_none=True)
result4 = api.marshal(data4, wild_fields4)
result5 = api.marshal(data5, wild_fields5)
result6 = api.marshal(data6, wild_fields6)
result7 = api.marshal(data7, wild_fields7, skip_none=True)
result8 = api.marshal(data8, wild_fields8, skip_none=True)
assert expected1 == result1
assert expected2 == result2
assert expected3 == result3
assert expected4 == result4
assert expected5 == result5
assert expected6 == result6
assert expected7 == result7
assert expected8 == result8
