def __init__(self, args):
self.test_config = TestConfig.load_from_file(args.tests)
self.classifier_config = Config.load_from_file(args.classify_config)
model_file = self.classifier_config.classify.model
if args.model_file:
model_file = args.model_file
path, ext = os.path.splitext(model_file)
keras_model = False
if ext == ".pb":
keras_model = True
self.clip_classifier = ClipClassifier(
self.classifier_config,
self.classifier_config.classify_tracking,
model_file,
keras_model,
)
# try download missing tests
if args.user and args.password:
api = UserAPI(args.server, args.user, args.password)
out_dir = Path(self.test_config.clip_dir)
if not out_dir.exists():
out_dir.mkdir()
for test in self.test_config.recording_tests:
filepath = out_dir / test.filename
if not filepath.exists():
if iter_to_file(out_dir / test.filename,
api.download_raw(test.rec_id)):
logging.info("Saved %s", filepath)
self.results = []
def setUp(self):
"""Initialize an instance of TestInfluxDBClient object."""
# By default, raise exceptions on warnings
warnings.simplefilter("error", FutureWarning)
self.cli = UserAPI(
baseurl=defaults["apiURL"],
username=defaults["defaultUsername"],
password=defaults["defaultuserPassword"],
)
def setUp(self):
"""Initialize an instance of mocked CacophonyServer object."""
# By default, raise exceptions on warnings
warnings.simplefilter("error", FutureWarning)
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"{apiURL}/authenticate_user".format(apiURL=defaults["apiURL"]),
status_code=204,
)
self.cli = UserAPI(
baseurl=defaults["apiURL"],
username=defaults["defaultUsername"],
password=defaults["defaultuserPassword"],
)
def test_scheme(self):
"""Set up the test schema for TestCacophonyClient object."""
cli = UserAPI(
baseurl=defaults["apiURL"],
username=defaults["defaultUsername"],
password=defaults["defaultuserPassword"],
)
self.assertEqual(defaults["apiURL"], cli._baseurl)
def process(self, url):
"""Downloads all requested files from specified server"""
self.ignored = 0
self.not_selected = 0
self.processed = 0
api = API(url, self.user, self.password)
if self.recording_id:
recording = api.get(self.recording_id)
self._download(recording, api, Path(self.out_folder))
return
print("Querying server {0}".format(url))
print("Limit is {0}".format(self.limit))
print("Tag mode {0}".format(self.tag_mode))
print("Dates are {0} - {1}".format(self.start_date, self.end_date))
print("Required tags are {0}".format(self.only_tags))
print("Ignore tags are {0}".format(self.ignore_tags))
pool = Pool(self.workers, self._downloader, api, Path(self.out_folder))
offset = 0
if len(self.only_tags) == 0:
self.only_tags = None
remaining = self.limit
while self.limit is None or offset < self.limit:
rows = api.query(
limit=remaining,
startDate=self.start_date,
endDate=self.end_date,
tagmode=self.tag_mode,
tags=self.only_tags,
offset=offset,
)
if len(rows) == 0:
break
offset += len(rows)
if remaining:
remaining -= len(rows)
if self.auto_delete:
self.update_file_locations()
for row in rows:
pool.put(row)
pool.stop()
def __init__(self, args):
self.test_config = TestConfig.load_from_file(args.tests)
self.classifier_config = Config.load_from_file(args.classify_config)
if args.model_file:
model_file = args.model_file
self.track_extractor = TrackExtractor(self.classifier_config)
# try download missing tests
if args.user and args.password:
api = UserAPI(args.server, args.user, args.password)
out_dir = Path(self.test_config.clip_dir)
if not out_dir.exists():
out_dir.mkdir()
for test in self.test_config.recording_tests:
filepath = out_dir / test.filename
if not filepath.exists():
if iter_to_file(
out_dir / test.filename, api.download_raw(test.rec_id)
):
logging.info("Saved %s", filepath)
self.results = []
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"server_url", help="Server (base) url to send the CPTV files to"
)
parser.add_argument("username", help="Username to upload as")
parser.add_argument("password", help="Password to authenticate with")
parser.add_argument("groupname", help="Group name to upload on behalf of")
parser.add_argument("devicename", help="Device to upload on behalf of")
parser.add_argument("filename", help="File to upload. If it is a directory will upload all cptv files in that dir")
args = parser.parse_args()
api = API(args.server_url, args.username, args.password)
if os.path.isdir(args.filename):
for file in os.listdir(args.filename):
if file.endswith(".cptv"):
filepath = os.path.join(args.filename, file)
print(filepath)
api.upload_recording(args.groupname, args.devicename, filepath)
else:
api.upload_recording(args.groupname, args.devicename, args.filename)
def main():
args = parse_args()
api = UserAPI(args.server, args.user, args.password)
out_dir = Path(args.out_folder)
tests = []
test_data = TestConfig(recording_tests=tests,
server=args.server,
clip_dir=args.out_folder)
for rec_id in args.ids:
rec_meta = api.get(rec_id)
tracks = api.get_tracks(rec_id)
filename = Path(rec_id).with_suffix(".cptv")
fullpath = out_dir / filename
tests.append(TestRecording.load_from_meta(rec_meta, tracks, filename))
if not fullpath.exists():
if iter_to_file(fullpath, api.download_raw(rec_id)):
print("Saved {} - {}".format(rec_id, fullpath))
else:
print("error saving {}".format(rec_id))
if os.path.exists(args.test_file):
os.remove(args.test_file)
with open(args.test_file, "w") as f:
yaml.dump(test_data, f)
def test_scheme(self):
"""Set up the test schema for mocked CacophonyServer object."""
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"{apiURL}/authenticate_user".format(apiURL=defaults["apiURL"]),
status_code=204,
)
cli = UserAPI(
baseurl=defaults["apiURL"],
username=defaults["defaultUsername"],
password=defaults["defaultuserPassword"],
)
# print(m.last_request.body)
self.assertEqual(defaults["apiURL"], cli._baseurl)
def test_version(self):
"""Test UserAPI.version"""
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"{apiURL}/authenticate_user".format(apiURL=defaults["apiURL"]),
status_code=204,
)
cli = UserAPI(
baseurl=defaults["apiURL"],
username=defaults["defaultUsername"],
password=defaults["defaultuserPassword"],
)
# print(m.last_request.body)
self.assertEqual(defaults["apiURL"], cli._baseurl)
versionResult = cli.version
print("version:{}".format(versionResult))
self.assertTrue(isinstance(versionResult, str))
self.assertTrue(len(versionResult) > 0)
class liveTESTcacophonyapi(unittest.TestCase):
"""Set up the TestCacophonyClient object."""
def setUp(self):
"""Initialize an instance of TestInfluxDBClient object."""
# By default, raise exceptions on warnings
warnings.simplefilter("error", FutureWarning)
self.cli = UserAPI(
baseurl=defaults["apiURL"],
username=defaults["defaultUsername"],
password=defaults["defaultuserPassword"],
)
def test_scheme(self):
"""Set up the test schema for TestCacophonyClient object."""
cli = UserAPI(
baseurl=defaults["apiURL"],
username=defaults["defaultUsername"],
password=defaults["defaultuserPassword"],
)
self.assertEqual(defaults["apiURL"], cli._baseurl)
def test_upload_recording(self):
"""Test UserAPI.upload_recordings ( parameters passed: groupname, devicename, filename, props) to mocked CacophonyServer object.
Default test user is read only so should fail with permission error"""
testcases = [
{
"filename": os.path.join(os.path.dirname(__file__),
"test1.cptv"),
"mock_file.side_effect": None,
"devicename": "test-device",
"groupname": "test-group",
"prop": None,
"expectedProp": {
"type": "thermalRaw"
},
"expectedResult": {
"outcome": "failureHTTPforbidden",
"validator": lambda test: test,
},
"mockRequestStatusCode": 403,
},
]
for tc in testcases:
print(tc)
if tc["expectedResult"]["outcome"] == "success":
# ----------------------- API call UNDERTEST ------------------
result = self.cli.upload_recording(tc["groupname"],
tc["devicename"],
tc["filename"], tc["prop"])
# ----------------------------------------------------------------
print(result)
self.assertTrue(result["success"])
elif tc["expectedResult"]["outcome"] == "failureHTTPforbidden":
try:
with self.assertRaises(requests.HTTPError):
# ----------------------- API call UNDERTEST ------------------
result = self.cli.upload_recording(
tc["groupname"],
tc["devicename"],
tc["filename"],
tc["prop"],
)
# ----------------------------------------------------------------
print(self)
# TODO: check the message returned
# self.assertTrue(result['success'])
except Exception as e:
self.fail("Unknown Exception_instance={})".format(e))
else:
# TODO: Check the mockrequest was called, and mockfile
self.assertTrue(
False,
"Something not being checked ----------------------------")
class Mocked_cacophonyapi(unittest.TestCase):
"""Test client calls to the mocked CacophonyServer object."""
def setUp(self):
"""Initialize an instance of mocked CacophonyServer object."""
# By default, raise exceptions on warnings
warnings.simplefilter("error", FutureWarning)
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"{apiURL}/authenticate_user".format(apiURL=defaults["apiURL"]),
status_code=204,
)
self.cli = UserAPI(
baseurl=defaults["apiURL"],
username=defaults["defaultUsername"],
password=defaults["defaultuserPassword"],
)
# print("SETUP: last request.body:{}".format(m.last_request.body))
def test_scheme(self):
"""Set up the test schema for mocked CacophonyServer object."""
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"{apiURL}/authenticate_user".format(apiURL=defaults["apiURL"]),
status_code=204,
)
cli = UserAPI(
baseurl=defaults["apiURL"],
username=defaults["defaultUsername"],
password=defaults["defaultuserPassword"],
)
# print(m.last_request.body)
self.assertEqual(defaults["apiURL"], cli._baseurl)
def test_version(self):
"""Test UserAPI.version"""
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"{apiURL}/authenticate_user".format(apiURL=defaults["apiURL"]),
status_code=204,
)
cli = UserAPI(
baseurl=defaults["apiURL"],
username=defaults["defaultUsername"],
password=defaults["defaultuserPassword"],
)
# print(m.last_request.body)
self.assertEqual(defaults["apiURL"], cli._baseurl)
versionResult = cli.version
print("version:{}".format(versionResult))
self.assertTrue(isinstance(versionResult, str))
self.assertTrue(len(versionResult) > 0)
def test_query(self):
"""Test UserAPI.query from mocked CacophonyServer object."""
testcases = [
{
"apiPATH": "/api/v1/recordings",
"parameters": {
"endDate": _strToSqlDateTime("2019-11-06 06:30:00"),
"startDate": _strToSqlDateTime("2019-11-01 19:00:00"),
"limit": 300,
"offset": 0,
"tagmode": "any",
},
"expectedRequestQS": {
"where": [
'{"recordingdatetime": {"$gte": "2019-11-01t19:00:00", "$lte": "2019-11-06t06:30:00"}}'
],
"limit": ["300"],
"offset": ["0"],
"tagmode": ["any"],
},
"expectedResult": {
"outcome": "success",
"validator": lambda test: test,
},
"mockRequestJsonResponse": {
"rows": {
"key1": "value1",
"key2": "value2"
}
},
"mockRequestStatusCode": 200,
},
# TODO: check expectedRequestQS 'tags'
{
"Description": """
All query paramters excluding raw_json
""",
"apiPATH": "/api/v1/recordings",
"parameters": {
"endDate": _strToSqlDateTime("2019-11-06 06:30:00"),
"startDate": _strToSqlDateTime("2019-11-01 19:00:00"),
"devices": ["test_device1", "test_device2"],
"type_": "thermalRaw",
"min_secs": 15, # duration
"limit": 300,
"offset": 0,
"tagmode": "any",
"tags": json.dumps('{fields:["animal"],unique:false}'),
},
"expectedRequestQS": {
"where": [
'{"type": "thermalraw", "duration": {"$gte": 15}, "recordingdatetime": {"$gte": "2019-11-01t19:00:00", "$lte": "2019-11-06t06:30:00"}, "deviceid": ["test_device1", "test_device2"]}'
],
"limit": ["300"],
"offset": ["0"],
"tagmode": ["any"],
"tags":
['"\\"{fields:[\\\\\\"animal\\\\\\"],unique:false}\\""'],
},
"expectedResult": {
"outcome": "success",
"validator": lambda test: test,
},
"mockRequestJsonResponse": {
"rows": {
"key1": "value1",
"key2": "value2"
}
},
"mockRequestStatusCode": 200,
},
# TODO: check expectedRequestQS 'tags'
{
"Description": """
All query paramters including raw_json=True
""",
"apiPATH": "/api/v1/recordings",
"parameters": {
"endDate": _strToSqlDateTime("2019-11-06 06:30:00"),
"startDate": _strToSqlDateTime("2019-11-01 19:00:00"),
"devices": ["test_device1", "test_device2"],
"type_": "thermalRaw",
"min_secs": 15, # duration
"limit": 300,
"offset": 0,
"tagmode": "any",
"tags": json.dumps('{fields:["animal"],unique:false}'),
"raw_json": True,
},
"expectedRequestQS": {
"where": [
'{"type": "thermalraw", "duration": {"$gte": 15}, "recordingdatetime": {"$gte": "2019-11-01t19:00:00", "$lte": "2019-11-06t06:30:00"}, "deviceid": ["test_device1", "test_device2"]}'
],
"limit": ["300"],
"offset": ["0"],
"tagmode": ["any"],
"tags":
['"\\"{fields:[\\\\\\"animal\\\\\\"],unique:false}\\""'],
},
"expectedResult": {
"outcome": "successRawData",
"validator": lambda test: test,
},
"mockRequestJsonResponse": {
"key1": "value1",
"key2": "value2"
},
"mockRequestStatusCode": 200,
},
]
for tc in testcases:
print(tc)
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"{apiURL}{apiPATH}".format(apiURL=defaults["apiURL"],
apiPATH=tc["apiPATH"]),
json=tc["mockRequestJsonResponse"],
status_code=200,
)
_ = self.cli.query(**tc["parameters"])
# _ = self.cli.query(
# endDate=_strToSqlDateTime("2019-11-06 06:30:00"),
# startDate=_strToSqlDateTime("2019-11-01 19:00:00"),
# limit=300,
# offset=0,
# tagmode="any")
print(m.last_request.qs)
self.assertEqual(m.last_request.qs, tc["expectedRequestQS"])
def test_get_valid_recordingId(self):
"""Test UserAPI.get with a valid recording_id from mocked CacophonyServer object."""
# TODO: handle error as correct result
int_recording_id = 432109
str_recording_id = "432109"
mock_json_result = {"key1": "value1", "key2": "value2"}
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"{apiURL}/api/v1/recordings/{recording_id}".format(
apiURL=defaults["apiURL"],
recording_id="{}".format(int_recording_id),
),
json={"recording": mock_json_result},
status_code=200,
)
result = self.cli.get(int_recording_id)
# print(m.last_request.qs)
self.assertEqual(result, mock_json_result)
self.assertEqual(m.last_request.qs, {})
self.assertEqual(
m.last_request.path,
"/api/v1/recordings/{str_recording_id}".format(
str_recording_id=str_recording_id),
)
def test_get_invalid_recordingId(self):
"""Test UserAPI.get with an invalid recording_id from mocked CacophonyServer object."""
int_recording_id = ""
str_recording_id = ""
mock_json_result = "some sort of error message"
# result=None
# TODO: What about exceptions 301, 100 etc
for status_code in [400, 422, 500, 501]:
with self.assertRaises(Exception) as context:
# Setup expected request
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"{apiURL}/api/v1/recordings/{recording_id}".format(
apiURL=defaults["apiURL"],
recording_id="{}".format(int_recording_id),
),
json={"message": mock_json_result},
status_code=status_code,
)
# TEST
_ = self.cli.get(int_recording_id)
if status_code in [400, 422]:
self.assertTrue(type(context.exception) == OSError)
self.assertTrue(
"request failed ({status_code}): {message}".format(
status_code=status_code, message=mock_json_result) in
str(context.exception))
elif status_code in [500, 501]:
# TODO: look at improved assertions
# print("{}".format(context.exception))
self.assertTrue(
type(context.exception) == requests.exceptions.HTTPError)
self.assertTrue("{status_code} Server Error: None".format(
status_code=status_code) in "{}".format(context.exception))
else:
self.assertFalse(True)
# TODO: check what exception was raised
# print("This exception raised:{}".format(context.exception))
def test_valid_get_tracks(self):
"""Test UserAPI.get_tracks with a valid recording_id from mocked CacophonyServer object."""
# TODO: handle error as correct result
int_recording_id = 432109
str_recording_id = "432109"
mock_json_result = {"key1": "value1", "key2": "value2"}
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"{apiURL}/api/v1/recordings/{recording_id}/tracks".format(
apiURL=defaults["apiURL"],
recording_id="{}".format(int_recording_id),
),
json=mock_json_result,
status_code=200,
)
result = self.cli.get_tracks(int_recording_id)
# print(m.last_request.qs)
self.assertEqual(result, mock_json_result)
self.assertEqual(m.last_request.qs, {})
self.assertEqual(
m.last_request.path,
"/api/v1/recordings/{str_recording_id}/tracks".format(
str_recording_id=str_recording_id),
)
def test_valid_get_groups(self):
"""Test UserAPI.get_groups (no parameters passed) from mocked CacophonyServer object."""
# TODO: handle error as correct result
mock_json_result = {"key1": "value1", "key2": "value2"}
mock_qs = {"where": ["{}"]}
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"{apiURL}/api/v1/groups".format(apiURL=defaults["apiURL"]),
json=mock_json_result,
status_code=200,
)
result = self.cli.get_groups_as_json()
# print(m.last_request.qs)
self.assertEqual(result, mock_json_result)
self.assertEqual(m.last_request.qs, mock_qs)
self.assertEqual(m.last_request.path, "/api/v1/groups")
def test_valid_get_devices(self):
"""Test UserAPI.get_devices (no parameters passed) from mocked CacophonyServer object."""
mock_apiPath = "/api/v1/devices"
mock_json_result = {
"devices": {
"rows": {
"key1": "value1",
"key2": "value2"
}
}
}
mock_qs = {"where": ["{}"]}
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"{apiURL}{apiPath}".format(apiURL=defaults["apiURL"],
apiPath=mock_apiPath),
json=mock_json_result,
status_code=200,
)
result = self.cli.get_devices_as_json()
# print(m.last_request.qs)
self.assertEqual(result, mock_json_result["devices"]["rows"])
self.assertEqual(m.last_request.qs, mock_qs)
self.assertEqual(m.last_request.path, mock_apiPath)
def test_valid_list_files(self):
"""Test UserAPI.list_files (no parameters passed) from mocked CacophonyServer object."""
mock_apiPath = "/api/v1/files"
mock_json_result = {"rows": {"key1": "value1", "key2": "value2"}}
mock_qs = {"where": ["{}"], "order": ['["id"]']}
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"{apiURL}{apiPath}".format(apiURL=defaults["apiURL"],
apiPath=mock_apiPath),
json=mock_json_result,
status_code=200,
)
result = self.cli.list_files()
# print(m.last_request.qs)
self.assertEqual(result, mock_json_result["rows"])
self.assertEqual(m.last_request.qs, mock_qs)
self.assertEqual(m.last_request.path, mock_apiPath)
def test_valid_delete_files(self):
"""Test UserAPI.delete_file (file_id passed) from mocked CacophonyServer object."""
mock_int_file_id = 123456
mock_str_file_id = "123456"
mock_apiPath = "/api/v1/files/{file_id}".format(
file_id=mock_int_file_id)
mock_json_result = {"rows": {"key1": "value1", "key2": "value2"}}
mock_qs = {}
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.DELETE,
"{apiURL}{apiPath}".format(apiURL=defaults["apiURL"],
apiPath=mock_apiPath),
json=mock_json_result,
status_code=200,
)
result = self.cli.delete_file(mock_int_file_id)
# print(m.last_request.qs)
self.assertEqual(result, mock_json_result)
self.assertEqual(m.last_request.qs, mock_qs)
self.assertEqual(m.last_request.path, mock_apiPath)
def test_valid_reprocess(self):
"""Test UserAPI.get_devices (no parameters passed) from mocked CacophonyServer object."""
# TODO: Construct the data to pass in the post
mock_apiPath = "/api/v1/reprocess"
mock_recordings = [1, 2, 3]
mock_postData = {"recordings": mock_recordings}
# mock_json_result = {"devices":{"rows":{'key1': 'value1', 'key2': 'value2'}}}
mock_qs = {}
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"{apiURL}{apiPath}".format(apiURL=defaults["apiURL"],
apiPath=mock_apiPath),
json=mock_postData,
status_code=200,
)
result = self.cli.reprocess(mock_recordings)
# print(m.last_request.qs)
self.assertEqual(result, mock_postData)
self.assertEqual(m.last_request.qs, mock_qs)
self.assertEqual(m.last_request.path, mock_apiPath)
def test_valid_upload_recording(self):
"""Test UserAPI.upload_recordings ( parameters passed: groupname, devicename, filename, props) to mocked CacophonyServer object.
# Range of expectedResult
'expectedResult':{'outcome':'success',
'validator':lambda test: test},
'expectedResult':{'outcome':'failurePreRequest',
'validator':lambda test: test},
'expectedResult':{'outcome':'failureOnRequest',
'validator':lambda test: test},
"""
# TODO: Construct the data to pass in the post
mock_apiPath = lambda devicename, groupname: "/api/v1/recordings/device/{devicename}/group/{groupname}".format(
devicename=devicename, groupname=groupname)
# TODO: construct the success validatior
# TODO: construct the failure validtor
testcases = [
{
"filename": "test1.cptv",
"mock_file.side_effect": None,
"devicename": "deviceTEST1234",
"groupname": "groupTEST1234",
"prop": None,
"expectedProp": {
"type": "thermalRaw"
},
"expectedResult": {
"outcome": "success",
"validator": lambda test: test,
},
"mockRequestJsonResponse": {
"success": True
},
"mockRequestStatusCode": 200,
},
# TODO: adjust test mock/asserts to handle IOError
# {'filename':"test1.cptv",
# 'mock_file.side_effect':IOError(),
# 'devicename':'deviceTEST1234',
# 'groupname':'groupTEST1234',
# 'prop':None,
# 'expectedProp':{'type':'thermalRaw'},
# 'mockRequestStatusCode':200
# },
{
"filename": "test2.mp3",
"mock_file.side_effect": None,
"devicename": "deviceTEST1234",
"groupname": "groupTEST1234",
"prop": None,
"expectedProp": {
"type": "audio"
},
"expectedResult": {
"outcome": "success",
"validator": lambda test: test,
},
"mockRequestJsonResponse": {
"success": True
},
"mockRequestStatusCode": 200,
},
# TODO: what assert/mock to do for unknown file type
{
"filename": "test3.xyz",
"mock_file.side_effect": None,
"devicename": "deviceTEST1234",
"groupname": "groupTEST1234",
"prop": None,
"expectedProp": None,
"expectedResult": {
"outcome": "failurePreRequest",
"validator": lambda test: test,
},
"mockRequestJsonResponse": None,
"mockRequestStatusCode": None,
},
]
for tc in testcases:
# TODO: pretty print
print(tc)
"""
This Test is tough to design/debug
made easier by using mock-open module https://github.com/nivbend/mock-open
mock the file to upload
Last, you also need to mock os.fstat() so that permissions are correct
(Clues in https://gist.github.com/dmerejkowsky/d11e3c68be6a96387dea3d8b6a409b40#file-test_netrc-py-L17)
"""
mock_open = MockOpen(read_data="0123456789012345678901234")
# mock_open = MockOpen(read_data='',side_effect=IOError('no file'))
mock_open[os.path.join(
os.path.dirname(__file__),
tc["filename"])].read_data = "0123456789012345678901234"
mock_open[os.path.join(
os.path.dirname(__file__),
tc["filename"])].side_effect = tc["mock_file.side_effect"]
fake_stat = mock.MagicMock("fstat")
# fake_stat.st_uid = # os.getuid()
fake_stat.st_mode = 0o600
fake_stat.st_size = 25
with patch("builtins.open",
mock_open), patch("os.fstat") as mock_fstat:
mock_fstat.return_value = fake_stat
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"{apiURL}{apiPath}".format(
apiURL=defaults["apiURL"],
apiPath=mock_apiPath(tc["devicename"],
tc["groupname"]),
),
json=tc["mockRequestJsonResponse"],
status_code=tc["mockRequestStatusCode"],
)
if tc["expectedResult"]["outcome"] == "success":
# ----------------------- API call UNDERTEST ------------------
result = self.cli.upload_recording(
tc["groupname"],
tc["devicename"],
tc["filename"],
tc["prop"],
)
# ----------------------------------------------------------------
print(result)
# TODO: FILL in the TEST assertions in place of the following
self.assertEqual(result, tc["mockRequestJsonResponse"])
self.assertTrue(
tc["expectedResult"]["validator"](True))
elif tc["expectedResult"][
"outcome"] == "failurePreRequest":
with self.assertRaises(ValueError):
# ----------------------- API call UNDERTEST ------------------
result = self.cli.upload_recording(
tc["groupname"],
tc["devicename"],
tc["filename"],
tc["prop"],
)
# ----------------------------------------------------------------
else:
# TODO: Check the mockrequest was called, and mockfile
self.assertTrue(
False,
"Something not being checked ----------------------------",
)
# mock_file.assert_called_with(tc['filename'], 'rb')
# self.assertEqual(m.last_request.path, mock_apiPath)
def test_valid_download(self):
"""Test UserAPI.download with a valid recording_id from mocked CacophonyServer object."""
int_recording_id = 432109
str_recording_id = "432109"
mock_apiPathTokenRequest = "/api/v1/recordings/{recording_id}".format(
recording_id="{}".format(int_recording_id))
mock_apiPathTokenProcess = "/api/v1/signedUrl"
mock_json_result = {"key1": "value1", "key2": "value2"}
token = "asfasdfdasfdasfadsf"
# First Test getting the download token
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"{apiURL}{apiPath}".format(apiURL=defaults["apiURL"],
apiPath=mock_apiPathTokenRequest),
json={"downloadFileJWT": token},
status_code=200,
)
recordingStreamGenerator = self.cli.download(int_recording_id)
self.assertEqual(type(recordingStreamGenerator),
types.GeneratorType)
self.assertEqual(m.last_request.qs, {})
self.assertEqual(
m.last_request.path,
"/api/v1/recordings/{str_recording_id}".format(
str_recording_id=str_recording_id),
)
# Now test the stream retrieval
# TODO: Is it necessary to consider smaller or bigger stream byte retures
expectedReturn1 = bytes([random.randint(1, 254) for _ in range(4096)])
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"{apiURL}{apiPath}".format(apiURL=defaults["apiURL"],
apiPath=mock_apiPathTokenProcess),
# params={"jwt":token},
content=expectedReturn1,
status_code=200,
)
buffer = next(recordingStreamGenerator)
self.assertEqual(buffer, expectedReturn1)
def test_valid__all_downloads(self):
"""Test UserAPI.download and Test UserAPI.download_raw with a valid recording_id from mocked CacophonyServer object."""
for test in [
{
"artifactID":
123490,
"apipath":
lambda int_recording_id: "/api/v1/recordings/{recording_id}".
format(recording_id=int_recording_id),
"UserAPIcall":
self.cli.download,
"UserAPIcallJSONresponse": {
"downloadFileJWT": "abcdTOKENdfg"
},
},
{
"artifactID":
123490,
"apipath":
lambda int_recording_id: "/api/v1/recordings/{recording_id}".
format(recording_id=int_recording_id),
"UserAPIcall":
self.cli.download_raw,
"UserAPIcallJSONresponse": {
"downloadRawJWT": "abcdTOKENdfg"
},
},
{
"artifactID":
123490,
"apipath":
lambda int_file_id: "/api/v1/files/{file_id}".format(
file_id=int_file_id),
"UserAPIcall":
self.cli.download_file,
"UserAPIcallJSONresponse": {
"file": "somedata",
"jwt": "abcdTOKENdfg"
},
},
]:
self._generic_download_test(downloadtype=test)
def _generic_download_test(self, downloadtype):
print(downloadtype)
mock_apiPathTokenRequest = downloadtype["apipath"](
downloadtype["artifactID"])
mock_apiPathTokenProcess = "/api/v1/signedUrl"
token = "asfasdfdasfdasfadsf"
# First Test getting the download token
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"{apiURL}{apiPath}".format(
apiURL=defaults["apiURL"],
apiPath=downloadtype["apipath"](
downloadtype["artifactID"]),
),
json=downloadtype["UserAPIcallJSONresponse"],
status_code=200,
)
result = downloadtype["UserAPIcall"](downloadtype["artifactID"])
# TODO: handle dowloadfile returns a tuple)
if isinstance((result), tuple):
streamGenerator = result[1]
self.assertIsNotNone(result[0])
else:
streamGenerator = result
self.assertEqual(type(streamGenerator), types.GeneratorType)
self.assertEqual(m.last_request.qs, {})
self.assertEqual(
m.last_request.path,
downloadtype["apipath"](downloadtype["artifactID"]))
# Now test the stream retrieval
# TODO: Is it necessary to consider smaller or bigger stream byte retures
expectedReturn1 = bytes([random.randint(1, 254) for _ in range(4096)])
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"{apiURL}{apiPath}".format(apiURL=defaults["apiURL"],
apiPath=mock_apiPathTokenProcess),
content=expectedReturn1,
status_code=200,
)
buffer = next(streamGenerator)
self.assertEqual(buffer, expectedReturn1)
# %%
test = ""
str(test)
# %%
from cacophonyapi.user import UserAPI
from cacophonyapi.config import Config
# %%
config = Config().load_config(
config_file=os.path.join(os.getcwd(), ".env", "defaultconfig.json"))
# %%
cp_client = UserAPI(
baseurl=config.api_url,
username=config.admin_username,
password=config.admin_password,
)
cp_client.version
# %% [markdown]
# ### SHOW devices and groups
# %%
pd.DataFrame(cp_client.get_devices_as_json())
# %%
pd.DataFrame(cp_client.get_groups_as_json()["groups"])
# %% [markdown]
#
def reprocess(args):
""" Reprocesses all specified recording from specified server """
api = API(args.server, args.user, args.password)
print(f"Querying server {args.server}")
print(f"Limit is {args.limit}")
if len(args.recording_id) == 1:
recordings = [
int(rec_id) for rec_id in args.recording_id[0].split(",") if rec_id
]
print(f"Reprocessing {recordings}")
api.reprocess(recordings)
return
hour_ago = datetime.now() - timedelta(hours=1)
where = {
"type": "thermalRaw",
"processingState": {
"$ne": "FINISHED"
},
"processingStartTime": {
"$or": {
"$gte": hour_ago.isoformat(),
"$eq": None
}
},
}
if args.algorithm_id:
where["$or"] = [
{
"additionalMetadata.algorithm": {
"$eq": None
}
},
{
"additionalMetadata.algorithm": {
"$lt": args.algorithm_id
}
},
]
else:
where["additionalMetadata.algorithm"] = {"$eq": None}
if len(args.recording_id) == 2:
if args.recording_id[0]:
where.setdefault("id", {})["$gte"] = int(args.recording_id[0])
if args.recording_id[1]:
where.setdefault("id", {})["$lte"] = int(args.recording_id[1])
q = api.query(where=where, limit=args.limit, raw_json=True)
count = q["count"]
if count:
print(f"Still reprocessing {count} records")
print([row["id"] for row in q["rows"]])
return
where["processingState"] = "FINISHED"
where["processingStartTime"] = {
"$or": {
"$lt": hour_ago.isoformat(),
"$eq": None
}
}
rows = api.query(where=where, limit=args.limit)
recording_ids = [row["id"] for row in rows]
print(f"Reprocessing recording ids: {recording_ids}")
api.reprocess(recording_ids)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("out_folder",
help="Folder to place downloaded files in")
parser.add_argument("user", help="API server username")
parser.add_argument("password", help="API server password")
parser.add_argument(
"-s",
"--server",
default="https://api.cacophony.org.nz",
help="Cacophony API Server URL",
)
parser.add_argument(
"--start-date",
type=parsedate,
help=
"If specified, only files recorded on or after this date will be downloaded.",
)
parser.add_argument(
"--end-date",
type=parsedate,
help=
"If specified, only files recorded before or on this date will be downloaded.",
)
# FIXME - this should take device names
parser.add_argument(
"-d",
"--device",
type=int,
action="append",
help="Limit to this integer device id (can be used multiple times)",
)
parser.add_argument(
"-w",
"--workers",
type=int,
default=4,
help="Number of concurrent downloads to run",
)
parser.add_argument(
"-L",
"--local-time",
default=False,
action="store_true",
help="Convert timestamps in filenames to local time",
)
parser.add_argument(
"-i",
"--id",
type=int,
default=None,
help="Download a specific recording (other options are ignored)",
)
args = parser.parse_args()
args.out_folder = Path(args.out_folder)
args.out_folder.mkdir(exist_ok=True)
print("Querying recordings")
api = API(args.server, args.user, args.password)
if args.id is not None:
recordings = [api.get(args.id)]
else:
recordings = api.query(
type_="audio",
startDate=args.start_date,
endDate=args.end_date,
devices=args.device,
limit=99999,
)
print("Found {} recordings".format(len(recordings)))
pool = Pool(args.workers, download, api, args)
for recording in recordings:
pool.put(recording)
pool.stop()
