def test_get_camera_info_bad():
"""
Test get_camera_info with bad camera ID.
:return:
"""
ci = get_camera_info(999999)
assert_equal(type(ci), dict)
def test_get_camera_info():
"""
Test get_camera_info basic case.
:return:
"""
ci = get_camera_info(21804)
assert_equal(type(ci), dict)
assert_equal(ci["id"], 21804)
assert_equal(type(ci["latitude"]), float)
assert_equal(type(ci["date_added"]), datetime.datetime)
def build_camera_database(num_cameras=None):
"""
Build camera metadata database, optionally using a random subset of all.
:param num_cameras: optional, number of cameras to randomly sample
:return: list, list of dictionaries containing camera info
"""
# Get camera list
camera_list = [c[0] for c in get_camera_list()]
# Subset if requested
if num_cameras is not None:
camera_list = numpy.random.choice(camera_list, num_cameras)
# Create dataset by iterating through IDs and querying
camera_data = []
for camera_id in camera_list:
try:
camera_data.append(get_camera_info(camera_id))
except ValueError as e:
print("Unable to retrieve camera {0}: {1}".format(camera_id, e))
return camera_data
def build_camera_image_database(camera_id, start_date=None, end_date=None, output_path="cameras", skip_existing=True):
"""
Build an image database on S3 for a single camera given start and end dates.
:param camera_id: int, camera ID to retrieve
:param start_date:, datetime.datetime, start date to pull year/month
:param end_date:, datetime.datetime, end date to pull year/month
:param output_path: str, path to store images/json like s3://{S3_BUCKET}/path/
:param skip_existing: bool, whether to skip existing camera IDs or update
:return:
"""
# Create S3 client to re-use
s3_client = boto3.client('s3', aws_access_key_id=S3_ACCESS_KEY, aws_secret_access_key=S3_SECRET_KEY)
# Get info
camera_info = get_camera_info(camera_id)
camera_output_path = posixpath.join(output_path, str(camera_id), )
if s3_path_exists(camera_output_path, s3_client):
# Skip camera ID if already exists
if skip_existing:
return True
# Save info as JSON
json_camera_info = copy.copy(camera_info)
json_camera_info["date_added"] = json_camera_info["date_added"].isoformat() \
if json_camera_info["date_added"] is not None else None
json_camera_info["last_capture"] = json_camera_info["last_capture"].isoformat() \
if json_camera_info["last_capture"] is not None else None
json_camera_buffer = json.dumps(json_camera_info)
response = s3_client.put_object(Bucket=S3_BUCKET, Key=posixpath.join(camera_output_path, "info.json"),
Body=json_camera_buffer)
if response["ResponseMetadata"]["HTTPStatusCode"] != 200:
raise RuntimeError("Unable to upload camera info to S3: response={0}".format(response["ResponseMetadata"]))
# Get start date
if start_date is not None and camera_info['date_added'] is not None:
start_date = max(start_date, camera_info['date_added'])
elif start_date is None and camera_info['date_added'] is not None:
start_date = camera_info['date_added']
else:
return False
# Get end date
if end_date is not None and camera_info['last_capture'] is not None:
end_date = min(end_date, camera_info['last_capture'])
elif end_date is None and camera_info['last_capture'] is not None:
end_date = camera_info['last_capture']
else:
return False
for year in range(start_date.year, end_date.year + 1):
for month in range(1, 12):
# skip early month in first year
if year == start_date.year and month < start_date.month:
continue
# skip late month in last year
if year == end_date.year and month > end_date.month:
continue
# retrieve archive and extract all
try:
camera_zip = get_camera_zip(camera_id, year, month)
if camera_zip is not None:
for zip_member_name in camera_zip.namelist():
image_buffer = camera_zip.read(zip_member_name)
response = s3_client.put_object(Bucket=S3_BUCKET,
Key=posixpath.join(camera_output_path, zip_member_name),
Body=image_buffer)
if response["ResponseMetadata"]["HTTPStatusCode"] != 200:
raise RuntimeError(
"Unable to upload camera info to S3: response={0}".format(response["ResponseMetadata"]))
else:
print("ZIP contents for cameraID={0}, year={1}, month={2} is malformed: {3}"
.format(camera_id, year, month, get_zip_url(camera_id, year, month)))
continue
except BadZipFile as e:
print("ZIP contents for cameraID={0}, year={1}, month={2} is malformed: {3}"
.format(camera_id, year, month, get_zip_url(camera_id, year, month)))
continue
return True
def build_camera_image_database(camera_id,
start_date=None,
end_date=None,
output_path="data",
skip_existing=True):
"""
Build an image database for a single camera given start and end dates.
:param camera_id: int, camera ID to retrieve
:param start_date:, datetime.datetime, start date to pull year/month
:param end_date:, datetime.datetime, end date to pull year/month
:param output_path: str, path to store images/json
:param skip_existing: bool, whether to skip existing camera IDs or update
:return:
"""
# Get info
camera_info = get_camera_info(camera_id)
camera_output_path = os.path.join(output_path, str(camera_id))
if not os.path.exists(camera_output_path):
os.makedirs(camera_output_path)
else:
# Skip camera ID if already exists
if skip_existing:
return True
# Save info as JSON
with open(os.path.join(camera_output_path, "info.json"),
"w") as json_output_file:
json_camera_info = copy.copy(camera_info)
json_camera_info["date_added"] = json_camera_info["date_added"].isoformat() \
if json_camera_info["date_added"] is not None else None
json_camera_info["last_capture"] = json_camera_info["last_capture"].isoformat() \
if json_camera_info["last_capture"] is not None else None
json.dump(json_camera_info, json_output_file)
# Get start date
if start_date is not None and camera_info['date_added'] is not None:
start_date = max(start_date, camera_info['date_added'])
elif start_date is None and camera_info['date_added'] is not None:
start_date = camera_info['date_added']
else:
return False
# Get end date
if end_date is not None and camera_info['last_capture'] is not None:
end_date = min(end_date, camera_info['last_capture'])
elif end_date is None and camera_info['last_capture'] is not None:
end_date = camera_info['last_capture']
else:
return False
for year in range(start_date.year, end_date.year + 1):
for month in range(1, 12):
# skip early month in first year
if year == start_date.year and month < start_date.month:
continue
# skip late month in last year
if year == end_date.year and month > end_date.month:
continue
# retrieve archive and extract all
try:
camera_zip = get_camera_zip(camera_id, year, month)
if camera_zip is not None:
camera_zip.extractall(path=camera_output_path)
else:
print(
"ZIP contents for cameraID={0}, year={1}, month={2} is malformed: {3}"
.format(camera_id, year, month,
get_zip_url(camera_id, year, month)))
continue
except BadZipFile as e:
print(
"ZIP contents for cameraID={0}, year={1}, month={2} is malformed: {3}"
.format(camera_id, year, month,
get_zip_url(camera_id, year, month)))
continue
return True