def test_parse_emsg(self):
emsg1 = Box.parse(self.l_emsg1)
self.assertEqual(emsg1["type"], b'emsg')
self.assertEqual(emsg1["version"], 0)
self.assertEqual(emsg1["id"], 811)
self.assertEqual(emsg1["event_duration"], 233472)
self.assertEqual(emsg1["scheme_id_uri"], b'urn:scte:scte35:2013:bin')
self.assertEqual(
emsg1["message_data"],
b'\xfc0!\x00\x00\x00\x00\x00\x00\x00\xff\xf0\x10\x05\x00\x00\x03+\x7f\xef\x7f\xfe\x00\x1a\x17\xb0\xc0\x00\x00\x00\x00\x00\xe4a$\x02'
)
emsg2 = Box.parse(self.l_emsg2)
self.assertEqual(emsg2["type"], b'emsg')
self.assertEqual(emsg2["version"], 0)
self.assertEqual(emsg2["id"], 812)
self.assertEqual(emsg2["event_duration"], 233472)
self.assertEqual(emsg2["scheme_id_uri"], b'urn:scte:scte35:2013:bin')
self.assertEqual(
emsg2["message_data"],
b'\xfc0!\x00\x00\x00\x00\x00\x00\x00\xff\xf0\x10\x05\x00\x00\x03,\x7f\xef\x7f\xfe\x00\x1a\x17\xb0\xc0\x00\x00\x00\x00\x00\xfe\xcc\xb92'
)
embe1 = Box.parse(self.l_embe)
self.assertEqual(embe1["type"], b'embe')
def test_mdhd_build(self):
mdhd_data = Box.build(
dict(type=b"mdhd",
creation_time=0,
modification_time=0,
timescale=1000000,
duration=0,
language=u"und"))
self.assertEqual(len(mdhd_data), 32)
self.assertEqual(
mdhd_data,
b'\x00\x00\x00\x20mdhd\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0fB@\x00\x00\x00\x00U\xc4\x00\x00'
)
mdhd_data64 = Box.build(
dict(type=b"mdhd",
version=1,
creation_time=0,
modification_time=0,
timescale=1000000,
duration=0,
language=u"und"))
self.assertEqual(len(mdhd_data64), 44)
self.assertEqual(
mdhd_data64,
b'\x00\x00\x00,mdhd\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0fB@\x00\x00\x00\x00\x00\x00\x00\x00U\xc4\x00\x00'
)
def decrypt(key, inp, out):
"""
decrypt()
@param key: AES-128 CENC key in bytes
@param inp: Open input file
@param out: Open output file
"""
with BufferedReader(inp) as reader:
senc_boxes = deque()
trun_boxes = deque()
while reader.peek(1):
box = Box.parse_stream(reader)
fix_headers(box)
for stsd_box in BoxUtil.find(box, b'stsz'):
sample_size = stsd_box.sample_size
if box.type == b'moof':
senc_boxes.extend(BoxUtil.find(box, b'senc'))
trun_boxes.extend(BoxUtil.find(box, b'trun'))
elif box.type == b'mdat':
senc_box = senc_boxes.popleft()
trun_box = trun_boxes.popleft()
clear_box = b''
with BytesIO(box.data) as box_bytes:
for sample, sample_info in zip(
senc_box.sample_encryption_info,
trun_box.sample_info):
counter = Counter.new(64,
prefix=sample.iv,
initial_value=0)
cipher = AES.new(key, AES.MODE_CTR, counter=counter)
if sample_size:
cipher_bytes = box_bytes.read(sample_size)
clear_box += cipher.decrypt(cipher_bytes)
elif not sample.subsample_encryption_info:
cipher_bytes = box_bytes.read(
sample_info.sample_size)
clear_box += cipher.decrypt(cipher_bytes)
else:
for subsample in sample.subsample_encryption_info:
clear_box += box_bytes.read(
subsample.clear_bytes)
cipher_bytes = box_bytes.read(
subsample.cipher_bytes)
clear_box += cipher.decrypt(cipher_bytes)
box.data = clear_box
out.write(Box.build(box))
return
def ismt_to_ttml(ismt_data):
fd = io.BytesIO(ismt_data)
while True:
x = Box.parse_stream(fd)
if x.type == b'mdat':
return x.data
def get_video_start_time_blackvue(video_file):
with open(video_file, 'rb') as fd:
fd.seek(0, io.SEEK_END)
eof = fd.tell()
fd.seek(0)
while fd.tell() < eof:
box = Box.parse_stream(fd)
if box.type.decode('utf-8') == "moov":
fd.seek(box.offset + 8, 0)
size = struct.unpack('>I', fd.read(4))[0]
typ = fd.read(4)
fd.seek(4, os.SEEK_CUR)
creation_time = struct.unpack('>I', fd.read(4))[0]
modification_time = struct.unpack('>I', fd.read(4))[0]
time_scale = struct.unpack('>I', fd.read(4))[0]
duration = struct.unpack('>I', fd.read(4))[0]
# from documentation
# in seconds since midnight, January 1, 1904
video_start_time_epoch = creation_time * 1000 - duration
epoch_start = datetime.datetime(year=1904, month=1, day=1)
video_start_time = epoch_start + \
datetime.timedelta(milliseconds=video_start_time_epoch)
return video_start_time
def get_video_start_time_blackvue(video_file):
fd = open(video_file, 'rb')
fd.seek(0, io.SEEK_END)
eof = fd.tell()
fd.seek(0)
while fd.tell() < eof:
box = Box.parse_stream(fd)
if box.type.decode('utf-8') == "moov":
fd.seek(box.offset + 8, 0)
size = struct.unpack('>I', fd.read(4))[0]
typ = fd.read(4)
fd.seek(4, os.SEEK_CUR)
creation_time = struct.unpack('>I', fd.read(4))[0]
modification_time = struct.unpack('>I', fd.read(4))[0]
time_scale = struct.unpack('>I', fd.read(4))[0]
duration = struct.unpack('>I', fd.read(4))[0]
# from documentation
# in seconds since midnight, January 1, 1904
video_start_time_epoch = creation_time * 1000 - duration
epoch_start = datetime.datetime(year=1904, month=1, day=1)
video_start_time = epoch_start + \
datetime.timedelta(milliseconds=video_start_time_epoch)
return video_start_time
def test_smhd_parse(self):
in_bytes = b'\x00\x00\x00\x10smhd\x00\x00\x00\x00\x00\x00\x00\x00'
self.assertEqual(
Box.parse(in_bytes + b'padding'),
Container(offset=0)
(type=b"smhd")(version=0)(flags=0)
(balance=0)(reserved=0)(end=len(in_bytes))
)
def test_mdhd_parse(self):
self.assertEqual(
Box.parse(
b'\x00\x00\x00\x20mdhd\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0fB@\x00\x00\x00\x00U\xc4\x00\x00'
),
Container(offset=0)(type=b"mdhd")(version=0)(flags=0)(
creation_time=0)(modification_time=0)(timescale=1000000)(
duration=0)(language="und")(end=32))
def test_tenc_build(self):
self.assertEqual(
Box.build(dict(
type=b"tenc",
key_ID=UUID('337b9643-21b6-4355-9e59-3eccb46c7ef7'),
iv_size=8,
is_encrypted=1)),
b'\x00\x00\x00 tenc\x00\x00\x00\x00\x00\x00\x01\x083{\x96C!\xb6CU\x9eY>\xcc\xb4l~\xf7')
def test_ftyp_build(self):
self.assertEqual(
Box.build(
dict(type=b"ftyp",
major_brand=b"iso5",
minor_version=1,
compatible_brands=[b"iso5", b"avc1"])),
b'\x00\x00\x00\x18ftypiso5\x00\x00\x00\x01iso5avc1')
def test_stsd_parse(self):
tx3g_data = b'\x00\x00\x00\x00\x01\xFF\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x12\xFF\xFF\xFF\xFF\x00\x00\x00\x12ftab\x00\x01\x00\x01\x05Serif'
in_bytes = b'\x00\x00\x00\x50stsd\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x40tx3g\x00\x00\x00\x00\x00\x00\x00\x01' + tx3g_data
self.assertEqual(
Box.parse(in_bytes + b'padding'),
Container(offset=0)(type=b"stsd")(version=0)(flags=0)(entries=[
Container(format=b'tx3g')(data_reference_index=1)(
data=tx3g_data)
])(end=len(in_bytes)))
def test_ftyp_parse(self):
self.assertEqual(
Box.parse(b'\x00\x00\x00\x18ftypiso5\x00\x00\x00\x01iso5avc1'),
Container(offset=0)
(type=b"ftyp")
(major_brand=b"iso5")
(minor_version=1)
(compatible_brands=[b"iso5", b"avc1"])
(end=24)
)
def test_parse_stsd_urim(self):
urim_sample_entry = Box.parse(self.l_stsd_emsg1)
self.assertEqual(len(urim_sample_entry["entries"]), 1)
for entry in urim_sample_entry["entries"]:
self.assertEqual(entry["format"], b"urim")
for l_child in entry["children"]:
if l_child["type"] != b"uriI":
self.assertEqual(l_child["type"][0:3], b'uri')
self.assertEqual(l_child["theURI"],
b"urn:mpeg:dash:event:2012")
def test_tenc_parse(self):
self.assertEqual(
Box.parse(b'\x00\x00\x00 tenc\x00\x00\x00\x00\x00\x00\x01\x083{\x96C!\xb6CU\x9eY>\xcc\xb4l~\xf7'),
Container(offset=0)
(type=b"tenc")
(version=0)
(flags=0)
(is_encrypted=1)
(iv_size=8)
(key_ID=UUID('337b9643-21b6-4355-9e59-3eccb46c7ef7'))
(end=32)
)
def test_build_emib(self):
emib_b = Box.build(
dict(type=b"emib",
version=0,
reserved=1,
presentation_time_delta=-1000,
value=b'',
id=1,
scheme_id_uri=b"my_test_scheme",
duration=2000,
message_data=b"asdfdasgfaghhgsdgh"))
emib_b_p = Box.parse(emib_b)
self.assertEqual(emib_b_p["type"], b'emib')
self.assertEqual(emib_b_p["presentation_time_delta"], -1000)
self.assertEqual(emib_b_p["value"], b'')
self.assertEqual(emib_b_p["id"], 1)
self.assertEqual(emib_b_p["scheme_id_uri"], b"my_test_scheme")
self.assertEqual(emib_b_p["duration"], 2000)
self.assertEqual(emib_b_p["reserved"], 1)
self.assertEqual(emib_b_p["message_data"], b"asdfdasgfaghhgsdgh")
def test_build_emsg(self):
emsg_b = Box.build(
dict(type=b"emsg",
version=1,
presentation_time=1000,
value=b'',
id=1,
scheme_id_uri=b"my_test_scheme",
event_duration=20,
timescale=1,
message_data=b"asdfdasgfaghhgsdgh"))
emsg_b_p = Box.parse(emsg_b)
self.assertEqual(emsg_b_p["type"], b'emsg')
self.assertEqual(emsg_b_p["version"], 1)
self.assertEqual(emsg_b_p["presentation_time"], 1000)
self.assertEqual(emsg_b_p["value"], b'')
self.assertEqual(emsg_b_p["id"], 1)
self.assertEqual(emsg_b_p["scheme_id_uri"], b"my_test_scheme")
self.assertEqual(emsg_b_p["event_duration"], 20)
self.assertEqual(emsg_b_p["timescale"], 1)
self.assertEqual(emsg_b_p["message_data"], b"asdfdasgfaghhgsdgh")
def dump():
parser = argparse.ArgumentParser(description='Dump all the boxes from an MP4 file')
parser.add_argument("input_file", type=argparse.FileType("rb"), metavar="FILE", help="Path to the MP4 file to open")
args = parser.parse_args()
fd = args.input_file
fd.seek(0, io.SEEK_END)
eof = fd.tell()
fd.seek(0)
while fd.tell() < eof:
box = Box.parse_stream(fd)
print(box)
def find_camera_model(videos_folder):
from mapillary_tools.uploader import get_video_file_list
file_list = get_video_file_list(videos_folder)
fd = open(file_list[0], "rb")
fd.seek(0, io.SEEK_END)
eof = fd.tell()
fd.seek(0)
while fd.tell() < eof:
box = Box.parse_stream(fd)
if box.type.decode("utf-8") == "free": # or 'ftyp':
return box.data[29:39]
def test_parse_trun(self):
trun_box1 = Box.parse(self.l_trun_emsg)
self.assertEqual(trun_box1["type"], b'trun')
self.assertEqual(trun_box1["version"], 0)
self.assertEqual(
trun_box1["flags"]["sample_composition_time_offsets_present"],
False)
self.assertEqual(trun_box1["flags"]["sample_flags_present"], False)
self.assertEqual(trun_box1["flags"]["sample_size_present"], True)
self.assertEqual(trun_box1["flags"]["sample_duration_present"], True)
self.assertEqual(trun_box1["flags"]["first_sample_flags_present"],
False)
self.assertEqual(trun_box1["flags"]["data_offset_present"], True)
self.assertEqual(trun_box1["sample_count"], 1)
self.assertEqual(trun_box1["data_offset"], 112)
self.assertEqual(len(trun_box1["sample_info"]), 1)
self.assertEqual(trun_box1["sample_info"][0]["sample_duration"], 25600)
self.assertEqual(trun_box1["sample_info"][0]["sample_size"], 8)
trun_box2 = Box.parse(self.l_trun2_emsg)
self.assertEqual(trun_box2["type"], b'trun')
self.assertEqual(trun_box2["version"], 0)
self.assertEqual(
trun_box2["flags"]["sample_composition_time_offsets_present"],
False)
self.assertEqual(trun_box2["flags"]["sample_flags_present"], False)
self.assertEqual(trun_box2["flags"]["sample_size_present"], True)
self.assertEqual(trun_box2["flags"]["sample_duration_present"], True)
self.assertEqual(trun_box2["flags"]["first_sample_flags_present"],
False)
self.assertEqual(trun_box2["flags"]["data_offset_present"], True)
self.assertEqual(trun_box2["sample_count"], 1)
self.assertEqual(trun_box2["data_offset"], 112)
self.assertEqual(len(trun_box2["sample_info"]), 1)
self.assertEqual(trun_box2["sample_info"][0]["sample_duration"],
233472)
self.assertEqual(trun_box2["sample_info"][0]["sample_size"], 90)
def test_parse_edit_list(self):
elst_b = Box.parse(
b'\x00\x00\x00\x1celst\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x0f\xa0\x00\x00\x04\x00\x00\x01\x00\x00'
)
[Container(edit_duration=4000)(media_time=1024)]
self.assertEqual(elst_b["type"], b"elst")
self.assertEqual(len(elst_b["entries"]), 1)
self.assertEqual(elst_b["entries"][0]["edit_duration"], 4000)
self.assertEqual(elst_b["entries"][0]["media_time"], 1024)
self.assertEqual(elst_b["entries"][0]["media_rate_integer"], 1)
self.assertEqual(elst_b["entries"][0]["media_rate_fraction"], 0)
t = dict(
type=b"elst",
version=1,
flags=0,
entries=[dict(edit_duration=1,media_time=1, media_rate_integer=1, media_rate_fraction=1)\
,dict(edit_duration=2,media_time=2, media_rate_integer=1,
media_rate_fraction=1) ],
)
elst_b = Box.build(t)
t2 = Box.parse(elst_b)
self.assertEqual(len(t["entries"]), len(t2["entries"]))
def find_camera_model(videos_folder) -> bytes:
from mapillary_tools.uploader import get_video_file_list
file_list = get_video_file_list(videos_folder)
if not file_list:
raise RuntimeError(f"No video found in {videos_folder}")
fd = open(file_list[0], "rb")
fd.seek(0, io.SEEK_END)
eof = fd.tell()
fd.seek(0)
while fd.tell() < eof:
box = Box.parse_stream(fd)
if box.type.decode("utf-8") == "free": # or 'ftyp':
return box.data[29:39]
raise RuntimeError(f"camera model not found in {file_list[0]}")
def test_parse_video_samples(self):
infile = open('3.cmfv', 'rb')
moov_box = []
moof_box = []
for i in range(5):
t = Box.parse_stream(infile)
if (t["type"] == b"moov"):
moov_box = t
if (t["type"] == b"moof"):
moof_box = t
res = BoxUtil.find_samples_fragmented(moov_box, moof_box, 1)
self.assertEqual(res[0]["decode_time"], 12288)
self.assertEqual(res[1]["decode_time"], 12800)
self.assertEqual(res[2]["decode_time"], 13312)
self.assertEqual(res[0]["offset_mdat"], 8)
self.assertEqual(res[1]["offset_mdat"], 2223)
self.assertEqual(res[2]["offset_mdat"], 2400)
def test_udta_terminator_parse(self):
ilst = b'\x00\x00\x00\x25\xA9\x74\x6F\x6F\x00\x00\x00\x1D\x64\x61\x74\x61\x00\x00\x00\x01\x00\x00\x00\x00\x4C\x61\x76\x66\x35\x37\x2E\x32\x35\x2E\x31\x30\x30'
input_bytes = (
b'\x00\x00\x00\x66udta'
b'\x00\x00\x00\x5ameta\x00\x00\x00\x00'
b'\x00\x00\x00\x21hdlr\x00\x00\x00\x00\x00\x00\x00\x00mdir\x61\x70\x70\x6C\x00\x00\x00\x00\x00\x00\x00\x00\x00'
b'\x00\x00\x00\x2Dilst' + ilst + b'\x00\x00\x00\x00')
self.assertEqual(
Box.parse(input_bytes),
Container(offset=0)(type=b'udta')(children=[
Container(offset=8)(type=b'meta')(version=0)(flags=0)(
children=[
Container(offset=20)(type=b'hdlr')(version=0)(flags=0)(
handler_type=b'mdir')(name=u'')(end=53),
Container(offset=53)(type=b'ilst')(data=ilst)(
end=len(input_bytes) - 4)
])(end=len(input_bytes) - 4)
])(end=len(input_bytes)))
def test_parse_stbl_progressive(self):
stbl = Box.parse(l_stbl_audio_4s)
self.assertNotEqual(stbl, None)
stsd = BoxUtil.find_and_return_first(stbl, b'stsd')
self.assertEqual(len(stsd["entries"]), 1)
self.assertEqual(stsd["entries"][0]["format"], b"mp4a")
mp4a = stsd["entries"][0]
self.assertEqual(mp4a["channels"], 2)
self.assertEqual(mp4a["sampling_rate"], 48000)
stsz = BoxUtil.find_and_return_first(stbl, b'stsz')
self.assertNotEqual(stsz, None)
self.assertEqual(stsz["sample_count"], 189)
stco = BoxUtil.find_and_return_first(stbl, b'stco')
self.assertNotEqual(stco, None)
self.assertEqual(len(stco["entries"]), 1)
stts = BoxUtil.find_and_return_first(stbl, b'stts')
self.assertNotEqual(stts, None)
self.assertEqual(len(stts["entries"]), 2)
def test_moov_build(self):
moov = \
Container(type=b"moov")(children=[ # 96 bytes
Container(type=b"mvex")(children=[ # 88 bytes
Container(type=b"mehd")(version=0)(flags=0)(fragment_duration=0), # 16 bytes
Container(type=b"trex")(track_ID=1), # 32 bytes
Container(type=b"trex")(track_ID=2), # 32 bytes
])
])
moov_data = Box.build(moov)
self.assertEqual(len(moov_data), 96)
self.assertEqual(
moov_data, b'\x00\x00\x00\x60moov'
b'\x00\x00\x00\x58mvex'
b'\x00\x00\x00\x10mehd\x00\x00\x00\x00\x00\x00\x00\x00'
b'\x00\x00\x00\x20trex\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
b'\x00\x00\x00\x20trex\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
)
def find_camera_model(videos_folder):
from mapillary_tools.uploader import get_video_file_list
file_list = get_video_file_list(videos_folder)
fd = open(file_list[0], 'rb')
fd.seek(0, io.SEEK_END)
eof = fd.tell()
fd.seek(0)
while fd.tell() < eof:
try:
box = Box.parse_stream(fd)
except RangeError:
print('error parsing blackvue GPS information, exiting')
sys.exit(1)
except ConstError:
print('error parsing blackvue GPS information, exiting')
sys.exit(1)
if box.type.decode('utf-8') == 'free':# or 'ftyp':
return box.data[29:39]
def find_camera_model(videos_folder):
from mapillary_tools.uploader import get_video_file_list
file_list = get_video_file_list(videos_folder)
fd = open(file_list[0], 'rb')
fd.seek(0, io.SEEK_END)
eof = fd.tell()
fd.seek(0)
while fd.tell() < eof:
try:
box = Box.parse_stream(fd)
except RangeError:
print('error parsing blackvue GPS information, exiting')
sys.exit(1)
except ConstError:
print('error parsing blackvue GPS information, exiting')
sys.exit(1)
if box.type.decode('utf-8') == 'free': # or 'ftyp':
return box.data[29:39]
def get_points_from_bv(path, use_nmea_stream_timestamp=False):
points = []
fd = open(path, 'rb')
fd.seek(0, io.SEEK_END)
eof = fd.tell()
fd.seek(0)
date = None
first_gps_date = None
first_gps_time = None
found_first_gps_date = False
found_first_gps_time = False
while fd.tell() < eof:
try:
box = Box.parse_stream(fd)
except RangeError:
print('error parsing blackvue GPS information, exiting')
sys.exit(1)
except ConstError:
print('error parsing blackvue GPS information, exiting')
sys.exit(1)
if box.type.decode('utf-8') == 'free':
length = len(box.data)
offset = 0
while offset < length:
newb = Box.parse(box.data[offset:])
if newb.type.decode('utf-8') == 'gps':
lines = newb.data
# Parse GPS trace
for l in lines.splitlines():
m = l.lstrip('[]0123456789')
# this utc millisecond timestamp seems to be the camera's
# todo: unused?
# match = re.search('\[([0-9]+)\]', l)
# if match:
# utcdate = match.group(1)
#By default, use camera timestamp. Only use GPS Timestamp if camera was not set up correctly and date/time is wrong
if use_nmea_stream_timestamp == False:
if "$GPGGA" in m:
match = re.search('\[([0-9]+)\]', l)
if match:
epoch_in_local_time = match.group(1)
camera_date = datetime.datetime.utcfromtimestamp(
int(epoch_in_local_time) / 1000.0)
data = pynmea2.parse(m)
if (data.is_valid):
if found_first_gps_time == False:
first_gps_time = data.timestamp
found_first_gps_time = True
lat, lon, alt = data.latitude, data.longitude, data.altitude
points.append((camera_date, lat, lon, alt))
if use_nmea_stream_timestamp == True or found_first_gps_date == False:
if "GPRMC" in m:
try:
data = pynmea2.parse(m)
if data.is_valid:
date = data.datetime.date()
if found_first_gps_date == False:
first_gps_date = date
except pynmea2.ChecksumError as e:
# There are often Checksum errors in the GPS stream, better not to show errors to user
pass
except Exception as e:
print(
"Warning: Error in parsing gps trace to extract date information, nmea parsing failed"
)
if use_nmea_stream_timestamp == True:
if "$GPGGA" in m:
try:
data = pynmea2.parse(m)
if (data.is_valid):
lat, lon, alt = data.latitude, data.longitude, data.altitude
if not date:
timestamp = data.timestamp
else:
timestamp = datetime.datetime.combine(
date, data.timestamp)
points.append(
(timestamp, lat, lon, alt))
except Exception as e:
print(
"Error in parsing gps trace to extract time and gps information, nmea parsing failed due to {}"
.format(e))
#If there are no points after parsing just return empty vector
if points == []:
return []
#After parsing all points, fix timedate issues
if use_nmea_stream_timestamp == False:
# If we use the camera timestamp, we need to get the timezone offset, since Mapillary backend expects UTC timestamps
first_gps_timestamp = datetime.datetime.combine(
first_gps_date, first_gps_time)
delta_t = points[0][0] - first_gps_timestamp
if delta_t.days > 0:
hours_diff_to_utc = round(delta_t.total_seconds() /
3600)
else:
hours_diff_to_utc = round(
delta_t.total_seconds() / 3600) * -1
utc_points = []
for idx, point in enumerate(points):
delay_compensation = datetime.timedelta(
seconds=-1.8
) #Compensate for solution age when location gets timestamped by camera clock. Value is empirical from various cameras/recordings
new_timestamp = points[idx][0] + datetime.timedelta(
hours=hours_diff_to_utc) + delay_compensation
lat = points[idx][1]
lon = points[idx][2]
alt = points[idx][3]
utc_points.append((new_timestamp, lat, lon, alt))
points = utc_points
points.sort()
else:
#add date to points that don't have it yet, because GPRMC message came later
utc_points = []
for idx, point in enumerate(points):
if type(points[idx][0]) != type(
datetime.datetime.today()):
timestamp = datetime.datetime.combine(
first_gps_date, points[idx][0])
else:
timestamp = points[idx][0]
lat = points[idx][1]
lon = points[idx][2]
alt = points[idx][3]
utc_points.append((timestamp, lat, lon, alt))
points = utc_points
points.sort()
offset += newb.end
break
return points
def get_points_from_bv(path,use_nmea_stream_timestamp=False):
points = []
with open(path, 'rb') as fd:
fd.seek(0, io.SEEK_END)
eof = fd.tell()
fd.seek(0)
date = None
first_gps_date = None
first_gps_time = None
found_first_gps_date = False
found_first_gps_time = False
while fd.tell() < eof:
try:
box = Box.parse_stream(fd)
except RangeError:
print('error parsing blackvue GPS information, exiting')
sys.exit(1)
except ConstError:
print('error parsing blackvue GPS information, exiting')
sys.exit(1)
if box.type.decode('utf-8') == 'free':
length = len(box.data)
offset = 0
while offset < length:
newb = Box.parse(box.data[offset:])
if newb.type.decode('utf-8') == 'gps':
lines = newb.data
# Parse GPS trace
for l in lines.splitlines():
m = l.lstrip('[]0123456789')
# this utc millisecond timestamp seems to be the camera's
# todo: unused?
# match = re.search('\[([0-9]+)\]', l)
# if match:
# utcdate = match.group(1)
#By default, use camera timestamp. Only use GPS Timestamp if camera was not set up correctly and date/time is wrong
if use_nmea_stream_timestamp==False:
if "$GPGGA" in m:
match = re.search('\[([0-9]+)\]', l)
if match:
epoch_in_local_time = match.group(1)
camera_date=datetime.datetime.utcfromtimestamp(int(epoch_in_local_time)/1000.0)
data = pynmea2.parse(m)
if(data.is_valid):
if found_first_gps_time == False:
first_gps_time = data.timestamp
found_first_gps_time = True
lat, lon, alt = data.latitude, data.longitude, data.altitude
points.append((camera_date, lat, lon, alt))
if use_nmea_stream_timestamp==True or found_first_gps_date==False:
if "GPRMC" in m:
try:
data = pynmea2.parse(m)
if data.is_valid:
date = data.datetime.date()
if found_first_gps_date == False:
first_gps_date=date
except pynmea2.ChecksumError as e:
# There are often Checksum errors in the GPS stream, better not to show errors to user
pass
except Exception as e:
print(
"Warning: Error in parsing gps trace to extract date information, nmea parsing failed")
if use_nmea_stream_timestamp==True:
if "$GPGGA" in m:
try:
data = pynmea2.parse(m)
if(data.is_valid):
lat, lon, alt = data.latitude, data.longitude, data.altitude
if not date:
timestamp = data.timestamp
else:
timestamp = datetime.datetime.combine(
date, data.timestamp)
points.append((timestamp, lat, lon, alt))
except Exception as e:
print(
"Error in parsing gps trace to extract time and gps information, nmea parsing failed due to {}".format(e))
#If there are no points after parsing just return empty vector
if points == []:
return []
#After parsing all points, fix timedate issues
if use_nmea_stream_timestamp==False:
# If we use the camera timestamp, we need to get the timezone offset, since Mapillary backend expects UTC timestamps
first_gps_timestamp = datetime.datetime.combine(first_gps_date, first_gps_time)
delta_t = points[0][0]-first_gps_timestamp
if delta_t.days>0:
hours_diff_to_utc = round(delta_t.total_seconds()/3600)
else:
hours_diff_to_utc = round(delta_t.total_seconds()/3600) * -1
utc_points=[]
for idx, point in enumerate(points):
delay_compensation = datetime.timedelta(seconds=-1.8) #Compensate for solution age when location gets timestamped by camera clock. Value is empirical from various cameras/recordings
new_timestamp = points[idx][0]+datetime.timedelta(hours=hours_diff_to_utc)+delay_compensation
lat = points[idx][1]
lon = points[idx][2]
alt = points[idx][3]
utc_points.append((new_timestamp, lat, lon, alt))
points = utc_points
points.sort()
else:
#add date to points that don't have it yet, because GPRMC message came later
utc_points=[]
for idx, point in enumerate(points):
if type(points[idx][0]) != type(datetime.datetime.today()):
timestamp = datetime.datetime.combine(
first_gps_date, points[idx][0])
else:
timestamp = points[idx][0]
lat = points[idx][1]
lon = points[idx][2]
alt = points[idx][3]
utc_points.append((timestamp, lat, lon, alt))
points = utc_points
points.sort()
offset += newb.end
break
return points
def test_parse_stbl_parse_samples_progressive(self):
moov_a = Box.parse(l_movie_box_audio)
res = BoxUtil.find_samples_progressive(moov_a, moov_a)
self.assertEqual(len(res), 189)
def test_build_evte(self):
evte = Box.build(dict(type=b"evte", children=[]))
def dump(file, dump_embedded, dump_raw_blocks, extended_scan, verbose):
""" Extract GPS and Acceleration data from BlackVue MP4 recordings.
BlackVue extracts data embedded in the MP4 recordings of a BlackVue Dashcam.
"""
for filename in file:
filebase = os.path.splitext(filename)[0]
with open(filename, 'rb') as fd:
fd.seek(0, io.SEEK_END)
eof = fd.tell()
fd.seek(0)
while fd.tell() < eof:
try:
box = Box.parse_stream(fd)
# print(box)
content = dict(box.__getstate__())
ctype = content['type'].decode('utf8')
if 'data' in content and ctype == 'free':
offset = content['offset']
end = content['end']
data = content['data']
if verbose:
print(
"Found container of type '{}', data has length {}."
.format(ctype, len(data)))
if dump_embedded:
with open(
filename +
"-{:012d}-{}.bin".format(offset, ctype),
"wb") as ofd:
ofd.write(data)
idx = 0
while idx < len(data):
block_len = int.from_bytes(data[idx:idx + 4],
'big')
block_data = data[idx + 4:idx + block_len]
if verbose:
print("Found block with len {}".format(
block_len))
if block_len == 0:
break
if dump_raw_blocks:
with open(
filebase +
"-{:012d}-{}-{:08d}.bin".format(
offset, ctype, idx), "wb") as ofd:
ofd.write(block_data)
if block_data[0:4] in emb_file_def:
first, has_length, termination, ext = emb_file_def[
block_data[0:4]]
# some data like thmb indicates the length:
last = first + int.from_bytes(
block_data[4:8],
'big') if has_length else block_len
# other data like gps is zero terminated:
last = block_data.find(
termination
) if termination is not None else last
if ext is not None:
with open(filebase + ext, "wb") as ofd:
ofd.write(block_data[first:last])
if ext == '.3gf':
with open(filebase + ext + ".txt",
"wt") as ofd:
n = 0
while first < last:
chunk = block_data[first:first +
10]
time_ms = int.from_bytes(
chunk[0:4], 'big')
if time_ms == 0xffffffff:
break
acc_1u = int.from_bytes(
chunk[4:6], 'big')
acc_2u = int.from_bytes(
chunk[6:8], 'big')
acc_3u = int.from_bytes(
chunk[8:10], 'big')
acc_1s = int.from_bytes(
chunk[4:6], 'big', signed=True)
acc_2s = int.from_bytes(
chunk[6:8], 'big', signed=True)
acc_3s = int.from_bytes(
chunk[8:10],
'big',
signed=True)
ofd.write(
("{:8d} " +
"{:08x} {:04x} {:04x} {:04x} "
+ "{:6d} {:6d} {:6d} {:6d}\n"
).format(
n, time_ms, acc_1u,
acc_2u, acc_3u, time_ms,
acc_1s, acc_2s, acc_3s))
first += 10
n += 1
idx += block_len
if verbose:
("Got to idx {}.".format(idx))
if not extended_scan:
break
except construct.core.ConstError:
break
except Exception as e:
print(e)
raise
def test_smhd_build(self):
smhd_data = Box.build(dict(type=b"smhd", balance=0))
self.assertEqual(len(smhd_data), 16),
self.assertEqual(
smhd_data, b'\x00\x00\x00\x10smhd\x00\x00\x00\x00\x00\x00\x00\x00')
