def read(self, session, media='video', format='BGR'):
if session is None or media not in session:
logging.error(f"{media} not in session to read")
return None
try:
if media == 'video':
meta = session[media]
stream = meta['stream']
codec = meta['codec']
meta['framer'] = framer = meta.get(
'framer', self.read_video(session, format))
meta, frame = next(framer)
elif media == 'audio':
meta = session[media]
stream = meta['stream']
meta['framer'] = framer = meta.get('framer',
self.read_audio(session))
meta, frame = next(framer)
except StopIteration:
# EOS
return None
except Exception as e:
logging.info(f"Failed to read a frame: {e}")
raise e
else:
return media, meta, frame
def transcode_to_h264(pth, cmd=None):
"""
Transcode video to h264(annexb)
params:
pth - path to video to be transcoded
Returns:
path to transcoded video
"""
output_pth = None
try:
basename = os.path.basename(pth).split('.')[0]
output = ''.join(basename)
output_pth = os.path.dirname(basename) + f'{output}.h264'
if not cmd:
cmd = f'''ffmpeg -i {pth} \
-c:v copy \
-bsf h264_mp4toannexb \
-y {output_pth}'''
cmd = shlex.split(cmd)
output = subprocess.call(cmd)
except Exception as e:
logging.error(f'Error transcoding the video at {pth}: {e}')
return output_pth
def get_nvr_credentials(rds, site_id, private_key) -> dict:
"""Get nvr_credentials given site_id
Params:
rds: RDS object to call sql execution
site_id: site id
private_key: private key to decrypt the password
Returns:
credentials(dict)
"""
from .utils import decrypt
sql = """SELECT * FROM nvr_credentials WHERE site_id = :site_id"""
sql_parameters = {'site_id': int(site_id)}
site_id = None
credentials = {}
try:
response = rds.execute_statement(sql, sql_parameters)
records = response.get('records')
if records:
credentials = records[0]
# decypt password with private key
passwd = credentials.get('passwd').encode()
decrypted_passwd = decrypt(private_key=private_key, value=passwd)
credentials['passwd'] = decrypted_passwd
except Exception as e:
logging.error(f"{e}, assume credentials in the environment")
return os.environ
else:
return credentials
def __next__(self):
boundary = self.stream.read(len(self.boundary))
#print('##### boundary: ', boundary)
assert boundary == self.boundary, f'Unexpected boundary: {boundary}'
self.buf += self.stream.read(self.bsize)
if len(self.buf) < self.bsize:
logging.error(f'EOS: only {len(buf)} bytes in the buffer < {self.bsize}')
raise StopIteration
#print(f'peek: {buf[:self.bsize]}')
part = BodyPart(self.buf, self.encoding)
if b'content-length' in part.headers:
length = int(part.headers[b'content-length'].decode(self.encoding)) # excluding newline
# assert length >= self.bsize, f"{length} < bsize({self.bsize})"
size = length - len(part.content) # + len(self.newline)
if size < 0: # over read
# print(f"[PartIterator] Over read {-size} bytes")
self.buf = part.content[size:]
part.content = part.content[:size]
if len(self.buf) < len(self.newline):
self.stream.read(len(self.newline) - len(self.buf))
self.buf.clear()
else:
self.buf = self.buf[len(self.newline):]
else: # under read
part.content += self.stream.read(size)
self.stream.read(len(self.newline))
self.buf.clear()
# print('[PartIterator]', part.headers, len(part.content))
return part
else:
logging.error(f'No Content-Length in the part headers: {part.headers}')
raise StopIteration
def get_secret(secret_id="KinesisVideoStreamProducerCredentials"):
""""
Helper function to get the credentials from AWS secret manager
Params:
secret_id: id to be fetched from secret store
Returns:
secret(json)
"""
secret = None
# init Secrets Manager client
session = boto3.session.Session()
client = session.client(
service_name='secretsmanager',
region_name='us-east-1',
)
try:
get_secret_value_response = client.get_secret_value(SecretId=secret_id)
except ClientError as e:
logging.error(f"{e}, assume secrets in the environment")
return os.environ
else:
# secret could be string or binary
if 'SecretString' in get_secret_value_response:
secret = get_secret_value_response['SecretString']
else:
secret = base64.b64decode(get_secret_value_response['SecretBinary'])
return json.loads(secret)
def get_s3_video(url,
*args,
transcode=False,
bucket='eigen-stream-videos',
pth='assets/',
**kwargs):
"""
Download video from s3 and transcode
params:
url - s3 key name
transcode - transcode video to h264
bucket - s3 key bucket
Returns:
video path or url(str)
"""
# create path if not exist
key = url.split('s3://')[1]
pth = Path(pth)
pth.mkdir(parents=True, exist_ok=True)
download_path = Path(f'{pth}/{key}')
# download from s3
s3.download_key(key=key, pth=download_path, bucket=bucket, reload=True)
if download_path.exists():
if transcode:
# transcode video to h264
download_path = utils.transcode_to_h264(pth=download_path)
if not pth:
logging.error(f'Error transcoding video at: {download_path}')
sys.exit(1)
else:
logging.error(f'Failed downloading key: {key} from bucket: {bucket}')
sys.exit(1)
return str(download_path)
def run(self):
# loop streaming thread until stop_event is set
while not self.stop_event.is_set():
try:
# get task_type and respective task function
task_type = self._args.get('stream_type')
TASKS[task_type](
self._args,
self.stop_event
)
except Exception as e:
# exception occured:
# put to monitor queue for updating the database
logging.error(e)
if self.exception_count == 0:
stream_id = self._args.get('stream_id')
msg = dict(
timestamp=time.time(),
payload=str(e),
msg_type=MSG_TYPE.ERROR,
stream_id=stream_id
)
self.monitor.put_msg(msg)
self.exception_count += 1
# exponential delay before restarting
time.sleep(self.exception_count * 10)
# XXX: make sure the delay does not get too big
if self.exception_count == self.max_exceptions:
self.exception_count = 1
logging.info(f'[{self.name}] Restarting streaming producer on error')
def send_msg(sqs, msg):
try:
global ECS_TASK_ID
global DOCKER_ID
if not ECS_TASK_ID:
try:
out = get_task_id_on_instance()
ECS_TASK_ID = out.get('task_id', 'INVALID')
DOCKER_ID = out.get('docker_id', 'INVALID')
except Exception as e:
logging.error(e)
ECS_TASK_ID = uuid4().hex
DOCKER_ID = uuid4().hex
msg['ecs_task_id'] = ECS_TASK_ID
msg['ecs_docker_id'] = DOCKER_ID
msg['publisher'] = 'STREAMING_TASK'
stream_id = msg['stream_id']
# NOTE: Messages that belong to the same message group are always processed one by one,
# in a strict order relative to the message group
# (however, messages that belong to different message groups might be processed out of order).
sqs.send_message(
message_body=json.dumps(msg),
message_group_id=f'{stream_id}'
)
except Exception as e:
logging.error(f'SQS send message failed: {e}')
raise e
def open(self, start=None, end=None, timestamp='PRODUCER', **kwargs):
"""Start streaming from KVS.
Streaming may be interleaved with media of different formats.
Kwargs:
start(float): KVS start timestamp
start(float): KVS end timestamp
expires(int): streaming expiry
timestamp: [ SERVER | PRODUCER ]
DeepLens allows for either timestamp.
NUUO works only with PRODUCER timestamp.
"""
try:
now = start or time()
url = kvs_session_url(self.stream, start, end, timestamp, **kwargs)
session = openAV(url, adaptive=False, **kwargs)
except Exception as e:
# TODO Possible causes:
# - [tcp @ 0x7fde64038000] Connection to tcp://b-604520a7.kinesisvideo.us-east-1.amazonaws.com:443 failed: Connection timed out
# - [tls @ 0x7f4660073f40] error:00000000:lib(0):func(0):reason(0)
# - botocore.errorfactory.ResourceNotFoundException:
# An error occurred (ResourceNotFoundException) when calling the GetHLSStreamingSessionURL operation:
# No fragments found in the stream for the streaming request.
logging.error(f"Failed to open {self.url}: {e}")
raise e
else:
#logging.info(f"KVS session start: requested={now:.3f}s, actual={session['start']:.3f}s")
return session
def startStreaming(nvr, cfg, duration=None):
cam, profile, codec = cfg
host = f'{nvr.ip}:{nvr.port}'
url = f'http://{host}'
show = sys.x_available()
decoder = av.CodecContext.create(codec.replace('.', '').lower(), "r")
stream = nvr.startStreaming(cfg, debug=not True)
logging.info(
f"##### {cam['area']}: streaming({profile}/{codec}) for {duration} frames #####"
)
frames = 0
for i, (pkt, (media, cc)) in enumerate(stream):
try:
# Unsupported codecs, e.g.:
# - b'audio/x-g711-mulaw'
if media != 'video' or decoder.name not in cc:
print(f"[{i}] Skipped unsupported frame: {media}/{cc}")
continue
packet = av.Packet(pkt['payload'])
frame = decoder.decode(packet)[0].to_rgb().to_ndarray()[:, :, ::-1]
except Exception as e:
logging.error(f"[{frames}] Failed to decode frame: {e}")
break
else:
if show:
cv.imshow('Live: {url}', frame)
cv.waitKey(1)
frames += 1
logging.info(
f"[{i}] {media}/{cc} of {len(pkt)} bytes in {frame.shape}")
if duration is not None and frames == duration:
cv.destroyAllWindows()
break
def run(self):
while not self.stop_event.is_set():
try:
# blocking operation
source, headers, message = self.broker.get_msg()
self.handler(source, headers, message)
self.broker.task_done()
except Exception as e:
logging.error(f'[{self.name}] Driver Failed: {e}')
raise e
def open(self, *args, **kwargs):
try:
if not self.path:
self.path = yt_hls_url(self.url, *args, **kwargs)
session = openAV(self.path, *args, **kwargs)
except Exception as e:
logging.error(f"Failed to open {self.url}: {e}")
raise e
else:
return session
def __init__(self,
resolution=None,
fps=None,
gop=None,
bitrate=None,
ch=1):
if fps:
ret = os.system(f"{MXUVC_BIN} --ch {ch} framerate {fps}")
if ret > 0:
self.fps = None
logging.error(
f"Failed to set FPS to {fps} w/ ret={ret}, typically permission denied"
)
else:
self.fps = fps
if resolution:
res = RESOLUTION[resolution]
ret = os.system(
f"{MXUVC_BIN} --ch {ch} resolution {res[0]} {res[1]}")
if ret > 0:
self.resolution = None
logging.error(
f"Failed to set resolution to {res} w/ ret={ret}, typically permission denied"
)
else:
self.resolution = resolution
gop = gop or fps
if gop:
ret = os.system(f"{MXUVC_BIN} --ch {ch} gop {gop}")
if ret > 0:
self.gop = None
logging.error(
f"Failed to set GOP to {gop} w/ ret={ret}, typically permission denied"
)
else:
self.gop = gop
if bitrate:
ret = os.system(f"{MXUVC_BIN} --ch {ch} bitrate {bitrate}")
if ret > 0:
self.bitrate = None
logging.error(
f"Failed to set bitrate to {bitrate} w/ ret={ret}, typically permission denied"
)
else:
self.bitrate = bitrate
ret = os.system(f"{MXUVC_BIN} --ch {ch} iframe")
if ret > 0:
logging.error(f"Failed to force generating an IFrame")
self.stream = None
self.ch = ch
def open(self, *args, **kwargs):
try:
api_client = SUPPORTED_APIS[self.api]()
self.path = api_client.stream_url(self.channel_id,
self.stream_type)
session = openAV(self.path, **kwargs)
except Exception as e:
logging.error(f"Failed to open session: {e}")
raise e
else:
return session
def upload_s3(path, bucket, key):
'''
Args:
path(str): path to the file to upload
bucket(str): S3 bucket name
key(str): key to upload to the bucket where the ending '/' matters
'''
try:
import botocore, boto3
from botocore.exceptions import ClientError
except ImportError as e:
logging.warning(
f'botocore and boto3 are required to download from S3: {e}')
return False
else:
# XXX Amazon S3 supports buckets and objects, and there is no hierarchy.
path = Path(path)
s3 = boto3.resource('s3').meta.client
if not path.is_file():
logging.error(f"{path} not exist or not a file to upload")
return False
total = 0
start = time()
def callback(bytes):
nonlocal total
total += bytes
elapse = time() - start
if total < 1024:
print(
f"\rUploaded {total:4d} bytes at {total / elapse:.2f} bytes/s",
end='')
elif total < 1024**2:
KB = total / 1024
print(f"\rUploaded {KB:4.2f}KB at {KB/elapse:4.2f} KB/s",
end='')
else:
MB = total / 1024**2
print(f"\rUploaded {MB:8.2f}MB at {MB/elapse:6.2f} MB/s",
end='')
sys.stdout.flush()
try:
print(path, bucket, key)
s3.upload_file(str(path), bucket, key, Callback=callback)
except ClientError as e:
print()
logging.error(
f"Failed to upload {path} to s3://{bucket}/{key}: {e}")
else:
print()
logging.info(f"Succeeded to upload {path} to s3://{bucket}/{key}")
return True
def run(self):
while not self.stop_event.is_set():
try:
msg = self._queue.get(block=True)
if msg is not None:
if self.env == 'TEST':
logging.info(f'[{self.name}] Published {msg}')
else:
send_msg(self.sqs, msg)
self._queue.task_done()
except Exception as e:
logging.error(f'[{self.name}] Monitor Failed: {e}')
raise e
def download_gdrive(id='1mM8aZJlWTxOg7BZJvNUMrTnA2AbeCVzS',
path='/tmp/yolov5x.pt',
force=False):
# https://gist.github.com/tanaikech/f0f2d122e05bf5f971611258c22c110f
# Downloads a file from Google Drive, accepting presented query
# from utils.google_utils import *; gdrive_download()
import time
t = time.time()
if os.path.exists(path):
if force:
os.remove(path)
logging.warning(f"Removed existing download: {path}")
else:
logging.warning(
f"Download exists: {path}, specify force=True to remove if necessary"
)
return 0
logging.info(
f'Downloading https://drive.google.com/uc?export=download&id={id} to {path}...'
)
os.remove('cookie') if os.path.exists('cookie') else None
# Attempt file download
os.system(
f"curl -c ./cookie -s -L \'https://drive.google.com/uc?export=download&id={id}\' > /dev/null"
)
if os.path.exists('cookie'): # large file
# s = "curl -Lb ./cookie \"https://drive.google.com/uc?export=download&confirm=`awk '/download/ {print $NF}' ./cookie`&id=%s\" -o %s" % (id, path)
s = f"curl -Lb ./cookie \"https://drive.google.com/uc?export=download&confirm=`awk '/download/ {{print $NF}}' ./cookie`&id={id}\" -o {path}"
else: # small file
s = f"curl -s -L -o {path} 'https://drive.google.com/uc?export=download&id={id}'"
r = os.system(s) # execute, capture return values
os.remove('cookie') if os.path.exists('cookie') else None
# Error check
if r != 0:
os.remove(path) if os.path.exists(path) else None # remove partial
logging.error(f'Failed to download to {path}'
) # raise Exception('Download error')
return r
'''
# Unzip if archive
if path.endswith('.zip'):
logging.info('Unzipping... ')
os.system('unzip -q %s' % path) # unzip
os.remove(path) # remove zip to free space
'''
logging.info(f'Done in {time.time() - t:.1f}s')
return r
def open(self, *args, **kwargs):
try:
if not self.path:
transcode = kwargs.pop('transcode', False)
bucket = kwargs.pop('bucket', 'eigen-stream-videos')
self.path = get_s3_video(self.url,
transcode=transcode,
bucket=bucket)
session = openAV(self.path, **kwargs)
except Exception as e:
logging.error(f"Failed to open {self.url}: {e}")
return None
else:
return session
def send_records(self, records, attempt=0):
"""Send records to the Kinesis stream.
Falied records are sent again with an exponential backoff decay.
Parameters
----------
records : array
Array of formated records to send.
attempt: int
Number of times the records have been sent without success.
"""
# If we already tried more times than we wanted, save to a file
if attempt > self.max_retries:
logging.warning(
f'[{self._name}] Writing {len(records)} records to file')
with open('failed_records.dlq', 'ab') as f:
for r in records:
f.write(r.get('Data'))
return
# Sleep before retrying
if attempt:
time.sleep(2**attempt * .1)
try:
response = self.kinesis_client.put_records(
StreamName=self.stream_name, Records=records)
except Exception as e:
logging.error(f'[{self._name}]: {e}')
raise e
else:
failed_record_count = response['FailedRecordCount']
# Grab failed records
if failed_record_count:
logging.warning(f'[{self._name}] Retrying failed records')
failed_records = []
for i, record in enumerate(response['Records']):
if record.get('ErrorCode'):
failed_records.append(records[i])
# Recursive call
attempt += 1
self.send_records(failed_records, attempt=attempt)
def round(n, digits=None):
r"""Round to the specified number of digits by input types.
"""
if isinstance(n, float):
if digits:
return _round(n, digits)
else:
return int(Decimal(n).to_integral_value(rounding=ROUND_HALF_UP))
else:
try:
import torch as th
if th.is_tensor(n):
n.apply_(lambda x: round(x, digits))
return n
except ImportError as e:
logging.error(
f"No pytorch to round potential tensor input of type {type(n)}"
)
raise e
def setup_test():
import subprocess
import shlex
import socket
# XXX: ActiveMQ broker TEST server
# stomp protocol @port 61613
try:
cmd = shlex.split('docker run --rm --name activemq -d -p 61616:61616 -p 8161:8161 -p 61613:61613 rmohr/activemq')
subprocess.run(cmd)
except Exception as e:
logging.error(e)
ip = socket.gethostbyname(socket.gethostname())
broker_creds = {
'server': ip,
'user': '******',
'passwd': 'admin',
'port': 61613
}
return broker_creds
def encrypt(private_key, value):
"""
Encrypt value with the private key
Params:
private_key: encoded str or str
value: (encoded str or dict or str)value to be encrypted by the private key
Returns:
encrypted value(string)
"""
cipher_key = private_key.encode() if not isinstance(private_key, bytes) else private_key # encode private key
# init fernet
cipher = Fernet(cipher_key)
if isinstance(value, dict):
value = json.dumps(value)
value = value.encode() if not isinstance(value, bytes) else value # encode value
encrypted_value = None
try:
encrypted_value = cipher.encrypt(value) # encypt with cipher
encrypted_value = encrypted_value.decode()
except Exception as e:
logging.error(e)
return encrypted_value
def test_pool_map(chunksize):
from multiprocessing import Pool
import subprocess
import torch as th
pool = Pool(processes=3)
done = []
failed = []
t = time()
tasks = list(range(8))
args = [1,3,5,0,1,3,0,4]
chunksize = 16
for tid, res in tqdm(pool.imap(process_division, zip(tasks, args), chunksize), total=len(tasks)):
if isinstance(res, Exception):
e = res
failed.append(tid)
logging.error(f"task[{tid}] Failed division due to {e}")
else:
done.append(tid)
logging.info(f"Tasks={list(zip(tasks, args))}")
logging.info(f"Done tasks={done}")
logging.info(f"Failed tasks={failed}")
assert len(done) == 3
assert len(failed) == len(tasks) - len(done)
def posenet(pretrained=False,
arch='litehrnet_30_coco_384x288',
model_dir=None,
force_reload=False,
unload_after=False,
**kwargs):
"""
Kwargs:
pretrained(bool, str): True for official checkpoint or path(str) to load a custom checkpoint
tag(str): git repo tag to explicitly specify a particular commit
url(str): direct url to download checkpoint
s3(dict): S3 source containing bucket and key to download a checkpoint from
threshold(float):
"""
ARCH = dict(litehrnet_18_coco_256x192='1ZewlvpncTvahbqcCFb-95C3NHet30mk5',
litehrnet_18_coco_384x288='1E3S18YbUfBm7YtxYOV7I9FmrntnlFKCp',
litehrnet_30_coco_256x192='1KLjNInzFfmZWSbEQwx-zbyaBiLB7SnEj',
litehrnet_30_coco_384x288='1BcHnLka4FWiXRmPnJgJKmsSuXXqN4dgn',
litehrnet_18_mpii_256x256='1bcnn5Ic2-FiSNqYOqLd1mOfQchAz_oCf',
litehrnet_30_mpii_256x256='1JB9LOwkuz5OUtry0IQqXammFuCrGvlEd')
tag = kwargs.get('tag', GITHUB['tag'])
modules = sys.modules.copy()
entry = 'posenet'
m = None
try:
logging.info(f"Creating '{entry}(arch={arch})'")
m = hub.load(github(tag=tag), entry, arch, force_reload=force_reload)
m.tag = tag
if pretrained:
if isinstance(pretrained, bool):
# official pretrained
state_dict = from_pretrained(f"{entry}.pt",
force_reload=force_reload,
gdrive=dict(id=ARCH[arch]))
else:
# custom checkpoint
path = Path(pretrained)
if not path.exists():
path = f"{hub.get_dir()}/{pretrained}"
state_dict = io.load(path, map_location='cpu')
state_dict = {
k: v
for k, v in state_dict.items()
if m.state_dict()[k].shape == v.shape
}
# load_checkpoint(model, path, map_location='cpu')
m.load_state_dict(state_dict, strict=True)
logging.info(f"kwargs={kwargs}")
if kwargs.get('fp16', False):
from mmpose.core import wrap_fp16_model
wrap_fp16_model(m)
logging.info(f"[posnet] wrapped in fp16")
if kwargs.get('fuse_conv_bn', True):
from mmcv.cnn import fuse_conv_bn
m = fuse_conv_bn(m)
logging.info(f"[posenet] fused conv and bn")
except Exception as e:
logging.error(f"Failed to load '{entry}': {e}")
raise e
finally:
# XXX Remove newly imported modules in case of conflict with next load
if unload_after:
for module in sys.modules.keys() - modules.keys():
del sys.modules[module]
m.to('cpu')
return m
def run(self):
try:
self.monitor()
except Exception as e:
logging.error(e)
raise e
def train_yolo5():
"""Train YOLOv5 over a dataset in YOLO5/COCO format.
Usage:
train_yolo5 ../../datasets/Retail81 --with-coco --names object --device 1 --batch-size 32
References:
python train.py --data coco.yaml --cfg yolov5x.yaml --weights yolov5x.pt --device 1 --batch-size 16
"""
parser = argparse.ArgumentParser(
'Train YOLO5 oover a dataset in YOLO5/COCO format')
parser.add_argument('path', help='Path to a dataset in YOLO/COCO format')
parser.add_argument('--arch',
choices=['yolov5l', 'yolov5x'],
default='yolov5x',
help='YOLOv5 model architecture to train')
parser.add_argument('--tag',
choices=['v1.0', 'v2.0', 'v3.0'],
default='v3.0',
help='YOLOv5 repo version tag')
parser.add_argument('--batch-size', type=int, default=16)
parser.add_argument('--device',
default='',
help='cuda device, i.e. 0 or 0,1,2,3 or cpu')
parser.add_argument('--epochs', type=int, default=100)
parser.add_argument('--names', nargs='*', default=[], help='class names')
parser.add_argument('--with-coco',
action='store_true',
help='Including COCO80 classes')
args = parser.parse_args()
logging.info(
f"dataset={args.path}, arch={args.arch}, names={args.names}, with_coco={args.with_coco}"
)
path = Path(args.path).resolve()
data = path / 'data'
data.mkdir(exist_ok=True)
data /= f"{path.name.lower()}.yaml"
models = path / "models"
models.mkdir(exist_ok=True)
cfg = f"{args.arch}.yaml"
weights = f"{args.arch}.pt"
names = args.names
if args.with_coco:
from ml.vision.datasets.coco import COCO80_CLASSES
names = COCO80_CLASSES + names
nc = len(names)
assert nc > 0
with open(data, 'w') as f:
yaml = f"""
train: { path / 'train.txt' }
val: { path / 'val.txt' }
nc: {nc}
names: {names}
"""
print(yaml, file=f)
logging.info(yaml)
from ml.vision.models.detection.yolo5 import github
from ml.utils import Config
from ml import hub
repo = hub.repo(github(tag=tag), force_reload=False)
config = Config().load(f"{repo}/models/{cfg}")
config.nc = nc
config.save(models / cfg)
os.chdir(repo)
cmd = f"python train.py --data {data.resolve()} --cfg {models / cfg} --weights {weights} --device {args.device} --batch-size {args.batch_size} --epochs {args.epochs}"
logging.info(f"wd={os.getcwd()}")
logging.info(cmd)
r = os.system(cmd)
if r:
logging.error(f"Failed train YOLOv5 over {path} with res={r}")
else:
# v1.0
# logging.info(f"Best trained checkpoiont at {repo}/weights/best.pt")
# last v1.0 and v2.0+
logging.info(
f"Best trained checkpoiont at {repo}/runs/expX/weights/best.pt")
def read_video(self, session, format='BGR'):
meta = session['video']
stream = meta['stream']
codec = meta['codec']
workaround = meta['workaround']
while True:
try:
pkt = next(stream)
except StopIteration:
pkt = None
now = time.time()
prev = meta.get('prev', None)
if prev is None:
if not pkt.is_keyframe:
# Some RTSP source may not send key frame to begin with e.g. wisecam
logging.warning(
f"No key frame to begin with, skip through")
session['start'] = now
continue
meta['keyframe'] = pkt.is_keyframe
meta['time'] = session['start']
streams = session['streams']
sformat = session['format']
# XXX Stream container package format determines H.264 NALUs in AVCC or Annex B.
# TODO Streaming NALUs in AVCC
if 'hls' in sformat or 'rtsp' in sformat or '264' in sformat:
# XXX In case of out of band CPD: SPS/PPS in AnnexB.
CPD = []
if codec.extradata is not None:
for (pos, _, _,
type), nalu in NALUParser(codec.extradata,
workaround=workaround):
if hasStartCode(nalu):
CPD.append(nalu)
logging.info(
f"CPD {NALU_t(type).name} at {pos}: {nalu[:8]} ending with {nalu[-1:]}"
)
else:
logging.warning(
f"Invalid CPD NALU({type}) at {pos}: {nalu[:8]} ending with {nalu[-1:]}"
)
if not CPD:
# Skip all
break
NALUs = []
if workaround:
# FIXME workaround before KVS MKVGenerator deals with NALUs ending with a zero byte
# https://github.com/awslabs/amazon-kinesis-video-streams-producer-sdk-cpp/issues/491
for (pos, _, _,
type), nalu in NALUParser(memoryview(pkt),
workaround=workaround):
assert hasStartCode(
nalu
), f"frame[{meta['count']+1}] NALU(type={type}) at {pos} without START CODE: {nalu[:8].tobytes()}"
if type in (NALU_t.SPS, NALU_t.PPS):
if CPD:
# NOTE: some streams could have multiple UNSPECIFIED(0) NALUs within a single packet with SPS/PPS
#assert len(CPD) == 2, f"len(CPD) == {len(CPD)}, not 2 for SPS/PPS"
ordinal = type - NALU_t.SPS
if nalu == CPD[ordinal]:
logging.info(
f"frame[{meta['count']+1}] same {NALU_t(type).name}({nalu[:8].tobytes()}) at {pos} as in CPD({CPD[ordinal][:8]})"
)
else:
# FIXME may expect the CPD to be inserted in the beginning?
logging.warning(
f"frame[{meta['count']+1}] inconsistent {NALU_t(type).name}({nalu[:8].tobytes()}) at {pos} with CPD({CPD[ordinal][:8]})"
)
print(f"CPD {NALU_t(type).name}:",
CPD[ordinal])
print(f"NALU {NALU_t(type).name}:",
nalu.tobytes())
# XXX bitstream may present invalid CPD => replacement with bitstream SPS/PPS
CPD[ordinal] = nalu
else:
NALUs.append(nalu)
logging.info(
f"frame[{meta['count']+1}] {NALU_t(type).name} at {pos}: {nalu[:8].tobytes()} ending with {nalu[-1:].tobytes()}"
)
# XXX KVS master is ready to filter out non-VCL NALUs as part of the CPD
# elif type in (NALU_t.IDR, NALU_t.NIDR):
elif type in (NALU_t.AUD, NALU_t.SEI, NALU_t.IDR,
NALU_t.NIDR):
NALUs.append(nalu)
logging.info(
f"frame[{meta['count']+1}] {NALU_t(type).name} at {pos}: {nalu[:8].tobytes()}"
)
else:
# FIXME may expect CPD to be inserted in the beginning?
logging.warning(
f"frame[{meta['count']+1}] skipped unexpected NALU(type={type}) at {pos}: {nalu[:8].tobytes()}"
)
logging.info(
f"{pkt.is_keyframe and 'key ' or ''}frame[{meta['count']}] combining CPD({len(CPD)}) and NALUs({len(NALUs)})"
)
else:
NALUs.append(memoryview(pkt))
logging.info(
f"{pkt.is_keyframe and 'key ' or ''}frame[{meta['count']}] prepending CPD({len(CPD)})"
)
packet = av.Packet(bytearray(b''.join(CPD + NALUs)))
packet.dts = pkt.dts
packet.pts = pkt.pts
packet.time_base = pkt.time_base
pkt = packet
if pkt.pts is None:
logging.warning(
f"Initial packet dts/pts={pkt.dts}/{pkt.pts}, time_base={pkt.time_base}"
)
elif pkt.pts > 0:
logging.warning(
f"Reset dts/pts of 1st frame from {pkt.pts} to 0")
pkt.pts = pkt.dts = 0
elif 'dash' in sformat:
# TODO In case of out of band CPD: SPS/PPS in AVCC.
logging.info(f"DASH AVCC extradata: {codec.extradata}")
logging.info(
f"pkt[:16]({pkt.is_keyframe}) {memoryview(pkt)[:16].tobytes()}"
)
else:
keyframe = pkt.is_keyframe
logging.debug(
f"packet[{meta['count']}] {keyframe and 'key ' or ''}dts/pts={pkt.dts}/{pkt.pts}, time_base={pkt.time_base}, duration={pkt.duration}"
)
if 'hls' in sformat or 'rtsp' in sformat or '264' in sformat:
NALUs = []
if workaround:
for (pos, _, _,
type), nalu in NALUParser(memoryview(pkt),
workaround=workaround):
# assert hasStartCode(nalu), f"frame[{meta['count']+1}] NALU(type={type}) at {pos} without START CODE: {nalu[:8].tobytes()}"
# FIXME KVS master is not ready to take AUD/SEI as part of the CPD
# if type in (NALU_t.SPS, NALU_t.PPS, NALU_t.IDR, NALU_t.NIDR):
if type in (NALU_t.AUD, NALU_t.SEI, NALU_t.SPS,
NALU_t.PPS, NALU_t.IDR, NALU_t.NIDR):
NALUs.append(nalu)
logging.debug(
f"frame[{meta['count']+1}] {NALU_t(type).name} at {pos}: {nalu[:8].tobytes()}"
)
else:
# FIXME may expect CPD to be inserted?
logging.debug(
f"frame[{meta['count']+1}] skipped NALU(type={type}) at {pos}: {nalu[:8].tobytes()} ending with {nalu[-1:].tobytes()}"
)
else:
NALUs.append(memoryview(pkt))
# XXX Assme no SPS/PPS change
packet = av.Packet(bytearray(b''.join(NALUs)))
packet.dts = pkt.dts
packet.pts = pkt.pts
packet.time_base = pkt.time_base
pkt = packet
frame = prev
if session['decoding']:
try:
frames = codec.decode(prev)
if not frames:
logging.warning(
f"Decoded nothing, continue to read...")
meta['prev'] = pkt
meta['count'] += 1
continue
except Exception as e:
logging.error(
f"Failed to decode video packet of size {prev.size}: {e}"
)
raise e
else:
# print(prev, frames)
frame = frames[0]
meta['width'] = frame.width
meta['height'] = frame.height
if format == 'BGR':
frame = frame.to_rgb().to_ndarray()[:, :, ::-1]
elif format == 'RGB':
frame = frame.to_rgb().to_ndarray()
if session['rt']:
'''
Live source from network or local camera encoder.
Bitstream contains no pts but frame duration.
Adaptive frame duration on drift from wall clock:
- Faster for long frame buffering
- Fall behind for being slower than claimed FPS: resync as now
'''
if pkt.pts is not None and not meta['drifting']:
# Check if drifting
if prev.pts is None:
prev.dts = prev.pts = 0
logging.warning(
"Reset previous packet dts/pts from None to 0")
duration = float((pkt.pts - prev.pts) * pkt.time_base)
# assert duration > 0, f"pkt.pts={pkt.pts}, prev.pts={prev.pts}, pkt.time_base={pkt.time_base}, pkt.duration={pkt.duration}, prev.duration={prev.duration}, duration={duration}"
if duration <= 0:
# FIXME RTSP from Dahua/QB and WiseNet/Ernie
pts = prev.pts + (meta['duration'] /
pkt.time_base) / 2
duration = float((pts - prev.pts) * pkt.time_base)
logging.warning(
f"Non-increasing pts: pkt.pts={pkt.pts}, prev.pts={prev.pts} => pts={pts}, duration={duration}"
)
pkt.pts = pts
timestamp = meta['time'] + duration
if meta['adaptive']:
# adaptive frame duration only if not KVS
diff = abs(timestamp - now)
threshold = meta['thresholds']['drifting']
if diff > threshold:
meta['drifting'] = True
logging.warning(
f"Drifting video timestamps: abs({timestamp:.3f} - {now:.3f}) = {diff:.3f} > {threshold}s"
)
if pkt.pts is None or meta['drifting']:
# Real-time against wall clock
duration = now - meta['time']
duration = min(1.5 / meta['fps'], duration)
duration = max(0.5 / meta['fps'], duration)
meta['duration'] = duration
yield meta, frame
meta['time'] += duration
else:
meta['duration'] = duration
yield meta, frame
meta['time'] = timestamp
else:
# TODO: no sleep for being handled by renderer playback
# Simulating RT
meta['duration'] = 1.0 / meta['fps']
slack = (meta['time'] + meta['duration']) - now
if slack > 0:
logging.debug(
f"Sleeping for {slack:.3f}s to simulate RT source")
time.sleep(slack)
yield meta, frame
meta['time'] += meta['duration']
meta['keyframe'] = keyframe
if pkt.size == 0:
logging.warning(f"EOF/EOS on empty packet")
return None
else:
meta['prev'] = pkt
meta['count'] += 1
except ImportError as e:
# XXX In case of torch <=1.4
from torch.multiprocessing import *
from multiprocessing import connection
import platform, sys, signal
import ctypes, ctypes.util
from ml import logging
if platform.system() == "Windows":
path_libc = ctypes.util.find_library("msvcrt")
else:
path_libc = ctypes.util.find_library("c")
try:
libc = ctypes.CDLL(path_libc)
except OSError as e:
logging.error("Unable to load the system C library: {e}")
sys.exit(1)
_supports_context = sys.version_info >= (3, 4)
def _python_version_check():
if not _supports_context:
raise RuntimeError("Requires python 3.4 or higher to use "
"ml.multiprocessing.spawn and "
"ml.multiprocessing.ProcessContext helper "
"to launch multiple processes.")
def _wrap(fn, i, args, error_queue):
# prctl(2) is a Linux specific system call.
# On other systems the following function call has no effect.
# This is set to ensure that non-daemonic child processes can
def batched_nms(predictions,
conf_thres=0.3,
iou_thres=0.6,
agnostic=False,
merge=True,
multi_label=False,
classes=None):
"""Perform NMS on inference results
Args:
prediction(B, AG, 4+1+80): center, width and height refinement, plus anchor and class scores
per anchor and grid combination
conf_thres(float): anchor confidence threshold
iou_thres(float): NMS IoU threshold
agnostic(bool): class agnostic NMS or per class NMS
merge(bool): weighted merge by IoU for best mAP
multi_label(bool): whether to select mulitple class labels above the threshold or just the max
classes(list | tuple): class ids of interest to retain
Returns:
output(List[Tensor[B, N, 6]]): list of detections per image in (x1, y1, x2, y2, conf, cls)
"""
min_wh, max_wh = 2, 4096 # minimum and maximum box width and height
B, _, nc = predictions.shape
nc -= 5
multi_label &= nc > 1 # multiple labels per box if nc > 1 too
output = [None] * B
for b, x in enumerate(predictions): # image index and inference
x = x[x[:, 4] > conf_thres] # Threshold anchors by confidence
x = x[((x[:, 2:4] > min_wh) &
(x[:, 2:4] < max_wh)).all(1)] # width/height constraints
if x.numel() == 0:
continue
# Compute resulting class scores = anchor * class
x[..., 5:] *= x[..., 4:5]
# xcycwh to xyxy
boxes = xcycwh2xyxy(x[:, :4].round())
# Single or multi-label boxes
if multi_label:
# Combinations of repeated boxes with different classes: [x1, y1, x2, y2, conf, class]
keep, cls = (x[:, 5:] > conf_thres).nonzero().t()
x = torch.cat((boxes[keep], x[keep, 5 + cls].unsqueeze(1),
cls.float().unsqueeze(1)), 1)
else:
# Best class only: [x1, y1, x2, y2, conf, class]
conf, cls = x[:, 5:].max(1)
x = torch.cat((boxes, conf.unsqueeze(1), cls.float().unsqueeze(1)),
1)[conf > conf_thres]
# Filter out boxes not in any specified classses
if classes:
x = x[(cls.view(-1, 1) == torch.tensor(classes,
device=cls.device)).any(1)]
if x.numel() == 0:
continue
# Batched NMS
scores = x[:, 4]
boxes = x[:, :4]
cls = x[:, 5] * 0 if agnostic else x[:, 5] # classes
boxes = boxes + cls.view(
-1, 1) * boxes.max() # boxes (offset by class), scores
keep = nms(boxes, scores, iou_thres)
if merge and (1 < x.shape[0] < 3e3):
# Weighted NMS box merge by IoU * scoress
try:
iou = box_iou(boxes[keep], boxes) > iou_thres # Filtered IoU
weights = iou * scores[None] # weighted IoU by class scores
x[keep, :4] = torch.mm(
weights, x[:, :4]).float() / weights.sum(1, keepdim=True)
except Exception as e: # possible CUDA error https://github.com/ultralytics/yolov3/issues/1139
logging.error(
f"Failed to merge NMS boxes by weighted IoU: {e}")
print(x, x.shape, keep, keep.shape)
output[b] = x[keep].to(predictions.dtype)
return output
def download_test(self,
area,
profile='Original',
fps=DEFAULT_FPS_VALUE,
duration=None):
'''Start streaming from the source area camera.
'''
from time import time
pFrame = ffi.new('PFrame')
pFrame.version = ffi.integer_const('FRAME_CURRENT_VERSION')
pFrame.trackId = ffi.integer_const('DEFAULT_VIDEO_TRACK_ID')
pFrame.duration = int(HUNDREDS_OF_NANOS_SEC /
(fps or DEFAULT_FPS_VALUE))
frameIndex = 0
start_time = time()
retStatus = ffi.integer_const('STATUS_SUCCESS')
streamStopTime = None
if duration is not None:
streamingDuration = duration * HUNDREDS_OF_NANOS_SEC
streamStopTime = lib.defaultGetTime() + streamingDuration
print(f"Streaming stops in {streamStopTime / 10000000:.3f}s")
else:
print(f"Streaming indefinitely")
stream_time = lib.defaultGetTime()
# Python signal delivery depends on the main thread not being blocked on system calls
import signal
signaled = False
def handler(sig, frame):
nonlocal signaled
print(f"Interrupted or stopped by {signal.Signals(sig).name}")
signaled = True
signal.signal(signal.SIGINT, handler)
signal.signal(signal.SIGTERM, handler)
sessions = self.nuuo.open(area,
profile,
decoding=False,
exact=True,
with_audio=False)
assert len(sessions) == 1, f"Only one streaming session at a time"
session = sessions.pop()
while (streamStopTime is None
or lib.defaultGetTime() < streamStopTime) and not signaled:
# TODO audio streaming
res = self.nuuo.read(session, media='video')
if res is None:
logging.error(f"Failed to read frame from NUUO NVR")
break
# TODO non-H.264 keyframe requires additional parsing
m, media, frame = res
keyframe = media.get('keyframe', False)
pFrame.flags = ffi.integer_const(
'FRAME_FLAG_KEY_FRAME') if keyframe else ffi.integer_const(
'FRAME_FLAG_NONE')
pFrame.frameData = ffi.cast('void*', frame.buffer_ptr)
pFrame.size = frame.size
pFrame.index = frameIndex
logging.info(f'FPS {area}: {frameIndex/(time() - start_time)}')
frameIndex += 1
self.nuuo.close(session)
