def CreateLog(self, name: str):
file_error_handler = logging.FileHandler(filename=name +
'-test-error.log',
encoding='utf-8')
file_error_handler.setLevel(logging.ERROR)
file_handler = logging.FileHandler(filename=name + '-test.log',
mode='w',
encoding='utf-8')
stdout_handler = logging.StreamHandler(sys.stdout)
handlers = [file_handler, stdout_handler, file_error_handler]
logging.basicConfig(
format=
'%(asctime)s|%(levelname)-.3s|%(name)s: %(message)s', # \t####=> %(filename)s:%(lineno)d
level=logging.DEBUG,
datefmt='%H:%M:%S',
handlers=handlers)
log = logging.getLogger("TEST")
self.helper = CommonHelper()
self.helper.installColoredLog(log)
log.info(f'start {__name__}: ⏱️ {datetime.datetime.now()}')
return log
def __init__(self, *args, **kwargs):
super(TestPipeline, self).__init__(*args, **kwargs)
self.testHelper = TestHelper()
self.log = self.testHelper.CreateLog(TestPipeline.__name__)
TestPipeline.log = self.log
self.archiver = CameraArchiveHelper(self.log)
self.helper = CommonHelper()
def __init__(self, camera: str, datetime: datetime, isSimulation=False):
super().__init__("ELSE")
self.camera = camera
self.datetime = datetime
self.helper = CommonHelper()
self.isSimulation = isSimulation
(self.elasticsearch_host, self.elasticsearch_port) = (None, None)
if AppSettings.ELASTICSEARCH_HOST:
(self.elasticsearch_host, self.elasticsearch_port
) = AppSettings.ELASTICSEARCH_HOST.split(':')
class DirectoryShotsProvider(PipelineShotProvider):
def __init__(self, folder: str = None):
super().__init__("DIRC")
self.helper = CommonHelper()
self.folder = folder
self.SourceImagePattern = re.compile(AppSettings.SOURCE_IMAGE_PATTARN)
def FromDir(self, folder: str):
self = DirectoryShotsProvider()
shots = asq.query(os.listdir(folder)) \
.where(lambda f: self.IsSourceImage(f)) \
.select(lambda f: CamShot(os.path.join(folder, f))) \
.to_list()
self.log.debug("Loaded {} shots from directory {}".format(
len(shots), folder))
for s in shots:
s.LoadImage()
return [PipelineShot(s, i) for i, s in enumerate(shots)]
def IsSourceImage(self, filename: str):
return self.SourceImagePattern.search(filename)
def GetShotsProtected(self, pShots: []):
dt: datetime = None
dtStr = None
folder = self.folder if self.folder else self.config.pathFrom()
if len(pShots) > 0 and 'PROV:IMAP' in pShots[
0].Metadata and 'start' in pShots[0].Metadata['PROV:IMAP']:
meta = pShots[0].Metadata
dtStr = meta['PROV:IMAP']['start']
dt = self.helper.FromTimeStampStr(dtStr)
self.log.debug(f'Search files: ⏱️ {dt} in 📁 {folder}')
else:
self.log.warning(
"No event start time found in metadata['PROV:IMAP']")
# path_from: F:\inetpub\ftproot\Camera\Foscam\FI9805W_C4D6553DECE1
# to found: \snap\MDAlarm_20190926-122821.jpg
filenames = list(
self.helper.WalkFiles(
folder, lambda x: self.helper.FileNameByDateRange(
x, dt, self.config.camera_triggered_interval_sec) and self.
helper.IsImage(x), self.config.ignore_dir)) # self.log
pShots = [
PipelineShot(CamShot(filenames[i]), i)
for i in range(len(filenames))
]
for pShot in pShots:
meta = self.CreateMetadata(pShot)
meta['start'] = dtStr if dtStr else self.helper.ToTimeStampStr(
pShots[0].Shot.GetDatetime())
meta['filename'] = pShot.OriginalShot.fullname
yield pShot
def move_files(self, files, config):
common = CommonHelper()
elastic = ElasticSearchHelper()
exts = {}
last_file_dir_relative_to = ''
bytes_moved = 0
files_moved = 0
for file in files:
dir_relative_to = file.to.get_dir_relative(config.path_to)
dir_relative_frm = file.frm.get_dir_relative(config.pathFrom())
if last_file_dir_relative_to != dir_relative_to:
if last_file_dir_relative_to != '':
ext_stat = ' '.join(
["{}:{}".format(k.upper(), exts[k]) for k in exts])
self.log.info('\t{} files moved: {}. Total {}'.format(
files_moved, ext_stat, common.size_human(bytes_moved)))
files_moved = 0
bytes_moved = 0
exts = {}
self.log.info('{} => {}'.format(dir_relative_frm,
dir_relative_to))
files_moved += 1
bytes_moved += file.frm.size()
#print('\t - {} ({})'.format(file.frm.filename, file.frm.size_human()))
# print(os.path.dirname(file.path_to))
#pprint(file.__dict__)
os.makedirs(file.to.dir, exist_ok=True)
## os.rename("path/to/current/file.foo", "path/to/new/destination/for/file.foo")
##shutil.copy2(file.frm.path, file.to.path)
if self.isSimulation:
self.log.info(f'{file.frm.path} => {file.to.path}')
else:
shutil.move(file.frm.path, file.to.path)
elastic.report_to_elastic(file)
ext = file.to.get_extension()
if ext not in exts:
exts[ext] = 0
exts[ext] += 1
last_file_dir_relative_to = dir_relative_to
#break
ext_stat = ' '.join(["{}:{}".format(k.upper(), exts[k]) for k in exts])
self.log.info('\t{} files moved: {}. Total {}'.format(
files_moved, ext_stat, common.size_human(bytes_moved)))
files_moved = 0
bytes_moved = 0
self.log.info(
'########################################################')
def __init__(self, filename=None):
if not filename:
return
self.filename = filename
self.helper = CommonHelper()
self.log = logging.getLogger(f"META")
self.log.info(f'========== {filename} ==========')
self.ProcessFileName()
if self.mediatype == 'image':
self.ProcessImageMeta()
elif self.mediatype == 'video':
self.ProcessVideoMeta()
self.ProcessFileAttributes()
class ElasticSearchProvider(PipelineShotProvider):
def __init__(self, camera: str, datetime: datetime, isSimulation=False):
super().__init__("ELSE")
self.camera = camera
self.datetime = datetime
self.helper = CommonHelper()
self.isSimulation = isSimulation
(self.elasticsearch_host, self.elasticsearch_port) = (None, None)
if AppSettings.ELASTICSEARCH_HOST:
(self.elasticsearch_host, self.elasticsearch_port
) = AppSettings.ELASTICSEARCH_HOST.split(':')
def GetShotsProtected(self, pShots: []):
dtUtc = self.helper.ToUtcTime(self.datetime)
index = self.helper.GetEsCameraArchiveIndex(dtUtc)
id = self.helper.GetEsShotId(self.camera, dtUtc)
if not self.isSimulation:
es = Elasticsearch([{
'host': self.elasticsearch_host,
'port': self.elasticsearch_port
}])
#res = es.get(index="cameraarchive-2019.10", doc_type='_doc', id='Foscam@2019-10-20T15:18:08.000Z')
res = es.get(index=index, doc_type='_doc', id=id)
path_cv = res['_source'][
'path_cv'] # /CameraArchive/Foscam/2019-10/20/20191020_171808_Foscam_cv.jpeg
path = res['_source'][
'path'] # /CameraArchive/Foscam/2019-10/20/20191020_171808_Foscam.jpg
else:
path_cv = "/CameraArchive/Foscam/2019-10/20/20191020_171808_Foscam_cv.jpeg"
path = "/CameraArchive/Foscam/2019-10/20/20191020_171808_Foscam.jpg"
shot = CamShot(
os.path.join(AppSettings.CAMERA_ARCHIVE_PATH,
path_cv.lstrip('/').lstrip('\\')))
pShot = PipelineShot(shot)
pShot.OriginalShot = CamShot(
os.path.join(AppSettings.CAMERA_ARCHIVE_PATH,
path.lstrip('/').lstrip('\\')))
meta = self.CreateMetadata(pShot)
meta['id'] = id
meta['index'] = index
return [pShot]
def __init__(self):
# self.Result = ProcessingResult()
# self.Result.Summary = {}
self.log = logging.getLogger(
"PROC:DIFF") # :{shot.filenameWithoutExtension}
# self.shot = ctx.Shot
# self.originalShot = ctx.OriginalShot
# self.originalShots = ctx.OriginalShots
# self.shots = ctx.Shots
self.helper = CommonHelper()
class FilesWalkerProvider(Provider):
def __init__(self, root: str = None, ignoreDirs=[], condition=None):
super().__init__("WALK")
self.helper = CommonHelper()
self.root = root
self.ignoreDirs = ignoreDirs
self.condition = condition if condition != None else (lambda f: True)
def GetProtected(self, context) -> []:
files = self.helper.WalkFiles(self.root, self.condition,
self.ignoreDirs)
return list(files)
def __init__(self, yolo_data):
self.log = logging.getLogger(f"PROC:TRAC:TBox")
self.helper = CommonHelper()
self.image = []
self.object_id = None
(self.w, self.h) = yolo_data['size']
self.center = yolo_data['center_coordinate']
self.yolo_label = yolo_data['label']
self.id = f"{self.center[0]}x{self.center[1]}"
(self.x, self.y) = self.center
(self.point_left_top,
self.point_right_bottom) = self.TransformCenterToLimits(
self.x, self.y, self.point_size, self.point_size)
(self.box_left_top,
self.box_right_bottom) = self.TransformCenterToLimits(
self.x, self.y, self.h, self.w)
self.pos_text = (self.x, self.y - 5)
class PhotosArrangeProcessor(Processor):
def __init__(self, label: str, isSimulation: bool = False):
super().__init__("PHOT", isSimulation)
self.label = label
self.helper = CommonHelper()
def ProcessItem(self, filename: str, context: dict):
# self.log.debug(f' From: {filename}')
basename = os.path.basename(filename)
dt = context['meta']['CRED'][filename]['shottime']
dtPath = dt.strftime('%Y/%y-%m') + ' ' + self.label
toPath = os.path.join(AppSettings.PHOTO_ARCHIVE_PATH, dtPath)
to = os.path.join(toPath, basename)
# self.log.debug(f' To: {to}')
meta = self.CreateMetadata(filename, context)
meta['arch_path'] = to
self.helper.EnsureMove(filename, to, 'copy' if self.isSimulation else 'move')
class CameraArchiveConfig:
sensor: {}
position: {}
camera: str
imap_folder: str
path_from: str
path_to: str
ignore_dir: []
camera_triggered_interval_sec: int
helper = CommonHelper()
def __init__(self):
self.ignore_dir = []
self.secretConfig = SecretConfig()
self.secretConfig.fromJsonFile()
def fromJsonFile(self, filename: str):
with open(filename, "r") as read_file:
self.__dict__ = json.load(read_file)
if not hasattr(self, 'ignore_dir'):
self.ignore_dir = []
def fromJson(self, json_dump):
self.__dict__ = json.loads(json_dump)
if not hasattr(self, 'ignore_dir'):
self.ignore_dir = []
def toJson(self):
return json.dumps(self.__dict__, indent=4)
def __repr__(self):
return 'CONFIG: {}: = {}'.format(self.camera, self.path_from)
def pathFrom(self):
return os.path.join(AppSettings.CAMERA_LIVE_PATH, self.path_from)
def pathTo(self):
return self.path_to
class ImapShotsProvider(PipelineShotProvider):
def __init__(self, tempFolder='temp'):
super().__init__("IMAP")
self.secretConfig = SecretConfig()
self.secretConfig.fromJsonFile()
self.tempFolder = tempFolder
self.helper = CommonHelper()
def GetShotsProtected(self, pShots: []):
self.Connect()
mail = self.GetLastMail(self.config.imap_folder)
os.makedirs(self.tempFolder, exist_ok=True)
file_template = self.tempFolder + '/{:%Y%m%d-%H%M%S}-{}.jpg'
shots = self.SaveAttachments(mail, file_template, self.CleanOldFiles)
pShots.extend(shots)
self.Disconnect()
return shots
def CleanOldFiles(self, shot: CamShot):
secs = self.config.camera_triggered_interval_sec
dt = shot.GetDatetime(False)
if not dt:
return
condition = lambda f: self.helper.FileNameByDateRange(f, dt, secs)
removed = self.helper.CleanFolder(self.tempFolder, condition)
[self.log.info(f'REMOVED: {f}') for f in removed]
# filePattern : /path_to_file/{:%Y%m%d-%H%M%S}-{}.jpg
def SaveAttachments(self, mail, filePattern: str, beforeFirstSave: None):
index = 0
result = []
first_dt = None
for part in mail.walk():
if (part.get_content_maintype() != 'image'):
continue
fileName = part.get_filename()
if bool(fileName):
memShot = CamShot(fileName)
dtShot = memShot.GetDatetime()
#dtShot = dt + datetime.timedelta(0,index)
shot = CamShot(filePattern.format(dtShot, self.config.camera))
if index == 0:
first_dt = dtShot
if beforeFirstSave:
beforeFirstSave(shot)
if not shot.Exist():
shot.Write(part.get_payload(decode=True))
else:
self.log.info(
f'Attachment already exists: {shot.fullname}')
pShot = PipelineShot(shot, index)
meta = self.CreateMetadata(pShot)
meta["start"] = self.helper.ToTimeStampStr(first_dt)
meta["filename"] = fileName
result.append(pShot)
index += 1
return result
def GetLastMail(self, imap_folder: str):
self.imapSession.select(imap_folder)
typ, data = self.imapSession.search(None, 'ALL')
if typ != 'OK':
self.log.error(f'Error searching in imap folder: {imap_folder}')
raise ValueError('Error searching in imap folder: ' + imap_folder)
# Iterating over all emails
ids = data[0].split()
self.log.debug(f'Found {len(ids)} mails in "{imap_folder}"')
msgId = ids[-1].decode('utf-8')
#for msgId in data[0].split():
typ, messageParts = self.imapSession.fetch(msgId, '(RFC822)')
if typ != 'OK':
self.log.error(f'Error fetching mail: {msgId}')
raise ValueError('Error fetching mail: ' + msgId)
emailBody = messageParts[0][1].decode('utf-8')
mail = email.message_from_string(emailBody)
subject, _ = email.header.decode_header(mail['subject'])[0]
self.log.info("#{} | {}".format(msgId, subject.decode('utf-8')))
return mail
def Connect(self):
self.imapSession = imaplib.IMAP4_SSL('imap.gmail.com')
typ, accountDetails = self.imapSession.login(
self.secretConfig.gmail_username, self.secretConfig.gmail_password)
if typ != 'OK':
self.log.debug('Not able to sign in!')
print('Not able to sign in!')
raise ConnectionError('imap.gmail.com')
self.log.debug(f'Connection: {accountDetails}')
def Disconnect(self):
self.imapSession.select()
self.imapSession.close()
self.imapSession.logout()
class GmailContext():
def __init__(self):
self.log = logging.getLogger('IMAP')
self.config = SecretConfig()
self.config.fromJsonFile()
self.helper = CommonHelper()
def DownoadLastAttachments(self, imap_folder: str, temp: str):
self.Connect()
mail = self.GetLastMail(imap_folder)
os.makedirs(temp, exist_ok=True)
self.SaveAttachments(mail, temp + '/MDAlarm_{:%Y%m%d-%H%M%S}-{}.jpg')
self.Disconnect()
def Connect(self):
self.imapSession = imaplib.IMAP4_SSL('imap.gmail.com')
typ, accountDetails = self.imapSession.login(
self.config.gmail_username, self.config.gmail_password)
if typ != 'OK':
print('Not able to sign in!')
raise
self.log.debug('Connection: ', accountDetails)
def Disconnect(self):
self.imapSession.close()
self.imapSession.logout()
def GetLastMail(self, imap_folder: str):
self.imapSession.select(imap_folder)
typ, data = self.imapSession.search(None, 'ALL')
if typ != 'OK':
print('Error searching Inbox.')
raise
# Iterating over all emails
ids = data[0].split()
self.log.debug('Found {} mails in "{}"'.format(len(ids), imap_folder))
msgId = ids[-1].decode('utf-8')
#for msgId in data[0].split():
typ, messageParts = self.imapSession.fetch(msgId, '(RFC822)')
if typ != 'OK':
print('Error fetching mail.')
raise
emailBody = messageParts[0][1].decode('utf-8')
mail = email.message_from_string(emailBody)
subject, _ = email.header.decode_header(mail['subject'])[0]
self.log.info("#{} | {}".format(msgId, subject.decode('utf-8')))
return mail
# filePattern : /path_to_file/MDAlarm_{:%Y%m%d-%H%M%S}-{}.jpg
def SaveAttachments(self, mail, filePattern: str):
index = 0
for part in mail.walk():
if (part.get_content_maintype() != 'image'):
continue
fileName = part.get_filename()
if bool(fileName):
dt = self.helper.get_datetime(fileName)
filePath = filePattern.format(dt, index)
#filePath = os.path.join(output_dir, fileName)
if not os.path.isfile(filePath):
self.log.info('Save mail attachment to: ', filePath)
fp = open(filePath, 'wb')
fp.write(part.get_payload(decode=True))
fp.close()
else:
self.log.info('[MAIL] Attachment already exists: ',
filePath)
index += 1
class FileInfo:
filename: str # file.jpg
path: str # C:\Path\to\file.jpg
dir: str # C:\Path\to\
def __init__(self, full_filename):
self.helper = CommonHelper()
self.path = full_filename
self.filename = os.path.basename(full_filename)
self.dir = os.path.dirname(full_filename)
def get_extension(self):
_, file_extension = os.path.splitext(self.filename)
return file_extension.replace('.', '')
def get_datetime(self) -> datetime.datetime:
re_groups = re.search(
"(20\\d\\d)[_-]?(\\d\\d)[_-]?(\\d\\d)[_-]?(\\d\\d)[_-]?(\\d\\d)[_-]?(\\d\\d)",
self.filename)
if not re_groups:
print('Cant parse datetime in file : {}'.format(self.path))
raise ValueError('Cant parse datetime in file : {}'.format(
self.path))
year = int(re_groups.group(1))
month = int(re_groups.group(2))
day = int(re_groups.group(3))
hour = int(re_groups.group(4))
minute = int(re_groups.group(5))
seconds = int(re_groups.group(6))
return datetime.datetime(year, month, day, hour, minute, seconds)
def get_datetime_utc(self):
naive = self.get_datetime()
return self.helper.ToUtcTime(naive)
def get_month_id_utc(self):
return self.get_datetime_utc().strftime('%Y')
def get_timestamp(self): # 'MDAlarm_20190131-153706'
return self.helper.ToTimeStampStr(self.get_datetime())
def get_timestamp_utc(self): # 'MDAlarm_20190131-153706'
return self.helper.ToTimeStampStr(self.get_datetime_utc())
def size(self):
return os.path.getsize(self.path)
def size_human(self):
size = self.size()
return self.helper.size_human(size)
'''
//diskstation/CameraArchive/Foscam :
//diskstation/CameraArchive/Foscam/2016-07-26/record/alarm_20160404_010956.mkv => /2016-07-26/record/alarm_20160404_010956.mkv
'''
def get_path_relative(self, dir_base: str):
return self.path.replace(dir_base, '')
'''
//diskstation/CameraArchive/Foscam :
//diskstation/CameraArchive/Foscam/2016-07-26/record/alarm_20160404_010956.mkv => /2016-07-26/record
'''
def get_dir_relative(self, dir_base: str):
path_relative = self.get_path_relative(dir_base)
return os.path.dirname(path_relative)
def __init__(self, full_filename):
self.helper = CommonHelper()
self.path = full_filename
self.filename = os.path.basename(full_filename)
self.dir = os.path.dirname(full_filename)
def __init__(self):
self.log = logging.getLogger('IMAP')
self.config = SecretConfig()
self.config.fromJsonFile()
self.helper = CommonHelper()
def __init__(self, name, isSimulation=False):
super().__init__(name, isSimulation)
self.helper = CommonHelper()
'''
run locally :
> set FLASK_APP=app.py
> flask run
-OR-
> run.cmd
'''
temp = 'temp'
imap_folder = 'camera/foscam'
camera = 'Foscam'
isSimulation = False
secretConfig = SecretConfig()
secretConfig.fromJsonFile()
helper = CommonHelper()
file_error_handler = logging.FileHandler(filename='camera-tools-error.log')
file_error_handler.setLevel(logging.ERROR)
file_handler = logging.handlers.TimedRotatingFileHandler('camera-tools.log',
when='midnight',
backupCount=7)
file_handler.suffix = '_%Y-%m-%d.log'
stdout_handler = logging.StreamHandler(sys.stdout)
handlers = [file_handler, stdout_handler, file_error_handler]
logging.basicConfig(
format=
'%(asctime)s|%(levelname)-.3s|%(name)s: %(message)s', # \t####=> %(filename)s:%(lineno)d
level=logging.DEBUG,
datefmt='%H:%M:%S',
class Shot:
filename = ''
contourArea: int
image: any
image_timestamp: any
Contours = []
YoloResult = []
datetime: datetime
helper = CommonHelper()
imageAnalyseResult = ImageAnalyseResult()
magnifiedRegion = []
def FromFile(self, path: str):
self = Shot()
self.log = logging.getLogger("SHOT")
self.filename = path
self.log.info('Open file: {}'.format(self.filename))
self.image = cv2.imread(self.filename, cv2.IMREAD_GRAYSCALE)
self.image_color = cv2.imread(self.filename, cv2.IMREAD_COLOR)
self.image_contours = self.image_color.copy()
if len(self.image) == 0:
raise ValueError('cant load: {}'.format(self.filename))
self.image_timestamp = self.image[:22, :230]
self.image[:22, :230] = 0 # remove timestamp
self.datetime = self.helper.get_datetime(self.filename)
return self
def show_cv(self):
cv2.imshow('shot.show_cv (press any key for close)', self.image)
cv2.waitKey(0)
cv2.destroyAllWindows()
def show_plt(self):
plt.axis("off")
#plt.title('shot.show_plt')
img = cv2.cvtColor(self.image_contours, cv2.COLOR_BGR2RGB)
plt.imshow(img, interpolation="bilinear")
def DrawContours(self):
cv2.drawContours(self.image_contours, self.Contours, -1, (0, 255, 255),
1)
for c in self.Contours[0:2]:
area = int(cv2.contourArea(c) / 100)
#print('Contour: {}'.format(area))
(x, y, w, h) = cv2.boundingRect(c)
cv2.rectangle(self.image_contours, (x, y), (x + w, y + h),
(0, 255, 0), 1, 8)
cv2.putText(self.image_contours, str(area), (x, y - 3),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2,
cv2.LINE_AA)
def MagnifyMotion(self):
if len(self.Contours) == 0:
return
counts = 1
margin = 5
zoom = 2
for c in self.Contours[0:counts]:
(x, y, w, h) = cv2.boundingRect(c)
if math.sqrt(w * w + h * h) > 200:
return
self.magnifiedRegion = [x, y, w, h]
img = self.image_color
motion_area = img[y:(y + h), x:(x + w)]
img_xmax = len(img[0])
img_ymax = len(img)
motion_area = cv2.resize(motion_area,
None,
fx=zoom,
fy=zoom,
interpolation=cv2.INTER_CUBIC)
y_max = img_ymax - margin
y_min = y_max - zoom * h
x_max = img_xmax - margin
x_min = x_max - zoom * w
if y_min < margin or x_min < margin:
continue # can fit, too large
self.image_contours[y_min:y_max, x_min:x_max] = motion_area
cv2.rectangle(self.image_contours, (x_min, y_min), (x_max, y_max),
(127, 127, 127), 2)
def CalcHaarBody(self):
cascadePath = "cascade\\haarcascade_frontalface_alt.xml"
cascade = cv2.CascadeClassifier(cascadePath)
objects = cascade.detectMultiScale(self.image,
scaleFactor=1.1,
minNeighbors=1,
minSize=(30, 30))
#print('=HAAR=: Detected bodies : {}'.format(len(objects)))
for (x, y, w, h) in objects:
#crop = image[y: y + h, x: x + w]
cv2.rectangle(self.image_contours, (x, y), (x + w, y + h),
(255, 0, 0), 2)
def CalcContoursAnalyseResult(self):
self.imageAnalyseResult.contours = []
for c in self.Contours:
area = int(cv2.contourArea(c))
#print('Contour: {}'.format(area))
(x, y, w, h) = cv2.boundingRect(c)
(center_x, center_y) = (x + w // 2, y + h // 2)
result = ContourAnalyseResult()
result.area = area
result.profile_proportion = round(h / w, 2)
result.center_coordinate = [center_x, center_y]
self.imageAnalyseResult.contours.append(result)
def __init__(self, isDebug=False):
super().__init__("TRAC")
self.boxes_last = []
self.helper = CommonHelper()
self.isDebug = isDebug
def __init__(self, tempFolder='temp'):
super().__init__("IMAP")
self.secretConfig = SecretConfig()
self.secretConfig.fromJsonFile()
self.tempFolder = tempFolder
self.helper = CommonHelper()
class TestPipeline(unittest.TestCase):
log: logging.Logger = None
def __init__(self, *args, **kwargs):
super(TestPipeline, self).__init__(*args, **kwargs)
self.testHelper = TestHelper()
self.log = self.testHelper.CreateLog(TestPipeline.__name__)
TestPipeline.log = self.log
self.archiver = CameraArchiveHelper(self.log)
self.helper = CommonHelper()
# self.log.info('')
# self.log.info(' ############################ ')
# self.log.info(' ### SETUP ################## ')
# self.log.info(' ############################ ')
# from Common import HtmlLogger
# html_handler = HtmlLogger.HTMLFileHandler('camera-tools.html')
# html_handler.suffix = '_%Y-%m-%d.html'
def setUp(self):
self.testHelper.setUp(self.log, self._testMethodName)
def tearDown(self):
self.testHelper.tearDown(self.log, self._testMethodName)
# @classmethod
# def tearDownClass(self):
# TestPipeline.log.info(' ############################ ')
# TestPipeline.log.info(' ### TEARDOWN ############### ')
# TestPipeline.log.info(' ############################ ')
def test_imapShotsProvider(self):
# python -m unittest tests.test_pipeline.TestPipeline.test_imapShotsProvider
target = ImapShotsProvider('temp/queue')
target.config = self.archiver.load_configs('configs', ['Foscam'])[0]
shots = target.GetShots([])
self.assertEqual(3, len(shots))
self.assertIsNotNone(shots[0].Shot.filename)
self.assertIsNotNone(shots[0].Shot.fullname)
self.assertIsNotNone(shots[0].Shot.Exist())
#self.assertEqual(shots[0].Metadata['PROV:IMAP']['filename'], 'Snap_20191028-192355-0.jpg')
start = shots[0].Metadata['PROV:IMAP']['start']
self.assertEqual(shots[1].Metadata['PROV:IMAP']['start'], start)
self.assertEqual(shots[2].Metadata['PROV:IMAP']['start'], start)
def test_DirectoryShotsProvider(self):
# python -m unittest tests.test_pipeline.TestPipeline.test_DirectoryShotsProvider
folder = '../camera-OpenCV-data/Camera/Foscam/Day_Lilia_Gate'
shots = DirectoryShotsProvider.FromDir(None, folder)
self.assertEqual(3, len(shots))
self.assertIsNotNone(shots[0].Shot.filename)
self.assertIsNotNone(shots[0].Shot.fullname)
self.assertIsNotNone(shots[0].Shot.Exist())
#shots[0].Show()
def test_DirectoryShotsProvider_SearchByDate(self):
# python -m unittest tests.test_pipeline.TestPipeline.test_DirectoryShotsProvider_SearchByDate
folder = '../camera-OpenCV-data/Camera/Foscam/Day_Lilia_Gate'
pShots = DirectoryShotsProvider.FromDir(None, folder)
# Create base Shot
for pShot in pShots:
pShot.Metadata['PROV:IMAP'] = {}
pShot.Metadata['PROV:IMAP']['start'] = self.helper.ToTimeStampStr(
pShot.Shot.GetDatetime())
target = DirectoryShotsProvider()
target.config = self.archiver.load_configs('configs', ['Foscam'])[0]
target.config.path_from = 'Foscam\\2019-02'
pShots = target.GetShots(pShots)
# search in C:\Src\camera-OpenCV-data\Camera\Foscam\2019-02\06
self.assertEqual('Snap_20190206-090254-0.jpg', pShots[0].Shot.filename)
self.assertEqual('Snap_20190206-090254-1.jpg', pShots[1].Shot.filename)
self.assertEqual('Snap_20190206-090254-2.jpg', pShots[2].Shot.filename)
self.assertEqual('MDAlarm_20190206-090259.jpg',
pShots[3].Shot.filename)
self.assertEqual('MDAlarm_20190206-090304.jpg',
pShots[4].Shot.filename)
def test_DiffContoursProcessor(self):
# python -m unittest tests.test_pipeline.TestPipeline.test_DiffContoursProcessor
folder = '../camera-OpenCV-data/Camera/Foscam/Day_Lilia_Gate'
target = DiffContoursProcessor()
pipelineShots = DirectoryShotsProvider.FromDir(None, folder)
target.Process({'items': pipelineShots})
# pp.pprint(pipelineShots[0].Metadata, indent=2)
# pp.pprint(pipelineShots[1].Metadata, indent=2)
# pp.pprint(pipelineShots[2].Metadata, indent=2)
# result[0].Shot.Show()
# result[1].Shot.Show()
# result[2].Shot.Show()
metadata = pipelineShots[0].Metadata['DIFF']
self.assertGreater(metadata['Diff']['TotalArea'], 5000)
self.assertLess(metadata['Diff']['TotalArea'], 6000)
self.assertEqual(metadata['Diff']['Count'], 3)
self.assertEqual(len(metadata['boxes']), 3)
self.assertGreater(metadata['boxes'][0]['area'], 5000)
self.assertLess(metadata['boxes'][0]['area'], 6000)
def test_YoloObjDetectionProcessor_noObjects(self):
# python -m unittest tests.test_pipeline.TestPipeline.test_YoloObjDetectionProcessor_noObjects
# INIT
folder = '../camera-OpenCV-data/Camera/Foscam/Day_No_Objects'
pipeline = ShotsPipeline('Foscam', self.log)
pipeline.providers.append(DirectoryShotsProvider(folder))
pipeline.processors.append(
TestProcessor({'YOLO': {
'labels': 'person:2 car bird'
}}))
pipeline.processors.append(YoloObjDetectionProcessor())
pipeline.PreLoad()
# TEST
shots = pipeline.GetShots()
shots = pipeline.Process(shots)
# ASSERT
self.assertEqual(5, len(shots))
self.assertEqual(shots[0].Shot.GetDatetime(),
datetime.datetime(2019, 10, 16, 14, 21, 48))
self.assertEqual(shots[1].Shot.GetDatetime(),
datetime.datetime(2019, 10, 16, 14, 21, 50))
self.assertEqual(shots[2].Shot.GetDatetime(),
datetime.datetime(2019, 10, 16, 14, 21, 52))
self.assertEqual(shots[3].Shot.GetDatetime(),
datetime.datetime(2019, 10, 16, 14, 21, 53))
self.assertEqual(shots[4].Shot.GetDatetime(),
datetime.datetime(2019, 10, 16, 14, 21, 58))
self.assertFalse('YOLO' in shots[0].Metadata)
self.assertFalse('YOLO' in shots[1].Metadata)
self.assertFalse('YOLO' in shots[2].Metadata)
def test_YoloObjDetectionProcessor(self):
# python -m unittest tests.test_pipeline.TestPipeline.test_YoloObjDetectionProcessor
folder = '../camera-OpenCV-data/Camera/Foscam/Day_Lilia_Gate'
target = YoloObjDetectionProcessor()
target.PreLoad()
shots = DirectoryShotsProvider.FromDir(None, folder)
target.Process({'items': shots})
metadata0 = shots[0].Metadata['YOLO']
metadata1 = shots[1].Metadata['YOLO']
metadata2 = shots[2].Metadata['YOLO']
pp.pprint(metadata0, indent=2)
self.assertEqual(len(metadata0['areas']), 1)
self.assertEqual(len(metadata1['areas']), 1)
self.assertEqual(len(metadata2['areas']), 1)
self.assertEqual(metadata0['areas'][0]['label'], 'person')
self.assertEqual(metadata1['areas'][0]['label'], 'person')
self.assertEqual(metadata2['areas'][0]['label'], 'person')
self.assertEqual(metadata0['labels'], 'person')
self.assertEqual(metadata1['labels'], 'person')
self.assertEqual(metadata2['labels'], 'person')
#pipelineShots[0].Shot.Show()
def test_TrackingProcessor(self):
# python -m unittest tests.test_pipeline.TestPipeline.test_TrackingProcessor
folder = '../camera-OpenCV-data/Camera/Foscam/Day_Sergey_and_Olivia_tracking'
yolo = YoloObjDetectionProcessor()
yolo.PreLoad()
target = TrackingProcessor(isDebug=True)
pipelineShots = DirectoryShotsProvider.FromDir(None, folder)
yolo.Process({'items': pipelineShots})
target.Process({'items': pipelineShots})
meta = pipelineShots[1].Metadata["TRAC"]
pp.pprint(meta, indent=2)
#pipelineShots[0].Shot.Show()
#pipelineShots[1].Shot.Show()
boxesById = {b['id']: b for b in meta['boxes']}
self.assertEqual(15, boxesById['372x122']['angle'])
self.assertEqual(138, boxesById['372x122']['distance'])
self.assertEqual("372x122", boxesById['372x122']['center'])
self.assertEqual(16, boxesById['230x146']['angle'])
self.assertEqual(90, boxesById['230x146']['distance'])
self.assertEqual("230x146", boxesById['230x146']['center'])
meta = pipelineShots[2].Metadata["TRAC"]
pp.pprint(meta, indent=2)
#pipelineShots[2].Shot.Show()
boxesById = {b['id']: b for b in meta['boxes']}
self.assertEqual(10, boxesById["323x129"]['angle'])
self.assertEqual(94, boxesById["323x129"]['distance'])
self.assertEqual("323x129", boxesById["323x129"]['center'])
self.assertEqual(28, boxesById["432x89"]['angle'])
self.assertEqual(68, boxesById["432x89"]['distance'])
self.assertEqual("432x89", boxesById["432x89"]['center'])
def test_TrackingProcessor_Day_3Person_2person_same_color(self):
# python -m unittest tests.test_pipeline.TestPipeline.test_TrackingProcessor_Day_3Person_2person_same_color
folder = '../camera-OpenCV-data/Camera/Foscam/Day_3Person_2person_same_color'
yolo = YoloObjDetectionProcessor()
yolo.PreLoad()
target = TrackingProcessor(isDebug=True)
pipelineShots = DirectoryShotsProvider.FromDir(None, folder)
yolo.Process({'items': pipelineShots})
target.Process({'items': pipelineShots})
# pipelineShots[0].Shot.Show()
metadata0 = pipelineShots[0].Metadata["TRAC"]['boxes']
pp.pprint(metadata0, indent=2)
self.assertListEqual([{
'id': '293x148',
'object_id': 1
}, {
'id': '350x151',
'object_id': 2
}, {
'id': '379x103',
'object_id': 3
}], metadata0)
#pipelineShots[1].Shot.Show()
metadata1 = pipelineShots[1].Metadata["TRAC"]['boxes']
pp.pprint(metadata1, indent=2)
self.assertListEqual([{
'id': '290x153',
'angle': -178,
'center': '290x153',
'distance': 60,
'object_id': 2
}, {
'id': '349x101',
'angle': 176,
'center': '349x101',
'distance': 30,
'object_id': 3
}, {
'id': '218x168',
'angle': -165,
'center': '218x168',
'distance': 77,
'object_id': 1
}], metadata1)
#pipelineShots[2].Shot.Show()
metadata2 = pipelineShots[2].Metadata["TRAC"]
pp.pprint(metadata2, indent=2)
# TODO must be object='2' not '3'
# self.assertEqual(metadata2['193x176']['object_id'], 2)
def test_TrackingProcessor_Day_3Person_2stay(self):
# python -m unittest tests.test_pipeline.TestPipeline.test_TrackingProcessor_Day_3Person_2stay
folder = '../camera-OpenCV-data/Camera/Foscam/Day_3Person_2stay'
yolo = YoloObjDetectionProcessor()
yolo.PreLoad()
target = TrackingProcessor(isDebug=True)
pipelineShots = DirectoryShotsProvider.FromDir(None, folder)
yolo.Process({'items': pipelineShots})
target.Process({'items': pipelineShots})
for i, pShot in enumerate(pipelineShots):
meta = pShot.Metadata["TRAC"]['boxes']
title = ""
for box in meta:
title += f" B:{box['id']}-ID{box['object_id']}"
print(f"Shot #{i}: {pShot.Shot.filename}")
pp.pprint(meta, indent=2)
#pShot.Shot.Show(title)
# ID1 : Oli, ID2 : Fra
meta = pipelineShots[0].Metadata["TRAC"]
boxesById = {b['id']: b for b in meta['boxes']}
self.assertDictEqual(
boxesById, {
'201x281': {
'id': '201x281',
'object_id': 2
},
'240x373': {
'id': '240x373',
'object_id': 1
}
})
meta = pipelineShots[1].Metadata["TRAC"]
boxesById = {b['id']: b for b in meta['boxes']}
self.assertDictEqual(
boxesById, {
'183x288': {
'angle': -158,
'center': '183x288',
'distance': 19,
'id': '183x288',
'object_id': 2
},
'277x315': {
'angle': 57,
'center': '277x315',
'distance': 68,
'id': '277x315',
'object_id': 1
}
})
meta = pipelineShots[2].Metadata["TRAC"]
boxesById = {b['id']: b for b in meta['boxes']}
self.assertDictEqual(
boxesById,
{
'177x299': {
'angle': -118,
'center': '177x299',
'distance': 12,
'id': '177x299',
'object_id': 2
},
'189x247': {
'id': '189x247',
'object_id': 3
}, # Papa
'289x305': {
'angle': 39,
'center': '289x305',
'distance': 15,
'id': '289x305',
'object_id': 1
}
})
meta = pipelineShots[3].Metadata["TRAC"]
boxesById = {b['id']: b for b in meta['boxes']}
self.assertDictEqual(
boxesById['162x285'], {
'angle': 136,
'center': '162x285',
'distance': 20,
'id': '162x285',
'object_id': 2
})
def test_TrackingProcessor2(self):
# python -m unittest tests.test_pipeline.TestPipeline.test_TrackingProcessor2
folder = '../camera-OpenCV-data/Camera/Foscam/Day_Lilia_Gate'
pipeline = ShotsPipeline('Foscam', self.log)
pipeline.providers.append(DirectoryShotsProvider(folder))
pipeline.processors.append(YoloObjDetectionProcessor())
pipeline.processors.append(TrackingProcessor(isDebug=True))
pipeline.PreLoad()
shots = pipeline.GetShots()
result = pipeline.Process(shots)
meta = result[1].Metadata["TRAC"]
pp.pprint(meta, indent=2)
#result[1].Shot.Show()
boxesById = {b['id']: b for b in meta['boxes']}
self.assertDictEqual(
boxesById, {
'289x101': {
'angle': 25,
'center': '289x101',
'distance': 94,
'id': '289x101',
'object_id': 1
}
})
meta = result[2].Metadata["TRAC"]
pp.pprint(meta, indent=2)
boxesById = {b['id']: b for b in meta['boxes']}
#result[2].Shot.Show()
self.assertDictEqual(
boxesById, {
'377x84': {
'angle': 10,
'center': '377x84',
'distance': 89,
'id': '377x84',
'object_id': 1
}
})
def test_TrackingProcessor_SizeError(self):
# python -m unittest tests.test_pipeline.TestPipeline.test_TrackingProcessor_SizeError
folder = '../camera-OpenCV-data/Camera/Foscam/Morning_Sergey_Tracking_Error'
yolo = YoloObjDetectionProcessor()
yolo.PreLoad()
target = TrackingProcessor(isDebug=True)
pipelineShots = DirectoryShotsProvider.FromDir(None, folder)
yolo.Process({'items': pipelineShots})
target.Process({'items': pipelineShots})
metadata1 = pipelineShots[1].Metadata["TRAC"]
pp.pprint(metadata1, indent=2)
# pipelineShots[1].Shot.Show()
boxesById = {b['id']: b for b in metadata1['boxes']}
self.assertDictEqual(
boxesById, {
'436x69': {
'angle': 27,
'center': '436x69',
'distance': 83,
'id': '436x69',
'object_id': 1
}
})
def test_MailSend(self):
# python -m unittest tests.test_pipeline.TestPipeline.test_MailSend
folder = '../camera-OpenCV-data/Camera/Foscam/Day_Lilia_Gate'
pipeline = ShotsPipeline('Foscam', self.log)
pipeline.providers.append(DirectoryShotsProvider(folder))
pipeline.processors.append(
TestProcessor({'YOLO': {
'labels': 'person:2 car bird'
}}))
pipeline.processors.append(DiffContoursProcessor())
pipeline.processors.append(MailSenderProcessor(True))
pipeline.PreLoad()
shots = pipeline.GetShots()
shots[0].Metadata['YOLO'] = {}
shots[1].Metadata['YOLO'] = {}
shots[2].Metadata['YOLO'] = {}
shots[0].Metadata['YOLO']['labels'] = "person:2 car"
shots[1].Metadata['YOLO']['labels'] = "person bird car"
shots[2].Metadata['YOLO']['labels'] = ""
shots[0].Metadata['YOLO']['areas'] = [
self.getYoloArea('person'),
self.getYoloArea('person'),
self.getYoloArea('car')
]
shots[1].Metadata['YOLO']['areas'] = [
self.getYoloArea('person'),
self.getYoloArea('bird'),
self.getYoloArea('car')
]
shots[2].Metadata['YOLO']['areas'] = []
result = pipeline.Process(shots)
sendMeta = result[0].Metadata['SMTP']
self.assertEqual(sendMeta["Subject"],
"Foscam @09:02:54 person:2 car bird (06.02.2019)")
self.assertEqual(sendMeta["id"], 'Foscam@2019-02-06T09:02:56.000Z')
#self.assertEqual(sendMeta["Body"], "BODY")
#self.assertGreater(sendMeta["MessageSize"], 200000)
def getYoloArea(self, label: str):
return {
"area": 5562,
"center_coordinate": [169, 158],
"confidence": 0.99,
"label": label,
"profile_proportion": 1.91,
"size": [54, 103]
}
def test_ArchiveProcessor(self):
# python -m unittest tests.test_pipeline.TestPipeline.test_ArchiveProcessor
folder = '../camera-OpenCV-data/Camera/Foscam/Day_Lilia_Gate'
pipeline = ShotsPipeline('Foscam', self.log)
pipeline.providers.append(DirectoryShotsProvider(folder))
pipeline.processors.append(ArchiveProcessor(True))
pipeline.PreLoad()
shots = pipeline.GetShots()
result = pipeline.Process(shots)
archMD = result[0].Metadata['ARCH']
self.assertEqual(
archMD['archive_destination'], AppSettings.CAMERA_ARCHIVE_PATH +
'\\CameraArchive\\Foscam\\2019-02\\06\\20190206_090254_Foscam_cv.jpeg'
)
self.assertEqual(
archMD['archive_destination_orig'],
AppSettings.CAMERA_ARCHIVE_PATH +
'\\CameraArchive\\Foscam\\2019-02\\06\\20190206_090254_Foscam.jpg')
def test_SaveToTemp(self):
# python -m unittest tests.test_pipeline.TestPipeline.test_SaveToTemp
folder = '../camera-OpenCV-data/Camera/Foscam/Day_Lilia_Gate'
pipeline = ShotsPipeline('Foscam', self.log)
pipeline.providers.append(DirectoryShotsProvider(folder))
pipeline.processors.append(DiffContoursProcessor())
pipeline.processors.append(SaveToTempProcessor(True))
pipeline.PreLoad()
shots = pipeline.GetShots()
result = pipeline.Process(shots)
self.assertEqual(result[0].Shot.fullname,
"temp\\20190206_090254_Foscam_cv.jpeg")
self.assertEqual(result[0].OriginalShot.fullname,
"temp\\20190206_090254_Foscam.jpg")
self.assertTrue(os.path.isfile(result[0].Shot.fullname))
self.assertTrue(os.path.isfile(result[0].OriginalShot.fullname))
self.assertEqual(result[1].Shot.fullname,
"temp\\20190206_090255_Foscam_cv.jpeg")
self.assertEqual(result[1].OriginalShot.fullname,
"temp\\20190206_090255_Foscam.jpg")
self.assertEqual(result[2].Shot.fullname,
"temp\\20190206_090256_Foscam_cv.jpeg")
self.assertEqual(result[2].OriginalShot.fullname,
"temp\\20190206_090256_Foscam.jpg")
def test_ElasticSearchProcessor(self):
# python -m unittest tests.test_pipeline.TestPipeline.test_ElasticSearchProcessor
folder = '../camera-OpenCV-data/Camera/Foscam/Day_Lilia_Gate'
CommonHelper.networkConfig = {}
pipeline = ShotsPipeline('Foscam', self.log)
pipeline.providers.append(DirectoryShotsProvider(folder))
pipeline.processors.append(DiffContoursProcessor())
pipeline.processors.append(ArchiveProcessor(True))
pipeline.processors.append(ElasticSearchProcessor(True))
#pipeline.processors.append(SaveToTempProcessor())
pipeline.PreLoad()
shots = pipeline.GetShots()
result = pipeline.Process(shots)
els = result[0].Metadata["ELSE"]
self.assertIsNotNone(els['JSON'])
#print(els['JSON'])
dictEls = json.loads(els['JSON'])
analyse = dictEls['Analyse']
del dictEls['Analyse']
archive = AppSettings.CAMERA_ARCHIVE_PATH.replace('\\', '/')
expected = {
"@timestamp": "2019-02-06T08:02:54.000Z",
"camera": "Foscam",
"doc": "event",
"ext": "jpg",
"path": archive +
"/CameraArchive/Foscam/2019-02/06/20190206_090254_Foscam.jpg",
"position": {
"detail": "under the roof",
"floor": -1,
"room": "FrontWall"
},
"sensor": {
"device": "Foscam FI9805W",
"display": "Foscam(.) FrontWall",
"id": "FoscamCameraArchiveFI9805W_C4D6553DECE1",
"is_external": True,
"type": "CameraArchive",
"unit": "bytes"
},
"tags": ["synology_cameraarchive", "camera_tools"],
"value": 69623,
"volume": "/volume2"
}
for key in expected:
self.assertEqual(dictEls[key], expected[key])
self.assertAlmostEqual(
analyse['DIFF']['boxes'][0]['profile_proportion'],
1.77,
delta=0.01)
def test_WholePipeline(self):
# python -m unittest tests.test_pipeline.TestPipeline.test_WholePipeline
CommonHelper.networkConfig = {}
folder = '../camera-OpenCV-data/Camera/Foscam/Day_Sergey_and_Olivia_tracking'
hassioDir = "temp"
if not os.path.exists(hassioDir):
os.mkdir(hassioDir)
## INIT
pipeline = ShotsPipeline('Foscam', self.log)
pipeline.providers.append(DirectoryShotsProvider(folder))
# # proceccor : Analyse() GetJsonResult() Draw()
#pipeline.processors.append(ZonesProcessor())
pipeline.processors.append(DiffContoursProcessor())
#pipeline.processors.append(MagnifyProcessor())
pipeline.processors.append(YoloObjDetectionProcessor())
pipeline.processors.append(TrackingProcessor())
# #post processors:
########################################################################
# Save analysed files to temp
# original: temp\20190203_085908_{camera}.jpg
# analysed: temp\20190203_085908_{camera}_cv.jpeg
pipeline.processors.append(SaveToTempProcessor(True))
########################################################################
# mail analysed files to gmail
# attached: {ARCHIVE}\2019\02\03\cv_20190203-085908-{n}-{camera}_cv.(jpeg|png)
# attached: info.json
# body : Analysis Log
# Subject : {HH:MM} {detected objects} {total_area_countours}
pipeline.processors.append(MailSenderProcessor(True))
########################################################################
# update files in hassio server
pipeline.processors.append(HassioProcessor(hassioDir))
########################################################################
# save original files and analysed to archive directory by date
# move from local archive to distant server (windows => diskstation)
# original: {ARCHIVE}\2019\02\03\20190203-085908-{n}-{camera}.jpg
# analysed: {ARCHIVE}\2019\02\03\20190203-085908-{n}-{camera}_cv.(jpeg|png)
pipeline.processors.append(ArchiveProcessor(True))
########################################################################
# add to ES info about files + analysed info
pipeline.processors.append(ElasticSearchProcessor(True))
pipeline.PreLoad()
## Procerss on shots comming
shots = pipeline.GetShots()
result = pipeline.Process(shots)
# analyseResult[0].Shot.Show()
# analyseResult[1].Shot.Show()
# analyseResult[2].Shot.Show()
self.assertTrue("TRAC" in result[1].Metadata)
self.assertTrue("YOLO" in result[1].Metadata)
self.assertTrue("DIFF" in result[1].Metadata)
self.assertTrue("TRAC" in result[2].Metadata)
self.assertTrue("YOLO" in result[2].Metadata)
self.assertTrue("DIFF" in result[2].Metadata)
self.assertTrue("TEMP" in result[2].Metadata)
self.assertTrue("HASS" in result[2].Metadata)
self.assertTrue("ELSE" in result[2].Metadata)
tempMD = result[0].Metadata['TEMP']
self.assertEqual(tempMD['fullname'],
"temp\\20190328_080122_Foscam_cv.jpeg")
#self.assertEqual(tempMD['original_fullname'], "temp\\20190328_080122_Foscam.jpg")
tempMD = result[1].Metadata['TEMP']
self.assertEqual(tempMD['fullname'],
"temp\\20190328_080123_Foscam_cv.jpeg")
#self.assertEqual(tempMD['original_fullname'], "temp\\20190328_080123_Foscam.jpg")
mailMD = result[0].Metadata['SMTP']
self.assertEqual(mailMD["Subject"],
"Foscam @08:01:22 person:2 (28.03.2019)")
#self.assertEqual(mailMD["Body"], "/** Processing LOG **/")
#self.assertGreater(mailMD["MessageSize"], 200000)
hassMD = result[0].Metadata['HASS']
self.assertEqual(hassMD['hassio_location'], 'temp\\cv_Foscam_0.jpg')
archMD = result[0].Metadata['ARCH']
self.assertEqual(
archMD['archive_destination'], AppSettings.CAMERA_ARCHIVE_PATH +
'\\CameraArchive\\Foscam\\2019-03\\28\\20190328_080122_Foscam_cv.jpeg'
)
self.assertEqual(
archMD['archive_destination_orig'],
AppSettings.CAMERA_ARCHIVE_PATH +
'\\CameraArchive\\Foscam\\2019-03\\28\\20190328_080122_Foscam.jpg')
els = result[0].Metadata["ELSE"]
self.assertIsNotNone(els['JSON'])
dictEls = json.loads(els['JSON'])
print(els['JSON'])
self.assertIsNotNone(dictEls['Analyse'])
self.assertEqual(dictEls['Analyse']['YOLO']['areas'][0]['label'],
"person")
self.assertEqual(dictEls['Analyse']['YOLO']['labels'], "person:2")
self.assertEqual(dictEls['Analyse']['SMTP']['Subject'],
"Foscam @08:01:22 person:2 (28.03.2019)")
def test_ElasticSearchProvider(self):
# python -m unittest tests.test_pipeline.TestPipeline.test_ElasticSearchProvider
CommonHelper.networkConfig = {}
target = ElasticSearchProvider(
"Foscam", datetime.datetime(2019, 10, 20, 17, 18, 8), True)
result = target.GetShots([])
meta = result[0].Metadata['PROV:ELSE']
self.assertEqual("Foscam@2019-10-20T15:18:08.000Z", meta['id'])
self.assertEqual("cameraarchive-2019", meta['index'])
# if __name__ == '__main__':
# unittest.main()
def __init__(self, label: str, isSimulation: bool = False):
super().__init__("PHOT", isSimulation)
self.label = label
self.helper = CommonHelper()
class MediaInfo:
props = [
'datetime_meta_create', 'datetime_meta_modif', 'datetime_name',
'datetime_file_create', 'datetime_file_modif'
]
def __init__(self, filename=None):
if not filename:
return
self.filename = filename
self.helper = CommonHelper()
self.log = logging.getLogger(f"META")
self.log.info(f'========== {filename} ==========')
self.ProcessFileName()
if self.mediatype == 'image':
self.ProcessImageMeta()
elif self.mediatype == 'video':
self.ProcessVideoMeta()
self.ProcessFileAttributes()
def ProcessFileAttributes(self):
stat = os.stat(self.filename)
self.datetime_file_create = datetime.fromtimestamp(stat.st_ctime)
self.datetime_file_modif = datetime.fromtimestamp(stat.st_mtime)
def ProcessVideoMeta(self):
self.datetime_meta_create = None
self.datetime_meta_modif = None
parser = createParser(self.filename)
if not parser:
self.log.warning(f"Can't parse file: {self.filename}")
with parser:
try:
metadata = extractMetadata(parser)
except Exception as err:
self.log.error("Metadata extraction error: %s" % err)
metadata = None
if not metadata:
self.log.error("Unable to extract metadata")
else:
dic = metadata.exportDictionary()
# pprint(dic)
self.datetime_meta_create = self.helper.get_datetime(
dic['Metadata']['Creation date'])
self.datetime_meta_modif = self.helper.get_datetime(
dic['Metadata']['Last modification'])
def ProcessImageMeta(self):
self.datetime_meta_create = None
self.datetime_meta_modif = None
imgFile = PIL.Image.open(self.filename)
try:
exifDict = imgFile._getexif()
except AttributeError as err:
self.log.warning(f'Error during extract exif: {err}')
exifDict = None
if exifDict:
exif = {
PIL.ExifTags.TAGS[k]: v
for k, v in exifDict.items() if k in PIL.ExifTags.TAGS
}
# pprint(self.filename)
# pprint(exif['DateTimeDigitized'])
self.datetime_meta_create = self.helper.get_datetime(
exif['DateTimeDigitized'])
else:
self.log.warning(f"No exif data in {self.filename}")
def ProcessFileName(self):
self.datetime_name = self.helper.get_datetime(self.filename, False)
# if dt != None:
# self.log.debug(f"{dt:%Y-%m-%d %H:%M:%S} File: {f}")
kind = filetype.guess(self.filename)
if kind is None:
self.log.error(f'Cannot guess file type: {self.filename}')
else:
self.mediatype = kind.mime.split('/')[0]
self.mediaextension = kind.mime.split('/')[1]
def __str__(self):
dic = MediaInfo()
dtShot = self.GetShotTime()
result = '\n'
for k in MediaInfo.props:
if k not in self.__dict__:
continue
v = self.__dict__[k]
if not v:
continue
dtStr = self.helper.ToTimeStampStr(v)
dic.__setattr__(k, dtStr)
result += f' -- {k: <25}: {dtStr} {"*" if v == dtShot else ""}\n'
result += f' : {"Shot time": <25}: {dtShot}\n'
return result # str(vars(dic))
def GetShotTime(self):
dates = [
self.datetime_meta_create, self.datetime_meta_modif,
self.datetime_name, self.datetime_file_create,
self.datetime_file_modif,
datetime.now()
]
return query(dates) \
.where(lambda dt: dt != None) \
.min()
def __init__(self, root: str = None, ignoreDirs=[], condition=None):
super().__init__("WALK")
self.helper = CommonHelper()
self.root = root
self.ignoreDirs = ignoreDirs
self.condition = condition if condition != None else (lambda f: True)
def __init__(self, pShot: PipelineShot, yolo: YoloContext):
self.pShot = pShot
self.log = logging.getLogger(f"PROC:YOLO")
self.boxes = []
self.yolo = yolo
self.helper = CommonHelper()
class ApiContext:
# class ApiContextInstance:
# def __init__(self, arg):
# self.val = arg
# Instance = None
IsInitialized = False
Log = None
Helper = CommonHelper()
CameraArchive: CameraArchiveHelper
ElasticSearch: ElasticSearchHelper
ShotsPipeline: ShotsPipeline
def __init__(self, arg):
if not ApiContext.IsInitialized:
self.InitPrivate()
ApiContext.IsInitialized = True
# if not ApiContext.Instance:
# ApiContext.Instance = ApiContext.ApiContextInstance("TODO")
# else:
# ApiContext.Instance.val = arg
def InitPrivate(self):
file_error_handler = logging.FileHandler(
filename='logs/camera-tools-error.log', encoding='utf-8')
file_error_handler.setLevel(logging.ERROR)
file_handler = logging.handlers.TimedRotatingFileHandler(
'logs/camera-tools.log',
when='midnight',
backupCount=7,
encoding='utf-8')
file_handler.suffix = '_%Y-%m-%d.log'
# html_handler = HtmlLogger.HTMLRotatingFileHandler('camera-tools.html')
# html_handler.suffix = '_%Y-%m-%d_%H.html'
# html_handler.setFormatter(HtmlLogger.HTMLFormatter())
locale.getpreferredencoding() #need for display emoji in terminal
stdout_handler = logging.StreamHandler(sys.stdout)
handlers = [file_handler, stdout_handler, file_error_handler]
logging.basicConfig(
format=
'%(asctime)s|%(levelname)-.3s|%(name)s: %(message)s', # \t####=> %(filename)s:%(lineno)d
level=logging.DEBUG,
datefmt='%H:%M:%S',
handlers=handlers)
self.log = logging.getLogger("API")
ApiContext.Log = self.log
self.log.info(
'ℹ️ |############################################################|'
)
self.log.info(
f'ℹ️ |####### start API @ {str(datetime.datetime.now()) + " ":#<40}|'
)
self.log.info(
'ℹ️ |############################################################|'
)
self.log.info("USED_SETTINGS: " + AppSettings.USED_SETTINGS)
self.Helper.installColoredLog(self.log)
# Some examples.
# self.log.debug("this is a debugging message")
# self.log.info("this is an informational message")
# self.log.warning("this is a warning message")
# self.log.error("this is an error message")
# self.log.critical("this is a critical message")
self.isSimulation = False
self.secretConfig = SecretConfig()
self.secretConfig.fromJsonFile()
ApiContext.CameraArchive = CameraArchiveHelper(self.log)
ApiContext.ElasticSearch = ElasticSearchHelper()
self.camera = 'Foscam'
self.shotsPipeline = ShotsPipeline(self.camera, self.log)
self.InitShotsPipeline()
ApiContext.ShotsPipeline = self.shotsPipeline
self.InitDnsPipeline()
ApiContext.DnsPipeline = self.dnsPipeline
self.InitPhotosPipeline()
ApiContext.PhotosSergeyPipeline = self.photosSergeyPipeline
ApiContext.PhotosLiliaPipeline = self.photosLiliaPipeline
self.log.info(
f'initialization API finished @ {datetime.datetime.now()}')
def InitPhotosPipeline(self):
self.photosSergeyPipeline = Pipeline()
self.photosSergeyPipeline.providers.append(
FilesWalkerProvider("../camera-OpenCV-data/Mobile", ['Lilia']))
self.photosSergeyPipeline.processors.append(
MediaCreationDateProcessor())
self.photosSergeyPipeline.processors.append(
PhotosArrangeProcessor('Mobile Sergey'))
self.photosLiliaPipeline = Pipeline()
self.photosLiliaPipeline.providers.append(
FilesWalkerProvider("../camera-OpenCV-data/Mobile/Lilia"))
self.photosLiliaPipeline.processors.append(
MediaCreationDateProcessor())
self.photosLiliaPipeline.processors.append(
PhotosArrangeProcessor('Mobile Lilia'))
def InitShotsPipeline(self):
self.shotsPipeline.providers.clear()
self.shotsPipeline.processors.clear()
self.shotsPipeline.providers.append(ImapShotsProvider())
self.shotsPipeline.providers.append(DirectoryShotsProvider())
self.shotsPipeline.processors.append(DiffContoursProcessor())
self.shotsPipeline.processors.append(YoloObjDetectionProcessor())
self.shotsPipeline.processors.append(TrackingProcessor())
self.shotsPipeline.processors.append(SaveToTempProcessor())
self.shotsPipeline.processors.append(MailSenderProcessor())
self.shotsPipeline.processors.append(
HassioProcessor('temp' if self.isSimulation else None))
self.shotsPipeline.processors.append(
ArchiveProcessor(self.isSimulation))
self.shotsPipeline.processors.append(
ElasticSearchProcessor(self.isSimulation))
self.shotsPipeline.PreLoad()
def InitDnsPipeline(self):
self.dnsPipeline = Pipeline(self.log)
self.dnsPipeline.providers.append(DnsAdGuardProvider())
self.dnsPipeline.processors.append(ElasticSearchDnsProcessor())
def __init__(self, folder: str = None):
super().__init__("DIRC")
self.helper = CommonHelper()
self.folder = folder
self.SourceImagePattern = re.compile(AppSettings.SOURCE_IMAGE_PATTARN)
def __init__(self, *args, **kwargs):
super(TestArchPhotos, self).__init__(*args, **kwargs)
self.testHelper = TestHelper()
self.helper = CommonHelper()
self.log = self.testHelper.CreateLog(TestArchPhotos.__name__)