def kalman(A, B, C, Q, R):
"""Solves for the steady state kalman gain and covariance matricies.
Args:
A, B, C: SS matricies.
Q: The model uncertantity
R: The measurement uncertainty
Returns:
KalmanGain, Covariance.
"""
I = numpy.matrix(numpy.eye(Q.shape[0]))
Z = numpy.matrix(numpy.zeros(Q.shape[0]))
n = A.shape[0]
m = C.shape[0]
controllability_rank = numpy.linalg.matrix_rank(ctrb(A.T, C.T))
if controllability_rank != n:
glog.warning('Observability of %d != %d, unobservable state',
controllability_rank, n)
# Compute the steady state covariance matrix.
P_prior, rcond, w, S, T = slycot.sb02od(n=n, m=m, A=A.T, B=C.T, Q=Q, R=R, dico='D')
S = C * P_prior * C.T + R
K = numpy.linalg.lstsq(S.T, (P_prior * C.T).T)[0].T
P = (I - K * C) * P_prior
return K, P
def reg(id, func):
global MSG_MAP
if id in MSG_MAP:
glog.warning("gnet>id already exist|id:%d func:%s" % (id, repr(func)))
return False
MSG_MAP[id] = func
return True
def prepare_frames(split_name):
"""
extract fraems for dataset `split_name`.
pickle to disk.
frames_info[samp_id]
|- image: image path
|- trans: word label (without space)
`- frames: frame images filename list
e.g.
frames_info[train_0]: {
'frames': ['0.jpg', '1.jpg', '2.jpg', '3.jpg', '4.jpg', ...],
'image': '/home/paile/research/mjsynth/mnt/ramdisk/max/90kDICT32px/378/5/54_amethyst_2482.jpg',
'trans': 'amethyst'
}
"""
print 'processing:', split_name, '-------------------------'
# read split info
split = config.splits[split_name]
trans_path = split.trans_pth
frames_dir = split.frames_dir
if path.isdir(frames_dir):
glog.warning('%s already exists.', frames_dir)
sys.exit(-1)
print 'trans_path:', trans_path
print 'frames_dir:', frames_dir
# read trans
all_trans = utils.read_table(trans_path)
# assign sample id
trans_ids = ['%s_%d' % (split_name, i) for i in range(len(all_trans))]
all_trans = OrderedDict(zip(trans_ids, all_trans))
# frames info
frames_info = OrderedDict()
err_images = []
for samp_id in all_trans:
try:
image_pth = all_trans[samp_id][0]
im = load_image(image_pth)
frame_pths = extract_frames(im, samp_id, frames_dir)
frames_info[samp_id] = {}
frames_info[samp_id]['image'] = image_pth
frames_info[samp_id]['trans'] = all_trans[samp_id][1]
frames_info[samp_id]['frames'] = frame_pths
except Exception, e:
err_images.append(image_pth+'\n')
print 'Error:', image_pth
def nxCfg2Acfg(outputDir, malwareDirPrefix):
"""Path name format: class/graphId/pkl_name"""
logGraphPaths = open(outputDir + '/graph_pathnames.csv', 'w')
graphSizes = {x: [] for x in malwareNames}
realPklPath = []
for malwareDirname in glob.glob(malwareDirPrefix + '/*'):
log.debug('Enter malware directory %s' % malwareDirname)
malwareName = malwareDirname.split('/')[-1]
if malwareName not in malwareNames:
log.warning('%s not in known malware types' % malwareName)
continue
log.info("Search %s gpickles in %s" % (malwareName, malwareDirname))
pklPaths = glob.glob(malwareDirname + '/*')
if len(pklPaths) == 0:
log.warning('Pickle path %s is empty' % pklPaths)
for pklPath in pklPaths:
if pklPath[-8:] != '.gpickle':
log.warning('Ignore %s since it\'s not gpickle file' % pklPath)
continue
log.debug('Loading nx.Graph from %s' % pklPath)
logGraphPaths.write(pklPath + '\n')
try:
G = nx.read_gpickle(pklPath)
except UnicodeDecodeError as e:
log.error('Decode failed for %s: %s' % (pklPath, e))
realPklPath.append(pklPath)
graphSizes[malwareName].append(number_of_nodes(G))
log.info("Found %s gpickles" % len(realPklPath))
graphSizesDf = pd.DataFrame.from_dict(graphSizes, orient='index').T
graphSizesDf.to_csv(outputDir + '/graph_sizes.csv', index=False)
return realPklPath
def mkdir(path_to_dir):
if not os.path.exists(path_to_dir):
os.mkdir(path_to_dir)
else:
glog.warning('Dir {0} already exist.'.format(path_to_dir))
# 2. Loop for num_iterations:
# a. Forward propagation
# b. Compute cost function
# c. Backward propagation
# d. Update parameters (using parameters, and grads from backprop)
# 4. Use trained parameters to predict labels
log.info("loading 'cats' parameters on {0}".format(param_path))
parameters = np.load(param_path).item()
log.info("loaded 'cats' parameters on {0}".format(param_path))
log.info("starting worker on {0}".format(EP))
while True:
ITEM = fetch_item(EP)
if ITEM['error'] not in ['', u'']:
log.warning(ITEM['error'])
time.sleep(5)
continue
if ITEM['bucket'] == '/cats-request':
IMAGE_PATH = ITEM['value']
if not os.path.exists(IMAGE_PATH):
log.warning('cannot find image {0}'.format(IMAGE_PATH))
ITEM['progress'] = 100
ITEM['error'] = 'cannot find image {0}'.format(IMAGE_PATH)
else:
img_class = classify(IMAGE_PATH, parameters)
ITEM['progress'] = 100
ITEM['value'] = "[WORKER - ACK] it's a '{0}'!".format(img_class)
POST_RESPONSE = post_item(EP, ITEM)
classifier = classifier.cuda()
classifier.eval()
startTime = time.process_time()
testPredProb = predictDataset(testGraphs, classifier)
log.info(f'Net test time = {time.process_time() - startTime} seconds')
exportPredictions(testGraphs, testPredProb)
if __name__ == '__main__':
log.setLevel("INFO")
random.seed(cmd_args.seed)
np.random.seed(cmd_args.seed)
torch.manual_seed(cmd_args.seed)
if cmd_args.data == 'YANACFG':
log.warning(f'No testset for YANACFG data')
else:
startTime = time.process_time()
testGraphs = loadGraphsMayCache(cmd_args.test_dir, isTestSet=True)
normalizeFeatures(testGraphs,
isTestSet=True,
operation=cmd_args.norm_op)
dataReadyTime = time.process_time() - startTime
log.info('Testset ready takes %.2fs' % dataReadyTime)
trainGraphs = loadGraphsMayCache(cmd_args.train_dir, isTestSet=False)
trainGraphs = filterOutNoEdgeGraphs(trainGraphs)
decideHyperparameters(trainGraphs)
testWithModel(testGraphs)
def test_warning(self):
log.warning('test')
print('%d docs in index' % writer.numDocs())
print('Indexing json files...')
# For text field.
t1 = FieldType()
t1.setStored(False)
t1.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
with codecs.open(json_file, encoding='utf8') as f:
for line in tqdm(f):
line = line.strip()
try:
json_doc = json.loads(line)
except:
glog.warning('Error json parsing: {}'.format(line))
continue
# Delete existing abstracts. This is useful when adding
# update files from Medline.
try:
assert 'pmid' in json_doc
pmid_query = TermQuery(Term('pmid', json_doc['pmid']))
pmcid_query = TermQuery(Term('pmcid', json_doc['pmcid']))
id_query = IntPoint.newRangeQuery("id", json_doc['id'],
json_doc['id'])
bq = BooleanQuery.Builder()
bq.add(pmid_query, BooleanClause.Occur.MUST)
bq.add(pmcid_query, BooleanClause.Occur.MUST)
bq.add(id_query, BooleanClause.Occur.MUST)
q = bq.build()
def test_warning(self):
log.warning('test')
return
def insert_categories_data(xml_root, use_background_label, result_annotation,
xml_dir):
"""Get labels from input annotation and fill categories field in output annotation
Args:
xml_root: root for xml parser
use_background_label: key to enable using label background
result_annotation: output annotation in COCO representation
xml_dir: directory with input annotation
"""
log.info('Reading labels...')
categories = []
category_map = {}
bg_found = False
id = 0
for label in xml_root.iter('label'):
for name in label.findall("./name"):
if not use_background_label and name.text == 'background':
bg_found = True
continue
category_map[name.text] = id
categories.append({
'id': id,
'name': name.text,
'supercategory': ''
})
id += 1
if len(categories) == 0:
log.info(
'Labels in annotation were not found. Trying to find file <labels.txt> in <{}>'
.format(xml_dir))
if osp.isfile(osp.join(xml_dir, 'labels.txt')):
labels_file = osp.join(xml_dir, 'labels.txt')
log.info('File <labels.txt> was found in <{}>. Reading...'.format(
xml_dir))
with open(labels_file, 'r') as file:
string = ' '
id = 0
while string != '' and string != '\n':
string = file.readline()
labels = string.split(' ')
for l in labels:
if l == '\n':
continue
if not use_background_label and l == 'background':
bg_found = True
continue
category_map[l] = id
categories.append({
'id': id,
'name': l,
'supercategory': ''
})
id += 1
result_annotation['categories'] = categories
log.info('Found the next labels: {}'.format(category_map))
if bg_found:
log.warning(
'Label <background> was found but not used. '
'To enable it should use command line argument [--use_background_label]'
)
return category_map
def process_properties(self,
dump_video=True,
dump_images=False,
tstamps_of_interest=None):
assert (isinstance(self.response, Response))
self.med_ret = MediaRetriever(self.response.url)
dump_video = (self.request.get("dump_video") or dump_video) and \
self.med_ret.is_video
dump_images = (self.request.get("dump_images") or dump_images) or \
self.med_ret.is_image
if dump_images is False and self.med_ret.is_image:
dump_images = True
if dump_video is False and dump_images is False:
log.warning("`dump_video` and `dump_images` are both false."
" Unable to proceed.")
return
log.debug(f"Dumping Video: {dump_video}")
log.debug(f"Dumping Frames: {dump_images}")
# we get the frame iterator
frames_iterator = []
if tstamps_of_interest is not None:
if type(tstamps_of_interest) is list:
for t in tstamps_of_interest:
frame = self.med_ret.get_frame(tstamp=t)
frames_iterator.append((frame, t))
else:
try:
frames_iterator = self.med_ret.get_frames_iterator(
sample_rate=1.0)
except Exception:
log.error(traceback.format_exc())
raise Exception("Error loading media")
vid_file, images_dir, max_tstamp = self.dump_data(
frames_iterator, dump_video=dump_video, dump_images=dump_images)
props = []
if self.local_dir is not None and dump_video:
local_vid_path = self.copy_video(vid_file.name)
p = Property(
server=self.name,
ver=self.version,
value=local_vid_path,
property_type="dumped_video",
property_id=4,
)
props.append(p)
if self.local_dir is not None and dump_images:
local_frames_paths = self.copy_frames(images_dir.name)
ps = [
Property(
server=self.name,
ver=self.version,
value=path,
property_type="dumped_image",
property_id=3,
) for path in local_frames_paths
]
props.extend(ps)
if self.s3_bucket is not None and dump_video:
s3_vid_url = self.upload_video(vid_file.name)
p = Property(
server=self.name,
ver=self.version,
value=s3_vid_url,
property_type="dumped_video",
property_id=2,
)
props.append(p)
if self.s3_bucket is not None and dump_images:
s3_frames_urls = self.upload_frames(images_dir.name)
ps = [
Property(
server=self.name,
ver=self.version,
value=url,
property_type="dumped_image",
property_id=1,
) for url in s3_frames_urls
]
props.extend(ps)
images_dir.cleanup()
vid_file.close()
media_summary = VideoAnn(t1=0.0, t2=max_tstamp, props=props)
self.response.append_media_summary(media_summary)
def publish_intermediates(*, args, ro_client, rw_client):
local_intermediates = {}
remote_intermediates = {}
remote_error_records = []
run_identifiers = workflow.get_run_identifiers(args.filter_bucket)
if not run_identifiers:
log.warning("No run identifiers found")
return
run_id = run_identifiers[-1]
run_id_path = args.download_path / Path(run_id)
run_id_path.mkdir(parents=True, exist_ok=True)
intermediates_path = run_id_path / Path("enrolled.json")
workflow.download_and_retry_from_google_cloud(
args.filter_bucket,
f"{run_id}/enrolled.json",
intermediates_path,
timeout=timedelta(minutes=5),
)
with intermediates_path.open("r") as f:
for entry in json.load(f):
try:
intObj = Intermediate(**entry, debug=args.debug)
if intObj.unique_id() in local_intermediates:
log.warning(
f"[{intObj.unique_id()}] Local collision: {intObj} with "
+ f"{local_intermediates[intObj.unique_id()]}"
)
continue
local_intermediates[intObj.unique_id()] = intObj
except Exception as e:
log.error("Error importing file from {}: {}".format(args.inpath, e))
log.error("Record: {}".format(entry))
raise e
for record in ro_client.get_records(
collection=settings.KINTO_INTERMEDIATES_COLLECTION
):
try:
intObj = Intermediate(**record)
remote_intermediates[intObj.unique_id()] = intObj
except IntermediateRecordError as ire:
log.warning("Skipping broken intermediate record at Kinto: {}".format(ire))
remote_error_records.append(record)
except KeyError as ke:
log.error("Critical error importing Kinto dataset: {}".format(ke))
log.error("Record: {}".format(record))
raise ke
to_delete = set(remote_intermediates.keys()) - set(local_intermediates.keys())
to_upload = set(local_intermediates.keys()) - set(remote_intermediates.keys())
to_update = set()
for i in set(local_intermediates.keys()) & set(remote_intermediates.keys()):
if not local_intermediates[i].equals(remote_record=remote_intermediates[i]):
to_update.add(i)
expired = set()
for i in to_delete:
try:
if remote_intermediates[i].is_expired():
expired.add(i)
except Exception as e:
log.warning(f"Failed to track expiration for {i}: {e}")
to_delete_not_expired = to_delete - expired
delete_pubkeys = {remote_intermediates[i].pubKeyHash for i in to_delete}
upload_pubkeys = {local_intermediates[i].pubKeyHash for i in to_upload}
unenrollments = set()
new_enrollments = set()
update_other_than_enrollment = set()
for i in to_update:
if (
local_intermediates[i].crlite_enrolled
and not remote_intermediates[i].crlite_enrolled
):
new_enrollments.add(i)
elif (
remote_intermediates[i].crlite_enrolled
and not local_intermediates[i].crlite_enrolled
):
unenrollments.add(i)
else:
update_other_than_enrollment.add(i)
log.info(f"Total entries before update: {len(remote_intermediates)}")
log.info(f"To delete: {len(to_delete)} (Deletion enabled: {args.delete})")
log.info(f"- Expired: {len(expired)}")
log.info(f"To add: {len(to_upload)}")
log.info(
f"Certificates updated (without a key change): {len(delete_pubkeys & upload_pubkeys)}"
)
log.info(f"Remote records in an error state: {len(remote_error_records)}")
log.info(f"Total entries updated: {len(to_update)}")
log.info(f"- New enrollments: {len(new_enrollments)}")
log.info(f"- Unenrollments: {len(unenrollments)}")
log.info(f"- Other: {len(update_other_than_enrollment)}")
log.info(f"Total entries after update: {len(local_intermediates)}")
if args.export:
with open(Path(args.export) / Path("to_delete"), "w") as df:
export_intermediates(df, to_delete, remote_intermediates)
with open(Path(args.export) / Path("to_delete_not_expired"), "w") as df:
export_intermediates(df, to_delete_not_expired, remote_intermediates)
with open(Path(args.export) / Path("expired"), "w") as df:
export_intermediates(df, expired, remote_intermediates)
with open(Path(args.export) / Path("to_upload"), "w") as df:
export_intermediates(df, to_upload, local_intermediates)
with open(Path(args.export) / Path("to_update"), "w") as df:
export_intermediates(
df, to_update, local_intermediates, old=remote_intermediates
)
with open(Path(args.export) / Path("unenrollments"), "w") as df:
export_intermediates(
df, unenrollments, local_intermediates, old=remote_intermediates
)
with open(Path(args.export) / Path("new_enrollments"), "w") as df:
export_intermediates(
df, new_enrollments, local_intermediates, old=remote_intermediates
)
with open(Path(args.export) / Path("update_other_than_enrollment"), "w") as df:
export_intermediates(
df,
update_other_than_enrollment,
local_intermediates,
old=remote_intermediates,
)
if args.debug:
print("Variables available:")
print(" local_intermediates")
print(" remote_intermediates")
print(" remote_error_records")
print("")
print(" to_upload")
print(" to_delete")
print(" to_update")
print("")
print(" new_enrollments")
print(" unenrollments")
print("")
print(" delete_pubkeys")
print(" upload_pubkeys")
print(
" delete_pubkeys & upload_pubkeys # certs updated without changing the key"
)
print("")
print(" local_intermediates[to_update.pop()].cert # get cert object")
print("")
print("Use 'continue' to proceed")
print("")
breakpoint()
if args.noop:
log.info(f"Noop flag set, exiting before any intermediate updates")
return
# Don't accidentally use the ro_client beyond this point
ro_client = None
if len(remote_error_records) > 0:
log.info(f"Cleaning {len(remote_error_records)} broken records")
for record in remote_error_records:
try:
rw_client.delete_record(
collection=settings.KINTO_INTERMEDIATES_COLLECTION,
id=record["id"],
)
except KintoException as ke:
log.warning(f"Couldn't delete record id {record['id']}: {ke}")
for unique_id in to_delete:
intermediate = remote_intermediates[unique_id]
log.info(f"Deleting {intermediate} from Kinto (delete={args.delete})")
if args.delete:
try:
intermediate.delete_from_kinto(rw_client=rw_client)
except KintoException as ke:
log.warning(f"Couldn't delete record id {intermediate}: {ke}")
for unique_id in to_upload:
intermediate = local_intermediates[unique_id]
log.debug(f"Uploading {intermediate} to Kinto")
intermediate.add_to_kinto(rw_client=rw_client)
update_error_records = []
for unique_id in to_update:
local_int = local_intermediates[unique_id]
remote_int = remote_intermediates[unique_id]
if not local_int.equals(remote_record=remote_int):
try:
local_int.update_kinto(
rw_client=rw_client,
remote_record=remote_int,
)
except KintoException as ke:
update_error_records.append((local_int, remote_int, ke))
for (local_int, remote_int, ex) in update_error_records:
log.warning(
f"Failed to update local={local_int} remote={remote_int} exception={ex}"
)
log.info("Verifying correctness...")
verified_intermediates = {}
verification_error_records = []
for record in rw_client.get_records(
collection=settings.KINTO_INTERMEDIATES_COLLECTION
):
try:
intObj = Intermediate(**record)
verified_intermediates[intObj.unique_id()] = intObj
except IntermediateRecordError as ire:
log.warning(
"Verification found broken intermediate record at Kinto: {}".format(ire)
)
verification_error_records.append(record)
except KeyError as ke:
log.error("Critical error importing Kinto dataset: {}".format(ke))
log.error("Record: {}".format(record))
raise ke
if len(verification_error_records) > 0:
raise KintoException(
"There were {} broken intermediates. Re-run to fix.".format(
len(verification_error_records)
)
)
log.info(
"{} intermediates locally, {} at Kinto.".format(
len(local_intermediates), len(verified_intermediates)
)
)
if args.delete and set(local_intermediates.keys()) != set(
verified_intermediates.keys()
):
log.error("The verified intermediates do not match the local set. Differences:")
missing_remote = set(local_intermediates.keys()) - set(
verified_intermediates.keys()
)
missing_local = set(verified_intermediates.keys()) - set(
local_intermediates.keys()
)
for d in missing_remote:
log.error("{} does not exist at Kinto".format(d))
for d in missing_local:
log.error(
"{} exists at Kinto but should have been deleted (not in local set)".format(
d
)
)
raise KintoException("Local/Remote Verification Failed")
elif not args.delete and set(local_intermediates.keys()) > set(
verified_intermediates.keys()
):
log.error("The verified intermediates do not match the local set. Differences:")
missing_remote = set(local_intermediates.keys()) - set(
verified_intermediates.keys()
)
for d in missing_remote:
log.error("{} does not exist at Kinto".format(d))
raise KintoException("Local/Remote Verification Failed")
for unique_id in verified_intermediates.keys():
remote_int = verified_intermediates[unique_id]
if unique_id not in local_intermediates and not args.delete:
log.info(
"Remote {} has been deleted locally, but ignoring.".format(remote_int)
)
continue
local_int = local_intermediates[unique_id]
if not local_int.equals(remote_record=remote_int):
if not remote_int.pemAttachment.verify(pemData=local_int.pemData):
log.warning(
"PEM hash mismatch for {}; remote={} != local={}".format(
unique_id, remote_int, local_int
)
)
raise KintoException(
"Local/Remote PEM mismatch for uniqueId={}".format(unique_id)
)
else:
log.warning(
f"Local/Remote metadata mismatch, uniqueID={unique_id}, "
+ f"local={local_int.details()}, remote={remote_int.details()}"
)
raise KintoException(
"Local/Remote metadata mismatch for uniqueId={}".format(unique_id)
)
if to_update or to_upload or to_delete and not args.noop and args.request_review:
log.info(
f"Set for review, {len(to_update)} updates, {len(to_upload)} uploads, "
+ f"{len(to_delete)} deletions."
)
rw_client.request_review_of_collection(
collection=settings.KINTO_INTERMEDIATES_COLLECTION,
)
else:
log.info(f"No updates to do")
def process_properties(self,
dump_video=True,
dump_images=False,
tstamps_Of_Interest=None):
self.last_tstamp = 0.0
assert (self.response)
self.med_ret = MediaRetriever(self.response.url)
self.w, self.h = self.med_ret.get_w_h()
media_id = os.path.basename(self.response.url).rsplit(".", 1)[0]
self.media_id = "".join(
[e for e in media_id if e.isalnum() or e in ["/", "."]])
self.content_type_map = {}
# if there is no flag in the request of not request_api we'll get None.
self.dump_video = None
self.dump_images = None
try:
self.dump_video = self.request.get("dump_video")
self.dump_images = self.request.get("dump_images")
except Exception:
log.error(
"Unable to get flags from request dump_video or dump_images")
pass
if self.dump_video is None:
self.dump_video = dump_video
if self.dump_images is None:
self.dump_images = dump_images
if self.dump_video is False and self.dump_images is False:
log.warning(
"Not dumping anything--you might want to dump something.")
return
dump_folder = self.pushing_folder + '/' + self.media_id + '/'
self.dumping_folder_url = dump_folder
if dump_folder:
if not os.path.exists(dump_folder):
os.makedirs(dump_folder)
if self.dump_video:
filename = dump_folder + '/video.mp4'
fps = 1
frameSize = self.med_ret.shape
frameSize = (self.w, self.h)
fourcc = cv2.VideoWriter_fourcc(*'H264')
log.info("filename: " + filename)
log.info("fourcc: " + str(fourcc))
log.info("type(fourcc): " + str(type(fourcc)))
log.info("fps: " + str(fps))
log.info("type(fps): " + str(type(fps)))
log.info("frameSize: " + str(frameSize))
log.info("type(frameSize): " + str(type(frameSize)))
vid = cv2.VideoWriter(filename, fourcc, fps, frameSize)
self.content_type_map[os.path.basename(filename)] = 'video/mp4'
face = cv2.FONT_HERSHEY_SIMPLEX
scale = 0.65
thickness = 2
# we get the image_annotation tstamps
tstamps = self.response.get_timestamps()
tstamp_frame_anns = self.response.get_timestamps_from_frames_ann()
log.debug('tstamps: ' + str(tstamps))
log.debug('tstamps_dets: ' + str(tstamp_frame_anns))
# we get the frame iterator
frames_iterator = []
if tstamps_Of_Interest:
if type(tstamps_Of_Interest) is list:
for t in tstamps_Of_Interest:
frame = self.med_ret.get_frame(tstamp=t)
frames_iterator.append((frame, t))
elif tstamps_Of_Interest is None:
try:
frames_iterator = self.med_ret.get_frames_iterator(
sample_rate=1.0)
except Exception:
log.error(traceback.format_exc())
raise Exception("Error loading media")
for i, (img, tstamp) in enumerate(frames_iterator):
self.last_tstamp = tstamp
if img is None:
log.warning("Invalid frame")
continue
if tstamp is None:
log.warning("Invalid tstamp")
continue
# log.info('tstamp: ' + str(tstamp))
if tstamp in tstamp_frame_anns:
log.debug("drawing frame for tstamp: " + str(tstamp))
# we get image_ann for that time_stamps
regions = self.response.get_regions_from_tstamp(tstamp)
# log.info(json.dumps(image_ann, indent=2))
for region in regions:
rand_color = get_rand_bgr()
p0, p1 = p0p1_from_bbox_contour(region['contour'], self.w,
self.h)
anchor_point = [p0[0] + 3, p1[1] - 3]
if abs(p1[1] - self.h) < 30:
anchor_point = [p0[0] + 3, int(p1[1] / 2) - 3]
img = cv2.rectangle(img, p0, p1, rand_color, thickness)
prop_strs = get_props_from_region(region)
for i, prop in enumerate(prop_strs):
img = cv2.putText(
img, prop,
(anchor_point[0], anchor_point[1] + i * 25), face,
1.0, rand_color, thickness)
elif tstamp in tstamps:
log.debug("Making frame gray")
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
img = cv2.cvtColor(gray, cv2.COLOR_GRAY2BGR)
else:
log.debug("No processed frame")
continue
# Include the timestamp
img = cv2.putText(img, str(tstamp), (20, 20), face, scale,
[255, 255, 255], thickness)
if self.dump_video:
# we add the frame
log.debug("Adding frame")
vid.write(img)
if self.dump_images:
# we dump the frame
outfn = "{}/{}.jpg".format(dump_folder, tstamp)
log.debug("Writing to file: {}".format(outfn))
cv2.imwrite(outfn, img)
self.content_type_map[os.path.basename(outfn)] = 'image/jpeg'
if self.dump_video:
vid.release()
if self.s3_bucket:
try:
self.upload_files(dump_folder)
except Exception:
log.error(traceback.format_exc())
if self.pushing_folder == DEFAULT_DUMP_FOLDER:
log.info('Removing files in ' + dump_folder)
shutil.rmtree(dump_folder)
props = []
if self.dump_images:
props.append(
Property(
server=self.name,
ver=self.version,
value=self.dumping_folder_url,
property_type="dumped_images",
property_id=1,
))
if self.dump_video:
dumped_video_url = self.dumping_folder_url + '/video.mp4'
dumped_video_url = dumped_video_url.replace('//', '/')
dumped_video_url = dumped_video_url.replace('https:/', 'https://')
props.append(
Property(
server=self.name,
ver=self.version,
value=dumped_video_url,
property_type="dumped_video",
property_id=2,
))
media_summary = VideoAnn(t1=0.0, t2=self.last_tstamp, props=props)
self.response.append_media_summary(media_summary)
def run(args):
# Get experiment information from yaml file.
experiment_params = yaml.load(args.experiments_path)
regression_tests_dir = os.path.expandvars(
experiment_params['regression_tests_dir'])
params_dir = os.path.expandvars(experiment_params['params_dir'])
dataset_dir = os.path.expandvars(experiment_params['dataset_dir'])
executable_path = os.path.expandvars(experiment_params['executable_path'])
datasets_to_run = experiment_params['datasets_to_run']
regression_params = experiment_params['regression_parameters']
# Build dictionary from parameter name to list of parameter values
param_name_to_values = dict()
for regression_param in regression_params:
param_name_to_values[
regression_param['name']] = regression_param['values']
log.info("Setup regression tests.")
check_and_create_regression_test_structure(regression_tests_dir,
params_dir,
param_name_to_values,
dataset_dir)
# Run experiments.
log.info("Run regression tests.")
stats = dict()
for regression_param in tqdm(regression_params):
# Redirect to param_name_value dir param_name = regression_param['name']
param_name = regression_param['name']
stats[param_name] = dict()
log.info("Run for regression parameter: %s" % param_name)
for param_value in tqdm(regression_param['values'],
desc=regression_param['name'],
leave=False):
results_dir = os.path.join(regression_tests_dir, param_name,
str(param_value))
# Redirect to modified params_dir
params_dir = os.path.join(results_dir, 'params')
stats[param_name][param_value] = dict()
for dataset in datasets_to_run:
dataset_name = dataset['name']
log.info("\033[93mRun: %s = %f in dataset %s\033[00m" %
(param_name, param_value, dataset_name))
pipelines_to_run = dataset['pipelines']
if not run_dataset(results_dir,
params_dir,
dataset_dir,
dataset,
executable_path,
args.run_pipeline,
args.analyse_vio,
args.plot,
args.save_results,
args.save_plots,
args.save_boxplots,
pipelines_to_run,
dataset['initial_frame'],
dataset['final_frame'],
dataset['discard_n_start_poses'],
dataset['discard_n_end_poses'],
extra_flagfile_path='override.flags'):
log.warning("A pipeline run has failed...")
stats[param_name][param_value][dataset_name] = dict()
for pipeline in pipelines_to_run:
results_file = os.path.join(results_dir, dataset_name,
pipeline, "results.yaml")
if os.path.isfile(results_file):
stats[param_name][param_value][dataset_name][
pipeline] = yaml.load(open(results_file, 'r'))
else:
log.warning(
"Could not find results file: {}. \n Adding cross."
.format(results_file))
stats[param_name][param_value][dataset_name][
pipeline] = False
# Save all stats in regression tests root directory
# TODO(Toni) ideally save them as a pandas dataframe object! And put NaNs where there is a False
with open(os.path.join(regression_tests_dir, "all_stats.yaml"),
'w') as outfile:
outfile.write(yaml.dump(stats))
# Compile plots for all results, do that in jupyter notebook using plotly instead of here...
#plot_dir = os.path.join(regression_tests_dir, param_name, "results", dataset_name)
#max_y = -1
#draw_regression_simple_boxplot_APE(stats, plot_dir, max_y)
#log.info("Finished regression test for param_name: %s" % param_name)
return True
def run_dataset(results_dir,
params_dir,
dataset_dir,
dataset_properties,
executable_path,
run_pipeline,
analyse_vio,
plot,
save_results,
save_plots,
save_boxplots,
pipelines_to_run_list,
initial_k,
final_k,
discard_n_start_poses=0,
discard_n_end_poses=0,
extra_flagfile_path='',
verbose_sparkvio=False):
""" Evaluates pipeline using Structureless(S), Structureless(S) + Projection(P), \
and Structureless(S) + Projection(P) + Regular(R) factors \
and then compiles a list of results """
dataset_name = dataset_properties['name']
dataset_segments = dataset_properties['segments']
################### RUN PIPELINE ################################
pipeline_output_dir = os.path.join(results_dir, "tmp_output/output/")
evt.create_full_path_if_not_exists(pipeline_output_dir)
output_file = os.path.join(pipeline_output_dir, "output_posesVIO.csv")
has_a_pipeline_failed = False
if len(pipelines_to_run_list) == 0:
log.warning("Not running pipeline...")
for pipeline_type in pipelines_to_run_list:
has_a_pipeline_failed = not process_vio(
executable_path, dataset_dir, dataset_name, results_dir,
params_dir, pipeline_output_dir, pipeline_type, dataset_segments,
save_results, plot, save_plots, output_file, run_pipeline,
analyse_vio, discard_n_start_poses, discard_n_end_poses, initial_k,
final_k, extra_flagfile_path, verbose_sparkvio)
# Save boxplots
if save_boxplots:
# TODO(Toni) is this really saving the boxplots?
if not has_a_pipeline_failed:
stats = dict()
for pipeline_type in pipelines_to_run_list:
results_dataset_dir = os.path.join(results_dir, dataset_name)
results = os.path.join(results_dataset_dir, pipeline_type,
"results.yaml")
if not os.path.exists(results):
raise Exception(
"\033[91mCannot plot boxplots: missing results for %s pipeline \
and dataset: %s" %
(pipeline_type, dataset_name) + "\033[99m \n \
Expected results here: %s" % results)
try:
stats[pipeline_type] = yaml.load(open(results, 'r'),
Loader=yaml.Loader)
except yaml.YAMLError as e:
raise Exception("Error in results file: ", e)
log.info("Check stats %s in %s" % (pipeline_type, results))
check_stats(stats[pipeline_type])
log.info("Drawing boxplots.")
evt.draw_rpe_boxplots(results_dataset_dir, stats,
len(dataset_segments))
else:
log.warning(
"A pipeline run has failed... skipping boxplot drawing.")
if not has_a_pipeline_failed:
evt.print_green("All pipeline runs were successful.")
else:
log.error("A pipeline has failed!")
evt.print_green("Finished evaluation for dataset: " + dataset_name)
return not has_a_pipeline_failed
def edg_process_document(self, request):
try:
return self.stub.EdgProcessDocument(request, self.timeout_seconds)
except ExpirationError:
glog.warning('Expiration:' + '\t' +
','.join([d.doc_id for d in request.document]))
def fix_segments_intersections(polygons,
height,
width,
img_name,
use_background_label,
threshold=0.0,
ratio_tolerance=0.001):
"""Find all intersected regions and crop contour for back object by objects which
are in front of the first one. It is related to a specialty of segmentation
in CVAT annotation. Intersection is calculated via function 'iou' from cocoapi
Args:
polygons: all objects on image represented as 2D array of objects' contours
height: height of image
width: width of image
img_name: name of image file
threshold: threshold of intersection over union of two objects.
By default is set to 0 and processes any two intersected objects
ratio_tolerance: used for situation when one object is fully or almost fully
inside another one and we don't want make "hole" in one of objects
"""
converted_polygons = []
empty_polygon = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
# Convert points of polygons from string to coco's array.
# All polygons must be sorted in order from bottom to top
for polygon in polygons:
label = polygon['label']
points = polygon['points'].split(';')
new_polygon = []
for xy in points:
x = float(xy.split(',')[0])
y = float(xy.split(',')[1])
new_polygon.append(x)
new_polygon.append(y)
converted_polygons.append({'label': label, 'points': new_polygon})
for i in range(0, len(converted_polygons)):
rle_bottom = mask_util.frPyObjects([converted_polygons[i]['points']],
height, width)
segment_overlapped = False
for j in range(i + 1, len(converted_polygons)):
rle_top = mask_util.frPyObjects([converted_polygons[j]['points']],
height, width)
iou = mask_util.iou(rle_bottom, rle_top, [0, 0])
area_top = sum(mask_util.area(rle_top))
area_bottom = sum(mask_util.area(rle_bottom))
if area_bottom == 0:
continue
area_ratio = area_top / area_bottom
sum_iou = sum(iou)
# If segment is fully inside another one, save this segment as is
if area_ratio - ratio_tolerance < sum_iou[
0] < area_ratio + ratio_tolerance:
continue
# Check situation when bottom segment is fully inside top.
# It means that in annotation is mistake. Save this segment as is
if 1 / area_ratio - ratio_tolerance < sum_iou[
0] < 1 / area_ratio + ratio_tolerance:
continue
if sum_iou[0] > threshold:
segment_overlapped = True
bottom_mask = np.array(mask_util.decode(rle_bottom),
dtype=np.uint8)
top_mask = np.array(mask_util.decode(rle_top), dtype=np.uint8)
bottom_mask = np.subtract(bottom_mask, top_mask)
bottom_mask[bottom_mask > 1] = 0
bottom_mask = np.sum(bottom_mask, axis=2)
bottom_mask = np.array(bottom_mask > 0, dtype=np.uint8)
converted_polygons[i]['points'] = mask_to_polygon(bottom_mask)
# If some segment is empty, do small fix to avoid error in cocoapi function
if len(converted_polygons[i]['points']) == 0:
converted_polygons[i]['points'] = [empty_polygon]
rle_bottom = mask_util.frPyObjects(
converted_polygons[i]['points'], height, width)
if not segment_overlapped:
converted_polygons[i]['points'] = [converted_polygons[i]['points']]
output_polygons = []
for i in range(0, len(converted_polygons)):
if not use_background_label and converted_polygons[i][
'label'] == 'background':
continue
poly_len = len(converted_polygons[i]['points'])
if poly_len == 0 or converted_polygons[i]['points'] == [empty_polygon]:
log.warning('Image <{}> has an empty polygon with label <{}>. '
'Perhaps there is a mistake in annotation'.format(
img_name, converted_polygons[i]['label']))
else:
output_polygons.append(converted_polygons[i])
return output_polygons
def insert_categories_data(xml_root, use_background_label, result_annotation, labels_file=None):
"""Get labels from input annotation and fill categories field in output annotation
Args:
xml_root: root for xml parser
use_background_label: key to enable using label background
result_annotation: output annotation in COCO representation
labels_file: path to file with labels names.
If not defined, parse annotation to get labels names
"""
def get_categories(names, bg_found, use_background_label, sort=False):
bg_used = False
category_map = {}
categories = []
# Sort labels by its names to make the same order of ids for different annotations
if sort:
names.sort()
# Always use id = 0 for background
if bg_found and use_background_label:
category_map['background'] = 0
bg_used = True
cat_id = 1
# Define id for all labels beginning from 1
for name in names:
if name == 'background':
continue
category_map[name] = cat_id
categories.append({'id': cat_id, 'name': name, 'supercategory': ''})
cat_id += 1
return category_map, categories, bg_used
categories = []
category_map = {}
label_names = []
bg_found = False
bg_used = False
if labels_file is None:
log.info('Reading labels from annotation...')
for label in xml_root.iter('label'):
for name in label.findall("./name"):
if name.text == 'background':
bg_found = True
continue
label_names.append(name.text)
if len(label_names) == 0:
log.info('Labels in annotation were not found. Please use \'--labels\' argument to define file with labels.')
else:
category_map, categories, bg_used = get_categories(label_names, bg_found, use_background_label, sort=True)
else:
log.info('Parsing labels from file <{}>...'.format(labels_file))
with open(labels_file, 'r') as file:
string = ' '
while string != '' and string != '\n':
string = file.readline()
labels = string.split(' ')
for label in labels:
if label == '\n' or label == '':
continue
label = label.replace('\n', '')
if label == 'background':
bg_found = True
continue
label_names.append(label)
category_map, categories, bg_used = get_categories(label_names, bg_found, use_background_label)
if len(categories) == 0:
raise ValueError('Categories list is empty. Something wrong.')
result_annotation['categories'] = categories
log.info('Found the next labels: {}'.format(category_map))
if bg_found and not bg_used:
log.warning('Label <background> was found but not used. '
'To enable it should use command line argument [--use_background_label]')
return category_map
def main():
args = parse_args()
xml_file_name = args.cvat_xml
output_file_name = args.output
root = etree.parse(xml_file_name).getroot()
if args.draw != None:
log.info(
'Draw key was enabled. Images will be saved in directory <{}>'.
format(args.draw))
result_annotation = {
'licenses': [],
'info': {},
'categories': [],
'images': [],
'annotations': []
}
insert_license_data(result_annotation)
insert_info_data(root, result_annotation)
category_map = insert_categories_data(root, args.use_background_label,
result_annotation,
osp.dirname(xml_file_name))
if len(category_map) == 0:
sys.exit(
'Labels were not found. Be sure that annotation <{}> includes field <labels> or '
'annotation directory includes file <labels.txt>'.format(
xml_file_name))
segm_id = 0
z_order_off_counter = 0
# Parse original annotation
for img in tqdm(root.iter('image'),
desc='Processing images from ' + xml_file_name):
image = {}
for key, value in img.items():
image[key] = value
image['polygon'] = []
z_order_on_counter = 0
polygon_counter = 0
for poly in img.iter('polygon'):
polygon = {}
for key, value in poly.items():
polygon[key] = value
if key == 'z_order':
z_order_on_counter += 1
polygon_counter += 1
image['polygon'].append(polygon)
# If at least one of polygons on image does not have field 'z_order' do not sort them
if z_order_on_counter == polygon_counter:
image['polygon'].sort(key=lambda x: int(x['z_order']))
else:
z_order_off_counter += 1
# Create new image
insert_image_data(image, args.image_dir, result_annotation)
height = result_annotation['images'][-1]['height']
width = result_annotation['images'][-1]['width']
image['polygon'] = fix_segments_intersections(
image['polygon'], height, width, image['name'],
args.use_background_label)
# Create new annotation for this image
for poly in image['polygon']:
insert_annotation_data(image, category_map, segm_id, poly,
[height, width], result_annotation)
segm_id += 1
# Draw contours of objects on image
if args.draw != None:
draw_polygons(image['polygon'], image['name'], args.image_dir,
args.draw, args.draw_labels)
log.info('Processed images: {}'.format(len(result_annotation['images'])))
log.info('Processed objects: {}'.format(
len(result_annotation['annotations'])))
if z_order_off_counter > 0:
log.warning(
'Annotation does not have a field \'z_order\' for {} image(s). '
'Overlapped objects may be cropped incorrectly!'.format(
z_order_off_counter))
# Save created annotation
log.info('Saving annotation...')
with open(output_file_name, 'w') as outfile:
json.dump(result_annotation, outfile)
log.info(
'Annotation was saved in <{}> successfully'.format(output_file_name))
# Try to load created annotation via cocoapi
try:
log.info('Trying to load annotation <{}> via cocoapi...'.format(
output_file_name))
anno = coco_loader.COCO(output_file_name)
except:
raise
else:
log.info(
'Annotation in COCO representation <{}> created from <{}> successfully!'
.format(output_file_name, xml_file_name))
def watch(endpoint, key):
"""watch sends watch request to etcd.
Examples:
curl -L http://localhost:2379/v3alpha/watch \
-X POST -d ''{"create_request": {"key":"Zm9v"} }'
"""
# Python 2
# key_str = base64.b64encode(key)
# Python 3 base64 requires utf-08 encoded bytes
# Python 3 JSON encoder requires string
key_str = base64.b64encode(bytes(key, "utf-8")).decode()
req = {'create_request': {"key": key_str}}
while True:
try:
rresp = requests.post(endpoint + '/v3alpha/watch',
data=json.dumps(req),
stream=True)
for line in rresp.iter_lines():
# filter out keep-alive new lines
if line:
decoded_line = line.decode('utf-8')
resp = json.loads(decoded_line)
if 'result' not in resp:
log.warning('{0} does not have result'.format(resp))
return ''
if 'created' in resp['result']:
if resp['result']['created']:
log.warning('watching {0}'.format(key))
continue
if 'events' not in resp['result']:
log.warning('{0} returned no events: {1}'.format(
key, resp))
return None
if len(resp['result']['events']) != 1:
log.warning('{0} returned >1 event: {1}'.format(
key, resp))
return None
if 'kv' in resp['result']['events'][0]:
if 'value' in resp['result']['events'][0]['kv']:
val = resp['result']['events'][0]['kv']['value']
return base64.b64decode(val)
else:
log.warning('no value in ', resp)
return None
else:
log.warning('no kv in ', resp)
return None
except requests.exceptions.ConnectionError as err:
log.warning('Connection error: {0}'.format(err))
time.sleep(5)
except:
log.warning('Unexpected error:', sys.exc_info()[0])
raise
