def kalman(A, B, C, Q, R): """Solves for the steady state kalman gain and covariance matricies. Args: A, B, C: SS matricies. Q: The model uncertantity R: The measurement uncertainty Returns: KalmanGain, Covariance. """ I = numpy.matrix(numpy.eye(Q.shape[0])) Z = numpy.matrix(numpy.zeros(Q.shape[0])) n = A.shape[0] m = C.shape[0] controllability_rank = numpy.linalg.matrix_rank(ctrb(A.T, C.T)) if controllability_rank != n: glog.warning('Observability of %d != %d, unobservable state', controllability_rank, n) # Compute the steady state covariance matrix. P_prior, rcond, w, S, T = slycot.sb02od(n=n, m=m, A=A.T, B=C.T, Q=Q, R=R, dico='D') S = C * P_prior * C.T + R K = numpy.linalg.lstsq(S.T, (P_prior * C.T).T)[0].T P = (I - K * C) * P_prior return K, P
def reg(id, func): global MSG_MAP if id in MSG_MAP: glog.warning("gnet>id already exist|id:%d func:%s" % (id, repr(func))) return False MSG_MAP[id] = func return True
def prepare_frames(split_name): """ extract fraems for dataset `split_name`. pickle to disk. frames_info[samp_id] |- image: image path |- trans: word label (without space) `- frames: frame images filename list e.g. frames_info[train_0]: { 'frames': ['0.jpg', '1.jpg', '2.jpg', '3.jpg', '4.jpg', ...], 'image': '/home/paile/research/mjsynth/mnt/ramdisk/max/90kDICT32px/378/5/54_amethyst_2482.jpg', 'trans': 'amethyst' } """ print 'processing:', split_name, '-------------------------' # read split info split = config.splits[split_name] trans_path = split.trans_pth frames_dir = split.frames_dir if path.isdir(frames_dir): glog.warning('%s already exists.', frames_dir) sys.exit(-1) print 'trans_path:', trans_path print 'frames_dir:', frames_dir # read trans all_trans = utils.read_table(trans_path) # assign sample id trans_ids = ['%s_%d' % (split_name, i) for i in range(len(all_trans))] all_trans = OrderedDict(zip(trans_ids, all_trans)) # frames info frames_info = OrderedDict() err_images = [] for samp_id in all_trans: try: image_pth = all_trans[samp_id][0] im = load_image(image_pth) frame_pths = extract_frames(im, samp_id, frames_dir) frames_info[samp_id] = {} frames_info[samp_id]['image'] = image_pth frames_info[samp_id]['trans'] = all_trans[samp_id][1] frames_info[samp_id]['frames'] = frame_pths except Exception, e: err_images.append(image_pth+'\n') print 'Error:', image_pth
def nxCfg2Acfg(outputDir, malwareDirPrefix): """Path name format: class/graphId/pkl_name""" logGraphPaths = open(outputDir + '/graph_pathnames.csv', 'w') graphSizes = {x: [] for x in malwareNames} realPklPath = [] for malwareDirname in glob.glob(malwareDirPrefix + '/*'): log.debug('Enter malware directory %s' % malwareDirname) malwareName = malwareDirname.split('/')[-1] if malwareName not in malwareNames: log.warning('%s not in known malware types' % malwareName) continue log.info("Search %s gpickles in %s" % (malwareName, malwareDirname)) pklPaths = glob.glob(malwareDirname + '/*') if len(pklPaths) == 0: log.warning('Pickle path %s is empty' % pklPaths) for pklPath in pklPaths: if pklPath[-8:] != '.gpickle': log.warning('Ignore %s since it\'s not gpickle file' % pklPath) continue log.debug('Loading nx.Graph from %s' % pklPath) logGraphPaths.write(pklPath + '\n') try: G = nx.read_gpickle(pklPath) except UnicodeDecodeError as e: log.error('Decode failed for %s: %s' % (pklPath, e)) realPklPath.append(pklPath) graphSizes[malwareName].append(number_of_nodes(G)) log.info("Found %s gpickles" % len(realPklPath)) graphSizesDf = pd.DataFrame.from_dict(graphSizes, orient='index').T graphSizesDf.to_csv(outputDir + '/graph_sizes.csv', index=False) return realPklPath
def mkdir(path_to_dir): if not os.path.exists(path_to_dir): os.mkdir(path_to_dir) else: glog.warning('Dir {0} already exist.'.format(path_to_dir))
# 2. Loop for num_iterations: # a. Forward propagation # b. Compute cost function # c. Backward propagation # d. Update parameters (using parameters, and grads from backprop) # 4. Use trained parameters to predict labels log.info("loading 'cats' parameters on {0}".format(param_path)) parameters = np.load(param_path).item() log.info("loaded 'cats' parameters on {0}".format(param_path)) log.info("starting worker on {0}".format(EP)) while True: ITEM = fetch_item(EP) if ITEM['error'] not in ['', u'']: log.warning(ITEM['error']) time.sleep(5) continue if ITEM['bucket'] == '/cats-request': IMAGE_PATH = ITEM['value'] if not os.path.exists(IMAGE_PATH): log.warning('cannot find image {0}'.format(IMAGE_PATH)) ITEM['progress'] = 100 ITEM['error'] = 'cannot find image {0}'.format(IMAGE_PATH) else: img_class = classify(IMAGE_PATH, parameters) ITEM['progress'] = 100 ITEM['value'] = "[WORKER - ACK] it's a '{0}'!".format(img_class) POST_RESPONSE = post_item(EP, ITEM)
classifier = classifier.cuda() classifier.eval() startTime = time.process_time() testPredProb = predictDataset(testGraphs, classifier) log.info(f'Net test time = {time.process_time() - startTime} seconds') exportPredictions(testGraphs, testPredProb) if __name__ == '__main__': log.setLevel("INFO") random.seed(cmd_args.seed) np.random.seed(cmd_args.seed) torch.manual_seed(cmd_args.seed) if cmd_args.data == 'YANACFG': log.warning(f'No testset for YANACFG data') else: startTime = time.process_time() testGraphs = loadGraphsMayCache(cmd_args.test_dir, isTestSet=True) normalizeFeatures(testGraphs, isTestSet=True, operation=cmd_args.norm_op) dataReadyTime = time.process_time() - startTime log.info('Testset ready takes %.2fs' % dataReadyTime) trainGraphs = loadGraphsMayCache(cmd_args.train_dir, isTestSet=False) trainGraphs = filterOutNoEdgeGraphs(trainGraphs) decideHyperparameters(trainGraphs) testWithModel(testGraphs)
def test_warning(self): log.warning('test')
print('%d docs in index' % writer.numDocs()) print('Indexing json files...') # For text field. t1 = FieldType() t1.setStored(False) t1.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) with codecs.open(json_file, encoding='utf8') as f: for line in tqdm(f): line = line.strip() try: json_doc = json.loads(line) except: glog.warning('Error json parsing: {}'.format(line)) continue # Delete existing abstracts. This is useful when adding # update files from Medline. try: assert 'pmid' in json_doc pmid_query = TermQuery(Term('pmid', json_doc['pmid'])) pmcid_query = TermQuery(Term('pmcid', json_doc['pmcid'])) id_query = IntPoint.newRangeQuery("id", json_doc['id'], json_doc['id']) bq = BooleanQuery.Builder() bq.add(pmid_query, BooleanClause.Occur.MUST) bq.add(pmcid_query, BooleanClause.Occur.MUST) bq.add(id_query, BooleanClause.Occur.MUST) q = bq.build()
def test_warning(self): log.warning('test') return
def insert_categories_data(xml_root, use_background_label, result_annotation, xml_dir): """Get labels from input annotation and fill categories field in output annotation Args: xml_root: root for xml parser use_background_label: key to enable using label background result_annotation: output annotation in COCO representation xml_dir: directory with input annotation """ log.info('Reading labels...') categories = [] category_map = {} bg_found = False id = 0 for label in xml_root.iter('label'): for name in label.findall("./name"): if not use_background_label and name.text == 'background': bg_found = True continue category_map[name.text] = id categories.append({ 'id': id, 'name': name.text, 'supercategory': '' }) id += 1 if len(categories) == 0: log.info( 'Labels in annotation were not found. Trying to find file <labels.txt> in <{}>' .format(xml_dir)) if osp.isfile(osp.join(xml_dir, 'labels.txt')): labels_file = osp.join(xml_dir, 'labels.txt') log.info('File <labels.txt> was found in <{}>. Reading...'.format( xml_dir)) with open(labels_file, 'r') as file: string = ' ' id = 0 while string != '' and string != '\n': string = file.readline() labels = string.split(' ') for l in labels: if l == '\n': continue if not use_background_label and l == 'background': bg_found = True continue category_map[l] = id categories.append({ 'id': id, 'name': l, 'supercategory': '' }) id += 1 result_annotation['categories'] = categories log.info('Found the next labels: {}'.format(category_map)) if bg_found: log.warning( 'Label <background> was found but not used. ' 'To enable it should use command line argument [--use_background_label]' ) return category_map
def process_properties(self, dump_video=True, dump_images=False, tstamps_of_interest=None): assert (isinstance(self.response, Response)) self.med_ret = MediaRetriever(self.response.url) dump_video = (self.request.get("dump_video") or dump_video) and \ self.med_ret.is_video dump_images = (self.request.get("dump_images") or dump_images) or \ self.med_ret.is_image if dump_images is False and self.med_ret.is_image: dump_images = True if dump_video is False and dump_images is False: log.warning("`dump_video` and `dump_images` are both false." " Unable to proceed.") return log.debug(f"Dumping Video: {dump_video}") log.debug(f"Dumping Frames: {dump_images}") # we get the frame iterator frames_iterator = [] if tstamps_of_interest is not None: if type(tstamps_of_interest) is list: for t in tstamps_of_interest: frame = self.med_ret.get_frame(tstamp=t) frames_iterator.append((frame, t)) else: try: frames_iterator = self.med_ret.get_frames_iterator( sample_rate=1.0) except Exception: log.error(traceback.format_exc()) raise Exception("Error loading media") vid_file, images_dir, max_tstamp = self.dump_data( frames_iterator, dump_video=dump_video, dump_images=dump_images) props = [] if self.local_dir is not None and dump_video: local_vid_path = self.copy_video(vid_file.name) p = Property( server=self.name, ver=self.version, value=local_vid_path, property_type="dumped_video", property_id=4, ) props.append(p) if self.local_dir is not None and dump_images: local_frames_paths = self.copy_frames(images_dir.name) ps = [ Property( server=self.name, ver=self.version, value=path, property_type="dumped_image", property_id=3, ) for path in local_frames_paths ] props.extend(ps) if self.s3_bucket is not None and dump_video: s3_vid_url = self.upload_video(vid_file.name) p = Property( server=self.name, ver=self.version, value=s3_vid_url, property_type="dumped_video", property_id=2, ) props.append(p) if self.s3_bucket is not None and dump_images: s3_frames_urls = self.upload_frames(images_dir.name) ps = [ Property( server=self.name, ver=self.version, value=url, property_type="dumped_image", property_id=1, ) for url in s3_frames_urls ] props.extend(ps) images_dir.cleanup() vid_file.close() media_summary = VideoAnn(t1=0.0, t2=max_tstamp, props=props) self.response.append_media_summary(media_summary)
def publish_intermediates(*, args, ro_client, rw_client): local_intermediates = {} remote_intermediates = {} remote_error_records = [] run_identifiers = workflow.get_run_identifiers(args.filter_bucket) if not run_identifiers: log.warning("No run identifiers found") return run_id = run_identifiers[-1] run_id_path = args.download_path / Path(run_id) run_id_path.mkdir(parents=True, exist_ok=True) intermediates_path = run_id_path / Path("enrolled.json") workflow.download_and_retry_from_google_cloud( args.filter_bucket, f"{run_id}/enrolled.json", intermediates_path, timeout=timedelta(minutes=5), ) with intermediates_path.open("r") as f: for entry in json.load(f): try: intObj = Intermediate(**entry, debug=args.debug) if intObj.unique_id() in local_intermediates: log.warning( f"[{intObj.unique_id()}] Local collision: {intObj} with " + f"{local_intermediates[intObj.unique_id()]}" ) continue local_intermediates[intObj.unique_id()] = intObj except Exception as e: log.error("Error importing file from {}: {}".format(args.inpath, e)) log.error("Record: {}".format(entry)) raise e for record in ro_client.get_records( collection=settings.KINTO_INTERMEDIATES_COLLECTION ): try: intObj = Intermediate(**record) remote_intermediates[intObj.unique_id()] = intObj except IntermediateRecordError as ire: log.warning("Skipping broken intermediate record at Kinto: {}".format(ire)) remote_error_records.append(record) except KeyError as ke: log.error("Critical error importing Kinto dataset: {}".format(ke)) log.error("Record: {}".format(record)) raise ke to_delete = set(remote_intermediates.keys()) - set(local_intermediates.keys()) to_upload = set(local_intermediates.keys()) - set(remote_intermediates.keys()) to_update = set() for i in set(local_intermediates.keys()) & set(remote_intermediates.keys()): if not local_intermediates[i].equals(remote_record=remote_intermediates[i]): to_update.add(i) expired = set() for i in to_delete: try: if remote_intermediates[i].is_expired(): expired.add(i) except Exception as e: log.warning(f"Failed to track expiration for {i}: {e}") to_delete_not_expired = to_delete - expired delete_pubkeys = {remote_intermediates[i].pubKeyHash for i in to_delete} upload_pubkeys = {local_intermediates[i].pubKeyHash for i in to_upload} unenrollments = set() new_enrollments = set() update_other_than_enrollment = set() for i in to_update: if ( local_intermediates[i].crlite_enrolled and not remote_intermediates[i].crlite_enrolled ): new_enrollments.add(i) elif ( remote_intermediates[i].crlite_enrolled and not local_intermediates[i].crlite_enrolled ): unenrollments.add(i) else: update_other_than_enrollment.add(i) log.info(f"Total entries before update: {len(remote_intermediates)}") log.info(f"To delete: {len(to_delete)} (Deletion enabled: {args.delete})") log.info(f"- Expired: {len(expired)}") log.info(f"To add: {len(to_upload)}") log.info( f"Certificates updated (without a key change): {len(delete_pubkeys & upload_pubkeys)}" ) log.info(f"Remote records in an error state: {len(remote_error_records)}") log.info(f"Total entries updated: {len(to_update)}") log.info(f"- New enrollments: {len(new_enrollments)}") log.info(f"- Unenrollments: {len(unenrollments)}") log.info(f"- Other: {len(update_other_than_enrollment)}") log.info(f"Total entries after update: {len(local_intermediates)}") if args.export: with open(Path(args.export) / Path("to_delete"), "w") as df: export_intermediates(df, to_delete, remote_intermediates) with open(Path(args.export) / Path("to_delete_not_expired"), "w") as df: export_intermediates(df, to_delete_not_expired, remote_intermediates) with open(Path(args.export) / Path("expired"), "w") as df: export_intermediates(df, expired, remote_intermediates) with open(Path(args.export) / Path("to_upload"), "w") as df: export_intermediates(df, to_upload, local_intermediates) with open(Path(args.export) / Path("to_update"), "w") as df: export_intermediates( df, to_update, local_intermediates, old=remote_intermediates ) with open(Path(args.export) / Path("unenrollments"), "w") as df: export_intermediates( df, unenrollments, local_intermediates, old=remote_intermediates ) with open(Path(args.export) / Path("new_enrollments"), "w") as df: export_intermediates( df, new_enrollments, local_intermediates, old=remote_intermediates ) with open(Path(args.export) / Path("update_other_than_enrollment"), "w") as df: export_intermediates( df, update_other_than_enrollment, local_intermediates, old=remote_intermediates, ) if args.debug: print("Variables available:") print(" local_intermediates") print(" remote_intermediates") print(" remote_error_records") print("") print(" to_upload") print(" to_delete") print(" to_update") print("") print(" new_enrollments") print(" unenrollments") print("") print(" delete_pubkeys") print(" upload_pubkeys") print( " delete_pubkeys & upload_pubkeys # certs updated without changing the key" ) print("") print(" local_intermediates[to_update.pop()].cert # get cert object") print("") print("Use 'continue' to proceed") print("") breakpoint() if args.noop: log.info(f"Noop flag set, exiting before any intermediate updates") return # Don't accidentally use the ro_client beyond this point ro_client = None if len(remote_error_records) > 0: log.info(f"Cleaning {len(remote_error_records)} broken records") for record in remote_error_records: try: rw_client.delete_record( collection=settings.KINTO_INTERMEDIATES_COLLECTION, id=record["id"], ) except KintoException as ke: log.warning(f"Couldn't delete record id {record['id']}: {ke}") for unique_id in to_delete: intermediate = remote_intermediates[unique_id] log.info(f"Deleting {intermediate} from Kinto (delete={args.delete})") if args.delete: try: intermediate.delete_from_kinto(rw_client=rw_client) except KintoException as ke: log.warning(f"Couldn't delete record id {intermediate}: {ke}") for unique_id in to_upload: intermediate = local_intermediates[unique_id] log.debug(f"Uploading {intermediate} to Kinto") intermediate.add_to_kinto(rw_client=rw_client) update_error_records = [] for unique_id in to_update: local_int = local_intermediates[unique_id] remote_int = remote_intermediates[unique_id] if not local_int.equals(remote_record=remote_int): try: local_int.update_kinto( rw_client=rw_client, remote_record=remote_int, ) except KintoException as ke: update_error_records.append((local_int, remote_int, ke)) for (local_int, remote_int, ex) in update_error_records: log.warning( f"Failed to update local={local_int} remote={remote_int} exception={ex}" ) log.info("Verifying correctness...") verified_intermediates = {} verification_error_records = [] for record in rw_client.get_records( collection=settings.KINTO_INTERMEDIATES_COLLECTION ): try: intObj = Intermediate(**record) verified_intermediates[intObj.unique_id()] = intObj except IntermediateRecordError as ire: log.warning( "Verification found broken intermediate record at Kinto: {}".format(ire) ) verification_error_records.append(record) except KeyError as ke: log.error("Critical error importing Kinto dataset: {}".format(ke)) log.error("Record: {}".format(record)) raise ke if len(verification_error_records) > 0: raise KintoException( "There were {} broken intermediates. Re-run to fix.".format( len(verification_error_records) ) ) log.info( "{} intermediates locally, {} at Kinto.".format( len(local_intermediates), len(verified_intermediates) ) ) if args.delete and set(local_intermediates.keys()) != set( verified_intermediates.keys() ): log.error("The verified intermediates do not match the local set. Differences:") missing_remote = set(local_intermediates.keys()) - set( verified_intermediates.keys() ) missing_local = set(verified_intermediates.keys()) - set( local_intermediates.keys() ) for d in missing_remote: log.error("{} does not exist at Kinto".format(d)) for d in missing_local: log.error( "{} exists at Kinto but should have been deleted (not in local set)".format( d ) ) raise KintoException("Local/Remote Verification Failed") elif not args.delete and set(local_intermediates.keys()) > set( verified_intermediates.keys() ): log.error("The verified intermediates do not match the local set. Differences:") missing_remote = set(local_intermediates.keys()) - set( verified_intermediates.keys() ) for d in missing_remote: log.error("{} does not exist at Kinto".format(d)) raise KintoException("Local/Remote Verification Failed") for unique_id in verified_intermediates.keys(): remote_int = verified_intermediates[unique_id] if unique_id not in local_intermediates and not args.delete: log.info( "Remote {} has been deleted locally, but ignoring.".format(remote_int) ) continue local_int = local_intermediates[unique_id] if not local_int.equals(remote_record=remote_int): if not remote_int.pemAttachment.verify(pemData=local_int.pemData): log.warning( "PEM hash mismatch for {}; remote={} != local={}".format( unique_id, remote_int, local_int ) ) raise KintoException( "Local/Remote PEM mismatch for uniqueId={}".format(unique_id) ) else: log.warning( f"Local/Remote metadata mismatch, uniqueID={unique_id}, " + f"local={local_int.details()}, remote={remote_int.details()}" ) raise KintoException( "Local/Remote metadata mismatch for uniqueId={}".format(unique_id) ) if to_update or to_upload or to_delete and not args.noop and args.request_review: log.info( f"Set for review, {len(to_update)} updates, {len(to_upload)} uploads, " + f"{len(to_delete)} deletions." ) rw_client.request_review_of_collection( collection=settings.KINTO_INTERMEDIATES_COLLECTION, ) else: log.info(f"No updates to do")
def process_properties(self, dump_video=True, dump_images=False, tstamps_Of_Interest=None): self.last_tstamp = 0.0 assert (self.response) self.med_ret = MediaRetriever(self.response.url) self.w, self.h = self.med_ret.get_w_h() media_id = os.path.basename(self.response.url).rsplit(".", 1)[0] self.media_id = "".join( [e for e in media_id if e.isalnum() or e in ["/", "."]]) self.content_type_map = {} # if there is no flag in the request of not request_api we'll get None. self.dump_video = None self.dump_images = None try: self.dump_video = self.request.get("dump_video") self.dump_images = self.request.get("dump_images") except Exception: log.error( "Unable to get flags from request dump_video or dump_images") pass if self.dump_video is None: self.dump_video = dump_video if self.dump_images is None: self.dump_images = dump_images if self.dump_video is False and self.dump_images is False: log.warning( "Not dumping anything--you might want to dump something.") return dump_folder = self.pushing_folder + '/' + self.media_id + '/' self.dumping_folder_url = dump_folder if dump_folder: if not os.path.exists(dump_folder): os.makedirs(dump_folder) if self.dump_video: filename = dump_folder + '/video.mp4' fps = 1 frameSize = self.med_ret.shape frameSize = (self.w, self.h) fourcc = cv2.VideoWriter_fourcc(*'H264') log.info("filename: " + filename) log.info("fourcc: " + str(fourcc)) log.info("type(fourcc): " + str(type(fourcc))) log.info("fps: " + str(fps)) log.info("type(fps): " + str(type(fps))) log.info("frameSize: " + str(frameSize)) log.info("type(frameSize): " + str(type(frameSize))) vid = cv2.VideoWriter(filename, fourcc, fps, frameSize) self.content_type_map[os.path.basename(filename)] = 'video/mp4' face = cv2.FONT_HERSHEY_SIMPLEX scale = 0.65 thickness = 2 # we get the image_annotation tstamps tstamps = self.response.get_timestamps() tstamp_frame_anns = self.response.get_timestamps_from_frames_ann() log.debug('tstamps: ' + str(tstamps)) log.debug('tstamps_dets: ' + str(tstamp_frame_anns)) # we get the frame iterator frames_iterator = [] if tstamps_Of_Interest: if type(tstamps_Of_Interest) is list: for t in tstamps_Of_Interest: frame = self.med_ret.get_frame(tstamp=t) frames_iterator.append((frame, t)) elif tstamps_Of_Interest is None: try: frames_iterator = self.med_ret.get_frames_iterator( sample_rate=1.0) except Exception: log.error(traceback.format_exc()) raise Exception("Error loading media") for i, (img, tstamp) in enumerate(frames_iterator): self.last_tstamp = tstamp if img is None: log.warning("Invalid frame") continue if tstamp is None: log.warning("Invalid tstamp") continue # log.info('tstamp: ' + str(tstamp)) if tstamp in tstamp_frame_anns: log.debug("drawing frame for tstamp: " + str(tstamp)) # we get image_ann for that time_stamps regions = self.response.get_regions_from_tstamp(tstamp) # log.info(json.dumps(image_ann, indent=2)) for region in regions: rand_color = get_rand_bgr() p0, p1 = p0p1_from_bbox_contour(region['contour'], self.w, self.h) anchor_point = [p0[0] + 3, p1[1] - 3] if abs(p1[1] - self.h) < 30: anchor_point = [p0[0] + 3, int(p1[1] / 2) - 3] img = cv2.rectangle(img, p0, p1, rand_color, thickness) prop_strs = get_props_from_region(region) for i, prop in enumerate(prop_strs): img = cv2.putText( img, prop, (anchor_point[0], anchor_point[1] + i * 25), face, 1.0, rand_color, thickness) elif tstamp in tstamps: log.debug("Making frame gray") gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) img = cv2.cvtColor(gray, cv2.COLOR_GRAY2BGR) else: log.debug("No processed frame") continue # Include the timestamp img = cv2.putText(img, str(tstamp), (20, 20), face, scale, [255, 255, 255], thickness) if self.dump_video: # we add the frame log.debug("Adding frame") vid.write(img) if self.dump_images: # we dump the frame outfn = "{}/{}.jpg".format(dump_folder, tstamp) log.debug("Writing to file: {}".format(outfn)) cv2.imwrite(outfn, img) self.content_type_map[os.path.basename(outfn)] = 'image/jpeg' if self.dump_video: vid.release() if self.s3_bucket: try: self.upload_files(dump_folder) except Exception: log.error(traceback.format_exc()) if self.pushing_folder == DEFAULT_DUMP_FOLDER: log.info('Removing files in ' + dump_folder) shutil.rmtree(dump_folder) props = [] if self.dump_images: props.append( Property( server=self.name, ver=self.version, value=self.dumping_folder_url, property_type="dumped_images", property_id=1, )) if self.dump_video: dumped_video_url = self.dumping_folder_url + '/video.mp4' dumped_video_url = dumped_video_url.replace('//', '/') dumped_video_url = dumped_video_url.replace('https:/', 'https://') props.append( Property( server=self.name, ver=self.version, value=dumped_video_url, property_type="dumped_video", property_id=2, )) media_summary = VideoAnn(t1=0.0, t2=self.last_tstamp, props=props) self.response.append_media_summary(media_summary)
def run(args): # Get experiment information from yaml file. experiment_params = yaml.load(args.experiments_path) regression_tests_dir = os.path.expandvars( experiment_params['regression_tests_dir']) params_dir = os.path.expandvars(experiment_params['params_dir']) dataset_dir = os.path.expandvars(experiment_params['dataset_dir']) executable_path = os.path.expandvars(experiment_params['executable_path']) datasets_to_run = experiment_params['datasets_to_run'] regression_params = experiment_params['regression_parameters'] # Build dictionary from parameter name to list of parameter values param_name_to_values = dict() for regression_param in regression_params: param_name_to_values[ regression_param['name']] = regression_param['values'] log.info("Setup regression tests.") check_and_create_regression_test_structure(regression_tests_dir, params_dir, param_name_to_values, dataset_dir) # Run experiments. log.info("Run regression tests.") stats = dict() for regression_param in tqdm(regression_params): # Redirect to param_name_value dir param_name = regression_param['name'] param_name = regression_param['name'] stats[param_name] = dict() log.info("Run for regression parameter: %s" % param_name) for param_value in tqdm(regression_param['values'], desc=regression_param['name'], leave=False): results_dir = os.path.join(regression_tests_dir, param_name, str(param_value)) # Redirect to modified params_dir params_dir = os.path.join(results_dir, 'params') stats[param_name][param_value] = dict() for dataset in datasets_to_run: dataset_name = dataset['name'] log.info("\033[93mRun: %s = %f in dataset %s\033[00m" % (param_name, param_value, dataset_name)) pipelines_to_run = dataset['pipelines'] if not run_dataset(results_dir, params_dir, dataset_dir, dataset, executable_path, args.run_pipeline, args.analyse_vio, args.plot, args.save_results, args.save_plots, args.save_boxplots, pipelines_to_run, dataset['initial_frame'], dataset['final_frame'], dataset['discard_n_start_poses'], dataset['discard_n_end_poses'], extra_flagfile_path='override.flags'): log.warning("A pipeline run has failed...") stats[param_name][param_value][dataset_name] = dict() for pipeline in pipelines_to_run: results_file = os.path.join(results_dir, dataset_name, pipeline, "results.yaml") if os.path.isfile(results_file): stats[param_name][param_value][dataset_name][ pipeline] = yaml.load(open(results_file, 'r')) else: log.warning( "Could not find results file: {}. \n Adding cross." .format(results_file)) stats[param_name][param_value][dataset_name][ pipeline] = False # Save all stats in regression tests root directory # TODO(Toni) ideally save them as a pandas dataframe object! And put NaNs where there is a False with open(os.path.join(regression_tests_dir, "all_stats.yaml"), 'w') as outfile: outfile.write(yaml.dump(stats)) # Compile plots for all results, do that in jupyter notebook using plotly instead of here... #plot_dir = os.path.join(regression_tests_dir, param_name, "results", dataset_name) #max_y = -1 #draw_regression_simple_boxplot_APE(stats, plot_dir, max_y) #log.info("Finished regression test for param_name: %s" % param_name) return True
def run_dataset(results_dir, params_dir, dataset_dir, dataset_properties, executable_path, run_pipeline, analyse_vio, plot, save_results, save_plots, save_boxplots, pipelines_to_run_list, initial_k, final_k, discard_n_start_poses=0, discard_n_end_poses=0, extra_flagfile_path='', verbose_sparkvio=False): """ Evaluates pipeline using Structureless(S), Structureless(S) + Projection(P), \ and Structureless(S) + Projection(P) + Regular(R) factors \ and then compiles a list of results """ dataset_name = dataset_properties['name'] dataset_segments = dataset_properties['segments'] ################### RUN PIPELINE ################################ pipeline_output_dir = os.path.join(results_dir, "tmp_output/output/") evt.create_full_path_if_not_exists(pipeline_output_dir) output_file = os.path.join(pipeline_output_dir, "output_posesVIO.csv") has_a_pipeline_failed = False if len(pipelines_to_run_list) == 0: log.warning("Not running pipeline...") for pipeline_type in pipelines_to_run_list: has_a_pipeline_failed = not process_vio( executable_path, dataset_dir, dataset_name, results_dir, params_dir, pipeline_output_dir, pipeline_type, dataset_segments, save_results, plot, save_plots, output_file, run_pipeline, analyse_vio, discard_n_start_poses, discard_n_end_poses, initial_k, final_k, extra_flagfile_path, verbose_sparkvio) # Save boxplots if save_boxplots: # TODO(Toni) is this really saving the boxplots? if not has_a_pipeline_failed: stats = dict() for pipeline_type in pipelines_to_run_list: results_dataset_dir = os.path.join(results_dir, dataset_name) results = os.path.join(results_dataset_dir, pipeline_type, "results.yaml") if not os.path.exists(results): raise Exception( "\033[91mCannot plot boxplots: missing results for %s pipeline \ and dataset: %s" % (pipeline_type, dataset_name) + "\033[99m \n \ Expected results here: %s" % results) try: stats[pipeline_type] = yaml.load(open(results, 'r'), Loader=yaml.Loader) except yaml.YAMLError as e: raise Exception("Error in results file: ", e) log.info("Check stats %s in %s" % (pipeline_type, results)) check_stats(stats[pipeline_type]) log.info("Drawing boxplots.") evt.draw_rpe_boxplots(results_dataset_dir, stats, len(dataset_segments)) else: log.warning( "A pipeline run has failed... skipping boxplot drawing.") if not has_a_pipeline_failed: evt.print_green("All pipeline runs were successful.") else: log.error("A pipeline has failed!") evt.print_green("Finished evaluation for dataset: " + dataset_name) return not has_a_pipeline_failed
def edg_process_document(self, request): try: return self.stub.EdgProcessDocument(request, self.timeout_seconds) except ExpirationError: glog.warning('Expiration:' + '\t' + ','.join([d.doc_id for d in request.document]))
def fix_segments_intersections(polygons, height, width, img_name, use_background_label, threshold=0.0, ratio_tolerance=0.001): """Find all intersected regions and crop contour for back object by objects which are in front of the first one. It is related to a specialty of segmentation in CVAT annotation. Intersection is calculated via function 'iou' from cocoapi Args: polygons: all objects on image represented as 2D array of objects' contours height: height of image width: width of image img_name: name of image file threshold: threshold of intersection over union of two objects. By default is set to 0 and processes any two intersected objects ratio_tolerance: used for situation when one object is fully or almost fully inside another one and we don't want make "hole" in one of objects """ converted_polygons = [] empty_polygon = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0] # Convert points of polygons from string to coco's array. # All polygons must be sorted in order from bottom to top for polygon in polygons: label = polygon['label'] points = polygon['points'].split(';') new_polygon = [] for xy in points: x = float(xy.split(',')[0]) y = float(xy.split(',')[1]) new_polygon.append(x) new_polygon.append(y) converted_polygons.append({'label': label, 'points': new_polygon}) for i in range(0, len(converted_polygons)): rle_bottom = mask_util.frPyObjects([converted_polygons[i]['points']], height, width) segment_overlapped = False for j in range(i + 1, len(converted_polygons)): rle_top = mask_util.frPyObjects([converted_polygons[j]['points']], height, width) iou = mask_util.iou(rle_bottom, rle_top, [0, 0]) area_top = sum(mask_util.area(rle_top)) area_bottom = sum(mask_util.area(rle_bottom)) if area_bottom == 0: continue area_ratio = area_top / area_bottom sum_iou = sum(iou) # If segment is fully inside another one, save this segment as is if area_ratio - ratio_tolerance < sum_iou[ 0] < area_ratio + ratio_tolerance: continue # Check situation when bottom segment is fully inside top. # It means that in annotation is mistake. Save this segment as is if 1 / area_ratio - ratio_tolerance < sum_iou[ 0] < 1 / area_ratio + ratio_tolerance: continue if sum_iou[0] > threshold: segment_overlapped = True bottom_mask = np.array(mask_util.decode(rle_bottom), dtype=np.uint8) top_mask = np.array(mask_util.decode(rle_top), dtype=np.uint8) bottom_mask = np.subtract(bottom_mask, top_mask) bottom_mask[bottom_mask > 1] = 0 bottom_mask = np.sum(bottom_mask, axis=2) bottom_mask = np.array(bottom_mask > 0, dtype=np.uint8) converted_polygons[i]['points'] = mask_to_polygon(bottom_mask) # If some segment is empty, do small fix to avoid error in cocoapi function if len(converted_polygons[i]['points']) == 0: converted_polygons[i]['points'] = [empty_polygon] rle_bottom = mask_util.frPyObjects( converted_polygons[i]['points'], height, width) if not segment_overlapped: converted_polygons[i]['points'] = [converted_polygons[i]['points']] output_polygons = [] for i in range(0, len(converted_polygons)): if not use_background_label and converted_polygons[i][ 'label'] == 'background': continue poly_len = len(converted_polygons[i]['points']) if poly_len == 0 or converted_polygons[i]['points'] == [empty_polygon]: log.warning('Image <{}> has an empty polygon with label <{}>. ' 'Perhaps there is a mistake in annotation'.format( img_name, converted_polygons[i]['label'])) else: output_polygons.append(converted_polygons[i]) return output_polygons
def insert_categories_data(xml_root, use_background_label, result_annotation, labels_file=None): """Get labels from input annotation and fill categories field in output annotation Args: xml_root: root for xml parser use_background_label: key to enable using label background result_annotation: output annotation in COCO representation labels_file: path to file with labels names. If not defined, parse annotation to get labels names """ def get_categories(names, bg_found, use_background_label, sort=False): bg_used = False category_map = {} categories = [] # Sort labels by its names to make the same order of ids for different annotations if sort: names.sort() # Always use id = 0 for background if bg_found and use_background_label: category_map['background'] = 0 bg_used = True cat_id = 1 # Define id for all labels beginning from 1 for name in names: if name == 'background': continue category_map[name] = cat_id categories.append({'id': cat_id, 'name': name, 'supercategory': ''}) cat_id += 1 return category_map, categories, bg_used categories = [] category_map = {} label_names = [] bg_found = False bg_used = False if labels_file is None: log.info('Reading labels from annotation...') for label in xml_root.iter('label'): for name in label.findall("./name"): if name.text == 'background': bg_found = True continue label_names.append(name.text) if len(label_names) == 0: log.info('Labels in annotation were not found. Please use \'--labels\' argument to define file with labels.') else: category_map, categories, bg_used = get_categories(label_names, bg_found, use_background_label, sort=True) else: log.info('Parsing labels from file <{}>...'.format(labels_file)) with open(labels_file, 'r') as file: string = ' ' while string != '' and string != '\n': string = file.readline() labels = string.split(' ') for label in labels: if label == '\n' or label == '': continue label = label.replace('\n', '') if label == 'background': bg_found = True continue label_names.append(label) category_map, categories, bg_used = get_categories(label_names, bg_found, use_background_label) if len(categories) == 0: raise ValueError('Categories list is empty. Something wrong.') result_annotation['categories'] = categories log.info('Found the next labels: {}'.format(category_map)) if bg_found and not bg_used: log.warning('Label <background> was found but not used. ' 'To enable it should use command line argument [--use_background_label]') return category_map
def main(): args = parse_args() xml_file_name = args.cvat_xml output_file_name = args.output root = etree.parse(xml_file_name).getroot() if args.draw != None: log.info( 'Draw key was enabled. Images will be saved in directory <{}>'. format(args.draw)) result_annotation = { 'licenses': [], 'info': {}, 'categories': [], 'images': [], 'annotations': [] } insert_license_data(result_annotation) insert_info_data(root, result_annotation) category_map = insert_categories_data(root, args.use_background_label, result_annotation, osp.dirname(xml_file_name)) if len(category_map) == 0: sys.exit( 'Labels were not found. Be sure that annotation <{}> includes field <labels> or ' 'annotation directory includes file <labels.txt>'.format( xml_file_name)) segm_id = 0 z_order_off_counter = 0 # Parse original annotation for img in tqdm(root.iter('image'), desc='Processing images from ' + xml_file_name): image = {} for key, value in img.items(): image[key] = value image['polygon'] = [] z_order_on_counter = 0 polygon_counter = 0 for poly in img.iter('polygon'): polygon = {} for key, value in poly.items(): polygon[key] = value if key == 'z_order': z_order_on_counter += 1 polygon_counter += 1 image['polygon'].append(polygon) # If at least one of polygons on image does not have field 'z_order' do not sort them if z_order_on_counter == polygon_counter: image['polygon'].sort(key=lambda x: int(x['z_order'])) else: z_order_off_counter += 1 # Create new image insert_image_data(image, args.image_dir, result_annotation) height = result_annotation['images'][-1]['height'] width = result_annotation['images'][-1]['width'] image['polygon'] = fix_segments_intersections( image['polygon'], height, width, image['name'], args.use_background_label) # Create new annotation for this image for poly in image['polygon']: insert_annotation_data(image, category_map, segm_id, poly, [height, width], result_annotation) segm_id += 1 # Draw contours of objects on image if args.draw != None: draw_polygons(image['polygon'], image['name'], args.image_dir, args.draw, args.draw_labels) log.info('Processed images: {}'.format(len(result_annotation['images']))) log.info('Processed objects: {}'.format( len(result_annotation['annotations']))) if z_order_off_counter > 0: log.warning( 'Annotation does not have a field \'z_order\' for {} image(s). ' 'Overlapped objects may be cropped incorrectly!'.format( z_order_off_counter)) # Save created annotation log.info('Saving annotation...') with open(output_file_name, 'w') as outfile: json.dump(result_annotation, outfile) log.info( 'Annotation was saved in <{}> successfully'.format(output_file_name)) # Try to load created annotation via cocoapi try: log.info('Trying to load annotation <{}> via cocoapi...'.format( output_file_name)) anno = coco_loader.COCO(output_file_name) except: raise else: log.info( 'Annotation in COCO representation <{}> created from <{}> successfully!' .format(output_file_name, xml_file_name))
def watch(endpoint, key): """watch sends watch request to etcd. Examples: curl -L http://localhost:2379/v3alpha/watch \ -X POST -d ''{"create_request": {"key":"Zm9v"} }' """ # Python 2 # key_str = base64.b64encode(key) # Python 3 base64 requires utf-08 encoded bytes # Python 3 JSON encoder requires string key_str = base64.b64encode(bytes(key, "utf-8")).decode() req = {'create_request': {"key": key_str}} while True: try: rresp = requests.post(endpoint + '/v3alpha/watch', data=json.dumps(req), stream=True) for line in rresp.iter_lines(): # filter out keep-alive new lines if line: decoded_line = line.decode('utf-8') resp = json.loads(decoded_line) if 'result' not in resp: log.warning('{0} does not have result'.format(resp)) return '' if 'created' in resp['result']: if resp['result']['created']: log.warning('watching {0}'.format(key)) continue if 'events' not in resp['result']: log.warning('{0} returned no events: {1}'.format( key, resp)) return None if len(resp['result']['events']) != 1: log.warning('{0} returned >1 event: {1}'.format( key, resp)) return None if 'kv' in resp['result']['events'][0]: if 'value' in resp['result']['events'][0]['kv']: val = resp['result']['events'][0]['kv']['value'] return base64.b64decode(val) else: log.warning('no value in ', resp) return None else: log.warning('no kv in ', resp) return None except requests.exceptions.ConnectionError as err: log.warning('Connection error: {0}'.format(err)) time.sleep(5) except: log.warning('Unexpected error:', sys.exc_info()[0]) raise