def _write_coco_keypoint_results(self, keypoints, res_file): data_pack = [ { "cat_id": self._class_to_coco_ind[cls], "cls_ind": cls_ind, "cls": cls, "ann_type": "keypoints", "keypoints": keypoints, } for cls_ind, cls in enumerate(self.classes) if not cls == "__background__" ] results = self._coco_keypoint_results_one_category_kernel(data_pack[0]) logger.info("=> writing results json to %s" % res_file) with open(res_file, "w") as f: json.dump(results, f, sort_keys=True, indent=4) try: json.load(open(res_file)) except Exception: content = [] with open(res_file, "r") as f: for line in f: content.append(line) content[-1] = "]" with open(res_file, "w") as f: for c in content: f.write(c)
def _write_coco_keypoint_results(self, keypoints, res_file): data_pack = [ { 'cat_id': self._class_to_coco_ind[cls], 'cls_ind': cls_ind, 'cls': cls, 'ann_type': 'keypoints', 'keypoints': keypoints } for cls_ind, cls in enumerate(self.classes) if not cls == '__background__' ] results = self._coco_keypoint_results_one_category_kernel(data_pack[0]) logger.info('=> Writing results json to %s' % res_file) with open(res_file, 'w') as f: json.dump(results, f, sort_keys=True, indent=4) try: json.load(open(res_file)) except Exception: content = [] with open(res_file, 'r') as f: for line in f: content.append(line) content[-1] = ']' with open(res_file, 'w') as f: for c in content: f.write(c)
def generate_tasks(self, task_result: TaskResult) -> List[Task]: # initial/update temp config list if not self._temp_config_list: self._init_temp_config_list() else: score = self._tasks[task_result.task_id].score assert score is not None, 'SimulatedAnnealingTaskGenerator need each score is not None.' if not self._current_sparsity_list: self._current_sparsity_list = deepcopy(self._temp_sparsity_list) self._current_score = score else: delta_E = np.abs(score - self._current_score) probability = np.exp(-1 * delta_E / self.current_temperature) if self._current_score < score or np.random.uniform(0, 1) < probability: self._current_score = score self._current_sparsity_list = deepcopy(self._temp_sparsity_list) self.current_temperature *= self.cool_down_rate if self.current_temperature < self.stop_temperature: return [] self._update_with_perturbations() task_id = self._task_id_candidate new_config_list = self._recover_real_sparsity(deepcopy(self._temp_config_list)) config_list_path = Path(self._intermediate_result_dir, '{}_config_list.json'.format(task_id)) with Path(config_list_path).open('w') as f: json_tricks.dump(new_config_list, f, indent=4) task = Task(task_id, self.temp_model_path, self.temp_masks_path, config_list_path) self._tasks[task_id] = task self._task_id_candidate += 1 return [task]
def cli(train_size, test_size, output_file): result = preprocess_imdb(train_size=train_size, test_size=test_size) if output_file: json.dump(result, output_file, **JSON_ARGS) else: print(json.dumps(result, **JSON_ARGS))
def _dump_tasks_info(self): tasks = { task_id: task.to_dict() for task_id, task in self._tasks.items() } with Path(self._log_dir_root, '.tasks').open('w') as f: json_tricks.dump(tasks, f, indent=4)
def _write_keypoint_results(self, keypoints, output_file): # TODO turn list into numpy arrays if output_file.endswith('.json'): # Convert numpy arrays to Python lists for img_name, poses in keypoints.items(): for pose in poses: pose['center'] = pose['center'].tolist() pose['scale'] = pose['scale'].tolist() pose['keypoints'] = pose['keypoints'].ravel().tolist() pose['origbox'] = pose['origbox'].tolist() with open(output_file, 'w') as f: json.dump(keypoints, f, sort_keys=True, indent=4) elif output_file.endswith('npy'): frame_ind = keypoints.keys() assert all([f.startswith('videocap#') for f in frame_ind]) frame_ind = sorted(frame_ind, key=lambda x: int(x[len('videocap#'):])) kps = [] for f in frame_ind: assert len( keypoints[f] ) == 1, 'Only images with a single pose are supported in numpy save mode, found: ' + str( keypoints[f]) kps.append(keypoints[f][0]['keypoints']) kps = np.stack(kps, axis=0) print("shape:" + str(kps.shape)) np.save(output_file, kps) else: raise NotImplementedError('Unknown file ending: ' + output_file)
def compute_labels_and_activations(model, dataset, files, batch_size, save_path=None): #added files """ Load labels and compute activations with provided model on provided data loader Optionally store them. :param model: tf.keras model :param dataset: TrainingDataset (non-shuffleable to ensure that labels and activations match) :param save_path: if provided, store the labels and activations in json format. Use load_labels_and_activations to reload them without needing to recompute them. :return: labels, activations (numpy arrays) """ assert not dataset._do_shuffle, "Use non-shuffleable dataset for ensuring labels and predictions match" labels = load_labels(files) tf_dataset = dataset.dataset() tf_dataset = tf_dataset.batch(batch_size) #### ,drop_remainder=True ??? tf_dataset = tf_dataset.repeat() tf_dataset = tf_dataset.prefetch(tf.contrib.data.AUTOTUNE) activations = model.predict(tf_dataset, steps=dataset.num_samples() // batch_size + 1, verbose=1).squeeze() if save_path: results = {'labels': labels, 'activations': activations} with open(save_path, 'w') as json_file: jt.dump(results, json_file) return labels, activations
def log_results(results, output_dir): ''' .. versionchanged:: 1.28 Use json_tricks.dumps_ to dump results. .. _json_tricks.dumps: http://json-tricks.readthedocs.io/en/latest/#dumps Parameters ---------- results : dict Test results. output_dir : str Path to output directory. ''' output_dir = ph.path(output_dir) # Make output directory if it doesn't exist. output_dir.makedirs_p() # Construct filename based on current UTC date and time. filepath = output_dir.joinpath( 'results-%s.json' % dt.datetime.utcnow().isoformat().replace(':', '.')) # write the results to a file with filepath.open('w') as output: # XXX Use `json_tricks` rather than standard `json` to support # serializing [Numpy arrays and scalars][1]. # # [1]: http://json-tricks.readthedocs.io/en/latest/#numpy-arrays json_tricks.dump(results, output)
def _save_as_v1(self, fp): """Save copy to specified path. This mainly exists for testing and making sure we still read the old saved files. """ def _rewrite_dir(state): "Rewrite directories in the old format." state['files'] = [x[0] for x in state['files']] state['directories'] = [_rewrite_dir(d) for d in state['directories']] state.pop('relpath') state.pop('name') return state fp = open_file(fp, 'wb') media = [(key, self.get(key).to_dict()) for key in self._relpath2index] tags = [(t.name, t.type) for t in self.tags] root = _rewrite_dir(self.root.__getstate__()) processors = [processor.dump(x) for x in self.processors] for k, m in media: m['_ctime'] = long_to_datetime(m['_ctime']) m['_mtime'] = long_to_datetime(m['_mtime']) data = dict( version=1, path=self.path, name=self.name, description=self.description, tags=tags, media=media, root=root, processors=processors ) json_tricks.dump(data, fp, compression=True) fp.close() logger.info('Saved project: %s', self.name)
def cli(data_file, output_file): data = json.loads(data_file.read()) model = baseline_model(data["train_texts"], data["train_labels"]) args = (model, data["test_texts"], data["test_labels"]) if output_file: json.dump(evaluate(*args, output_dict=True), output_file) else: print(evaluate(*args))
def write_file(self): '''save config to local file''' try: with open(self.experiment_file, 'w') as file: json_tricks.dump(self.experiments, file, indent=4) except IOError as error: print('Error:', error) return ''
def save_model(self, file_name): file_extension = os.path.splitext(file_name)[1] filename = file_name if len(file_extension) > 0 else file_name+'.roa' fs_dict = {} fs_dict['optical_model'] = self with open(filename, 'w') as f: json_tricks.dump(fs_dict, f, indent=1, separators=(',', ':'), allow_nan=True)
def dump_measurement(self, measurement, data, title='dump'): fname = '{0}_{1}.json'.format(title, self._counter) path = join(self._output_directory, fname) results = dict() results[measurement] = self.analyze_measurement(measurement, data) with open(path, 'w') as f: dump(results, f) self._counter += 1 return path
def save_annotations(annotations, opt): os.makedirs(opt.annot_folder, exist_ok=True) for a in annotations: fname = get_output_filename(a, opt.annot_folder, 'json') with open(fname, 'w') as f: dump(a, f, indent=2) print(f'saved annotations to "{opt.annot_folder}" directory')
def save_agents(self, name) -> None: data = { 'agents': [agent.__class__.__name__ for ID, agent in self.agents.items()] } filename = f'saves/{name}.agents.json' with open(filename, mode='w') as file: jt.dump(data, file, indent=4)
def cli(data_file, model_file, output_file): data = json.load(data_file) model = BertBinaryClassifier().cuda() model.load_state_dict(torch.load(model_file)) predictions, _ = predict(model, data["test_tokens_ids"], BATCH_SIZE) json.dump( classification_report(data["test_y"], predictions, output_dict=True), output_file, **JSON_ARGS)
def _saveJSON(self): thisFileName = os.path.join(monitorFolder, self.name + ".json") # convert time structs to timestamps (floats) for calibName in self.calibs: calib = self.calibs[calibName] if isinstance(calib['calibDate'], time.struct_time): calib['calibDate'] = time.mktime(calib['calibDate']) with open(thisFileName, 'w') as outfile: json_tricks.dump(self.calibs, outfile, indent=2, allow_nan=True)
def save_json(obj, outfile, allow_nan=True, compression=False): """Save an ssbio object as a JSON file using json_tricks""" if compression: with open(outfile, 'wb') as f: dump(obj, f, allow_nan=allow_nan, compression=compression) else: with open(outfile, 'w') as f: dump(obj, f, allow_nan=allow_nan, compression=compression) log.info('Saved {} (id: {}) to {}'.format(type(obj), obj.id, outfile))
def track_with_manual_label_all(des_dir, now_split=0, label_name='object', global_tracker_=''): global global_tracker global_tracker = global_tracker_ info_list = generate_tracking_info(des_dir) info_list = info_list[now_split:] for info in info_list: print("NOW split:{}".format(now_split)) frame_split_list, start_bbox, end_bbox = info r1, forward_bbox_l = track_with_manual_label(frame_split_list, start_bbox, now_split) r2, backward_bbox_l = track_with_manual_label(frame_split_list[::-1], end_bbox, now_split) now_split += 1 if r1 == False or r2 == False: # this will make it this sequence unchanged continue forward_bbox_l = forward_bbox_l[: -1] # drop the last one, since it is manual labeled backward_bbox_l = backward_bbox_l[:-1] backward_bbox_l_to_forward = backward_bbox_l[::-1] for frame_f, forward_b, backward_b in zip(frame_split_list[1:-1], forward_bbox_l, backward_bbox_l_to_forward): print("NOW file:{}".format(frame_f)) f_basename, f_no_suffix = split_the_abs_filename(frame_f) des_f = des_dir + '/' + f_no_suffix + '.json' with open(des_f, 'r') as f: info = json_tricks.load(f) assert 'manual_label' not in info or not info['manual_label'] info['manual_label'] = False info['track_label'] = True info['label_list'] = [label_name] info['forward_bbox'] = forward_b info['backward_bbox'] = backward_b info['BBox_list'] = [(forward_b + backward_b) / 2] print(info['BBox_list'][0].shape) assert info['BBox_list'][0].shape == (4, 2) with open(des_f, 'w') as f: json_tricks.dump(info, f, sort_keys=True, indent=4)
def saveJSON(self): thisFileName = os.path.join(monitorFolder, self.name + ".json") # convert time structs to timestamps (floats) for calibName in self.calibs: calib = self.calibs[calibName] if isinstance(calib['calibDate'], time.struct_time): calib['calibDate'] = time.mktime(calib['calibDate']) with open(thisFileName, 'w') as outfile: json_tricks.dump(self.calibs, outfile, indent=2, allow_nan=True)
def write_json(json_obj, file_name): """ Writes json object to a json file :param json_obj: json object :param file_name: str :return: None """ with open(file_name, 'w') as f: json_tricks.dump(json_obj, f, indent=3) return None
def save(self): store_dict = { k: v for k, v in self.__dict__.items() if k in self.store_variables } filename = get_next_filename(self.storage_path, self.base_filename, extension="json") json_tricks.dump(store_dict, filename) print(f"Stored results in {filename} at {datetime.datetime.now()}")
def save_to_json(obj: (list, dict), path=None, fp=None, file_name=None): jsonable_dict = convert_to_jsonable(dict_or_list=obj) if fp: json.dump(obj=jsonable_dict, fp=fp, indent=4, sort_keys=True) fp.close() else: if file_name is not None: path = os.path.join(path, file_name) with open(path, 'w') as f: json.dump(obj=obj, fp=f, indent=4, sort_keys=True)
def _save_data(self, folder_name: str, model: Module, masks: Dict[str, Dict[str, Tensor]], config_list: List[Dict]): Path(self._log_dir_root, folder_name).mkdir(parents=True, exist_ok=True) torch.save(model, Path(self._log_dir_root, folder_name, 'model.pth')) torch.save(masks, Path(self._log_dir_root, folder_name, 'masks.pth')) with Path(self._log_dir_root, folder_name, 'config_list.json').open('w') as f: json_tricks.dump(config_list, f, indent=4)
def write_json(self, filename): data = {} data["map_obstacle"] = self.get_map_numpy() data["list_cities"] = self.get() print(type(data["map_obstacle"])) print(type(data["list_cities"])) # print(json_tricks.dumps(data, indent=4)) with open(filename, "w") as json_file: json_tricks.dump(data, fp=json_file, indent=4)
def _do_python_keypoint_eval(self, res_file, keypoint_accs, cfg): coco_dt = self.coco.loadRes(res_file) coco_eval = CustomEval(self.coco, coco_dt, 'keypoints') coco_eval.params.useSegm = None part, tfms_name, crop_size = self._get_occluded(cfg) # coco_eval.params.occlusion = part stats_names = ['AP', 'Ap .5', 'AP .75', 'AP (M)', 'AP (L)', 'AR', 'AR .5', 'AR .75', 'AR (M)', 'AR (L)'] ## Per keypoint evaluation metrics = {} if self.per_keypoint_eval: for i, keypoint in enumerate(keypoint_names["coco"]): coco_eval.params.keypoint = keypoint coco_eval.params.print = False coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() info_str = [] for ind, name in enumerate(stats_names): info_str.append((name, coco_eval.stats[ind])) info_str.append(("[email protected]", keypoint_accs[i + 1].avg)) metrics[keypoint] = dict(info_str) # All keypoint evaluation coco_eval.params.keypoint = "all" coco_eval.params.print = True coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() info_str = [] for ind, name in enumerate(stats_names): info_str.append((name, coco_eval.stats[ind])) info_str.append(("[email protected]", keypoint_accs[0].avg)) if self.per_keypoint_eval: metrics["all"] = dict(info_str) file_path = self._create_results_save_folder() if crop_size != 0: file_path = os.path.join(file_path, "crop_size_"+str(crop_size)) if not os.path.isdir(file_path): os.mkdir(file_path) filename = part + '_' + tfms_name + '.json' file_path = os.path.join(file_path, filename) print(file_path) with open(file_path, 'w', encoding='utf-8') as f: json.dump(metrics, f, ensure_ascii=False, indent=4) return info_str
def cli(texts_file, model_file, output_file, batch_size): texts: List[str] = json.load(texts_file) _, tokens_ids = tokenize(texts) model = BertBinaryClassifier().cuda() model.load_state_dict(torch.load(model_file)) predictions, logits = predict(model, tokens_ids, batch_size) json.dump({ "predictions": predictions, "logits": logits }, output_file, **JSON_ARGS)
def save_calibration(channels, filename): from psi.util import get_tagged_values from json_tricks import dump settings = {} for channel in channels: metadata = get_tagged_values(channel.calibration, 'metadata') metadata['calibration_type'] = channel.calibration.__class__.__name__ if 'source' in metadata: metadata['source'] = str(metadata['source']) settings[channel.name] = metadata with open(filename, 'w') as fh: dump(settings, fh, indent=4)
def main(args): paths = list_dir(args.data_dir) for path in paths: print(path) x = loadmat(path) # load mat file # x = h5py.File(path, 'r') # tables.openFile(path) # x = mat4py.loadmat(path) print(x["annolist"]["annorect"]) json_fn = os.path.splitext(path)[0] + ".json" with open(json_fn, 'wt') as json_f: json.dump(x, json_f)
def change_des_base_folder(des_folder, old_base_folder, new_base_folder, win2linux=True): des_all_files = preprocess.generate_all_abs_filenames(des_folder) for des_f in des_all_files: print("{} ...".format(des_f)) with open(des_f, 'r') as f: info = json_tricks.load(f) abs_file_name = info['abs_file_name'] # type: str abs_file_name = change_file_prefix(abs_file_name, old_base_folder, new_base_folder, win2linux=True) info['abs_file_name'] = abs_file_name with open(des_f, 'w') as f: json_tricks.dump(info, f, sort_keys=True, indent=4)
def startBag(self): self.bagger() self.metadata = self.metadataGen() # pickler(self.metadata, self.bagFilename) try: pickler(self.metadata, self.bagFilename + '.pickle') with open(self.bagFilename + ".json", 'w') as outfile: json.dump(self.metadata, outfile, indent=4, sort_keys=True, separators=(',', ':'),ensure_ascii=False) except UnicodeDecodeError as e: print "error with unicode FIX IT", e pass
def _write_coco_keypoint_results(self, keypoints, res_file): data_pack = [{ 'cat_id': self._class_to_coco_ind[cls], 'cls_ind': cls_ind, 'cls': cls, 'ann_type': 'keypoints', 'keypoints': keypoints } for cls_ind, cls in enumerate(self.classes) if not cls == '__background__'] results = self._coco_keypoint_results_one_category_kernel(data_pack[0]) logger.info('=> Writing results json to %s' % res_file) with open(res_file, 'w') as f: json.dump(results, f, sort_keys=True, indent=4)
def write_json(data, path, file_name): """ Write out data to a json file. Args: data: A dictionary representation of the data to write out path: The directory to output the file in file_name: The name of the file to write out """ if os.path.exists(path) and not os.path.isdir(path): return elif not os.path.exists(path): mkdir_p(path) with open(os.path.join(path, file_name), 'w') as f: json_tricks.dump(data, f, indent=4, primitives=True, allow_nan=True)
def saveAsJson(self, fileName=None, encoding='utf-8', fileCollisionMethod='rename'): """ Serialize the object to the JSON format. Parameters ---------- fileName: string, or None the name of the file to create or append. Can include a relative or absolute path. If `None`, will not write to a file, but return an in-memory JSON object. encoding : string, optional The encoding to use when writing the file. This parameter will be ignored if `append` is `False` and `fileName` ends with `.psydat` or `.npy` (i.e. if a binary file is to be written). fileCollisionMethod : string Collision method passed to :func:`~psychopy.tools.fileerrortools.handleFileCollision`. Can be either of `'rename'`, `'overwrite'`, or `'fail'`. Notes ----- Currently, a copy of the object is created, and the copy's .origin attribute is set to an empty string before serializing because loading the created JSON file would sometimes fail otherwise. """ self_copy = copy.deepcopy(self) self_copy.origin = '' msg = ('Setting attribute .origin to empty string during JSON ' 'serialization.') logging.warn(msg) if (fileName is None) or (fileName == 'stdout'): return json_tricks.dumps(self_copy) else: with openOutputFile(fileName=fileName, fileCollisionMethod=fileCollisionMethod, encoding=encoding) as f: json_tricks.dump(self_copy, f) logging.info('Saved JSON data to %s' % f.name)
ax[2].grid() ax[2].legend(frameon=False) ax[2].set_xlabel('Wavelength [nm]') ax[2].set_xlim(250,5000) ax[2].set_xticks([320,400,500,600,750,1000,1240,1600,2500,4000]) ax[2].set_xticklabels([320,400,500,600,750,1000,1240,1600,2500,4000]) plt.savefig('/mnt/c/Users/sleblanc/Research/ORACLES/aero_v4_yr.png',dpi=600,transparent=True) # In[78]: n['asy'][0,:] = new_asy n['ssa'][0,:] = new_ssa n['ext'][0,:] = new_ext # In[79]: help(dump) # In[80]: dump(n,'/mnt/c/Users/sleblanc/Research/ORACLES/aero_file_v4.txt')