def make_dds(config, id_symdds, outdir): dds = instance_dds(config, id_symdds) basename = os.path.join(outdir, id_symdds) filename_pickle = basename + '.discdds.pickle' logger.info('Writing to %r ' % friendly_path(filename_pickle)) safe_pickle_dump(dds, filename_pickle) filename_yaml = basename + '.discdds.yaml' description = { 'id': id_symdds, 'desc': 'Synthetically generated from symbolic DDS %r.' % id_symdds, 'code': [ 'diffeoplan.library.load_pickle', { 'file:pickle': id_symdds + '.discdds.pickle' } ] } logger.info('Writing to %r ' % friendly_path(filename_yaml)) with open(filename_yaml, 'w') as f: yaml.dump([description], f, default_flow_style=False, explicit_start=True)
def set(self, key, value): # @ReservedAssignment """ Return a dictionary with some statistics """ if not StorageFilesystem.checked_existence: StorageFilesystem.checked_existence = True if not os.path.exists(self.basepath): os.makedirs(self.basepath) # TODO: generalize this filename = self.filename_for_key(key) with warn_long_time(self.warn_long_time, 'dumping %r' % key) as moreinfo: protocol = HIGHEST_PROTOCOL paranoid = False if paranoid: safe_pickle_dump(value, filename, protocol) else: with open(filename, 'wb') as f: pickle.dump(value, f, protocol) moreinfo['size'] = os.stat(filename).st_size # TODO: remove this stats = {} stats['duration'] = 0 # XXX stats['clock'] = 0 # XXX stats['size'] = os.stat(filename).st_size return stats
def get_servo_commands(self, boot_data): self.y = boot_data['observations'] plus = 1 inst = self.get_closest_point(self.y) if self.index_cur is None: # self.index_cur = inst self.index_cur = self.get_initial_closest_point(self.y) self.index_target = self.index_cur + plus if self.index_cur >= self.nmap.npoints() - 1 - plus: self.info('starting from beginning') self.index_cur = 0 index_next = self.index_cur + 1 # if we are closer to target d_cur = self.get_distance( self.y, self.nmap.get_observations_at(self.index_cur)) d_next = self.get_distance(self.y, self.nmap.get_observations_at(index_next)) ratio = d_next / d_cur if False: print('ratio: %1.4f <= %1.4f d_next: %1.4f < %1.4f' % (ratio, self.ratio_threshold, d_next, self.d_next_threshold)) if (d_next < self.ratio_threshold * d_cur and d_next < self.d_next_threshold) or \ d_next < self.d_next_threshold_alone: self.info('closer to next than to cur') self.index_cur += 1 self.index_target = self.index_cur + plus indices = dict(inst=inst, cur=self.index_cur, next=index_next, target=self.index_target) print(self.get_navigation_status_string(self.y, indices)) goal = self.nmap.get_observations_at(self.index_target) self.set_goal_observations(goal) safe_pickle_dump(goal, ServoDemo2.state_filename) if self.started_now or self.e0 is None: # First iteration with new goal self.e0 = self.get_distance_to_goal(self.y) self.started_now = False self.servo_agent.process_observations(boot_data) u = self.servo_agent.choose_commands() self.boot_spec.get_commands().check_valid_value(u) return u
def get_servo_commands(self, boot_data): self.y = boot_data['observations'] plus = 1 inst = self.get_closest_point(self.y) if self.index_cur is None: # self.index_cur = inst self.index_cur = self.get_initial_closest_point(self.y) self.index_target = self.index_cur + plus if self.index_cur >= self.nmap.npoints() - 1 - plus: self.info('starting from beginning') self.index_cur = 0 index_next = self.index_cur + 1 # if we are closer to target d_cur = self.get_distance(self.y, self.nmap.get_observations_at(self.index_cur)) d_next = self.get_distance(self.y, self.nmap.get_observations_at(index_next)) ratio = d_next / d_cur if False: print('ratio: %1.4f <= %1.4f d_next: %1.4f < %1.4f' % (ratio, self.ratio_threshold, d_next, self.d_next_threshold)) if (d_next < self.ratio_threshold * d_cur and d_next < self.d_next_threshold) or \ d_next < self.d_next_threshold_alone: self.info('closer to next than to cur') self.index_cur += 1 self.index_target = self.index_cur + plus indices = dict(inst=inst, cur=self.index_cur, next=index_next, target=self.index_target) print(self.get_navigation_status_string(self.y, indices)) goal = self.nmap.get_observations_at(self.index_target) self.set_goal_observations(goal) safe_pickle_dump(goal, ServoDemo2.state_filename) if self.started_now or self.e0 is None: # First iteration with new goal self.e0 = self.get_distance_to_goal(self.y) self.started_now = False self.servo_agent.process_observations(boot_data) u = self.servo_agent.choose_commands() self.boot_spec.get_commands().check_valid_value(u) return u
def make_dds(config, id_symdds, outdir): dds = instance_dds(config, id_symdds) basename = os.path.join(outdir, id_symdds) filename_pickle = basename + '.discdds.pickle' logger.info('Writing to %r ' % friendly_path(filename_pickle)) safe_pickle_dump(dds, filename_pickle) filename_yaml = basename + '.discdds.yaml' description = { 'id': id_symdds, 'desc': 'Synthetically generated from symbolic DDS %r.' % id_symdds, 'code': ['diffeoplan.library.load_pickle', {'file:pickle': id_symdds + '.discdds.pickle'}] } logger.info('Writing to %r ' % friendly_path(filename_yaml)) with open(filename_yaml, 'w') as f: yaml.dump([description], f, default_flow_style=False, explicit_start=True)
def save(self, outdir): """ Creates outdir/<id_tc>.pickle and outdir/<>.yaml """ filename_pickle = self.id_tc + '.tc.pickle' filename_yaml = self.id_tc + '.tc.yaml' description = { 'id': self.id_tc, 'desc': 'Automatically generated test case', 'code': ['diffeoplan.library.load_pickle', {'file:pickle': filename_pickle}] } filename_pickle = os.path.join(outdir, filename_pickle) filename_yaml = os.path.join(outdir, filename_yaml) logger.info('Writing to %r ' % friendly_path(filename_pickle)) safe_pickle_dump(self, filename_pickle) write_entries([description], filename_yaml)
def save_report(data_central, report, filename, resources_dir=None, save_pickle=False, save_hdf=True, check_hdf_written_correctly=True): """ filename.html """ report.text('report_date', isodate()) # TODO: add other stuff ds = data_central.get_dir_structure() report.to_html(filename, resources_dir=resources_dir) ds.file_is_done(filename) if save_pickle: pickle_name = os.path.splitext(filename)[0] + '.pickle' with warn_long_time_writing(pickle_name): safe_pickle_dump(report, pickle_name, protocol=2) ds.file_is_done(pickle_name) if save_hdf: hdf_name = os.path.splitext(filename)[0] + '.rr1.h5' with warn_long_time_writing(hdf_name): report.to_hdf(hdf_name) if check_hdf_written_correctly: with warn_long_time_reading(hdf_name, logger=logger): r2 = report_from_hdf(hdf_name) if report != r2: logger.error("Dont match") logger.error(report.format_tree()) logger.error(r2.format_tree()) raise Exception() ds.file_is_done(hdf_name)
def save(self, outdir): """ Creates outdir/<id_tc>.pickle and outdir/<>.yaml """ filename_pickle = self.id_tc + '.tc.pickle' filename_yaml = self.id_tc + '.tc.yaml' description = { 'id': self.id_tc, 'desc': 'Automatically generated test case', 'code': [ 'diffeoplan.library.load_pickle', { 'file:pickle': filename_pickle } ] } filename_pickle = os.path.join(outdir, filename_pickle) filename_yaml = os.path.join(outdir, filename_yaml) logger.info('Writing to %r ' % friendly_path(filename_pickle)) safe_pickle_dump(self, filename_pickle) write_entries([description], filename_yaml)
def index_file_cached(self, filename, ignore_cache=False): cache = '%s.index_cache' % filename if os.path.exists(cache) and not ignore_cache: # TODO: mtime try: return safe_pickle_load(cache) except Exception as e: msg = 'Could not unpickle cache %r, deleting.' % friendly_path(cache) msg += '\n%s' % e logger.warning(msg) try: os.unlink(cache) except: pass logger.debug('Indexing file %r' % friendly_path(filename)) res = self.index_file(filename) for stream in res: assert isinstance(stream, BootStream) logger.debug('Now dumping file %r' % friendly_path(cache)) with warn_long_time(1, 'dumping %r' % friendly_path(cache)): safe_pickle_dump(res, cache, protocol=2) return res
def save_map(nmap, filename): print('Saving to %r' % filename) safe_pickle_dump(nmap, filename)
def set_goal_observations(self, y): self.y_goal = y.copy() safe_pickle_dump(self.y_goal, ServoDemo2.state_filename) self.servo_agent.set_goal_observations(self.y_goal)