def request(): f = tempfile.NamedTemporaryFile(suffix='.h5', dir=tempfile.gettempdir(), delete=False) f.close() attempts = 0 while True: try: path = super(ParallelFeature, self).fetch(session, name, partial_resource_list, path=f.name) except socket.error as e: #if connection fails if attempts>MAX_ATTEMPTS: log.debug('Connection fail: Reached max attempts') break if e.errno == errno.WSAECONNRESET: attempts+=1 log.debug('Connection fail: Attempting to reconnect (try: %s)' % attempts) try: tables.is_pytables_file(path) except tables.HDF5ExtError: #if fail gets corrupts during download if attempts>MAX_ATTEMPTS: log.debug('Failed to open hdf5 file: Reached max attempts') break attempts+=1 log.debug('HDF5 file may be corrupted: Attempted to redownload (try: %s)' % attempts) if os.path.exists(path): os.remove(path) write_queue.put(path) break
def __init__(self, h5parmFile, readonly = True, complevel = 5, complib='zlib'): """ Keyword arguments: h5parmFile -- H5parm filename readonly -- if True the table is open in readonly mode (default=True) complevel -- compression level from 0 to 9 (default=5) when creating the file complib -- library for compression: lzo, zlib, bzip2 (default=zlib) """ if os.path.isfile(h5parmFile): if tables.is_pytables_file(h5parmFile) == None: logging.critical('Wrong HDF5 format for '+h5parmFile+'.') raise Exception('Wrong HDF5 format for '+h5parmFile+'.') if readonly: logging.debug('Reading from '+h5parmFile+'.') self.H = tables.openFile(h5parmFile, 'r') else: logging.debug('Appending to '+h5parmFile+'.') self.H = tables.openFile(h5parmFile, 'r+') else: if readonly: raise Exception('Missing file '+h5parmFile+'.') else: logging.debug('Creating '+h5parmFile+'.') # add a compression filter f = tables.Filters(complevel=complevel, complib=complib) self.H = tables.openFile(h5parmFile, filters=f, mode='w') self.fileName = h5parmFile
def __init__(self, filename, model_state, proposal_state): """ Initialize the object. """ self.filename = filename self.ChainRecordDType = state_to_table_dtype(model_state) self.ChainRecordDType['step'] = pt.UInt32Col() self.ChainRecordDType['accepted'] = pt.UInt32Col() self.ChainRecordDType['proposal'] = pt.UInt16Col() self.ProposalRecordDType = state_to_table_dtype(proposal_state) self.ChainCounterDType = {'id': pt.UInt16Col(), 'name': pt.StringCol(itemsize=32), 'date': pt.StringCol(itemsize=26) } if os.path.exists(filename) and pt.is_pytables_file(filename): self.fd = pt.open_file(filename, mode='a') else: self.fd = pt.open_file(filename, mode='w') self.fd.create_group('/', 'mcmc', 'Metropolis-Hastings Algorithm Data') self.fd.create_table('/mcmc', 'proposals', self.ProposalRecordDType, 'MCMC Proposals') self.fd.create_table('/mcmc', 'chain_counter', self.ChainCounterDType, 'Chain Counter') self.fd.create_group('/mcmc', 'data', 'Collection of Chains')
def __init__(self, h5parmFile, readonly=True, complevel=5, complib='lzo'): """ Keyword arguments: h5parmFile -- H5parm filename readonly -- if True the table is open in readonly mode (default=True) complevel -- compression level from 0 to 9 (default=5) when creating the file complib -- library for compression: lzo, zlib, bzip2 (default=lzo) """ if os.path.isfile(h5parmFile): if tables.is_pytables_file(h5parmFile) == None: logging.critical('Wrong HDF5 format for ' + h5parmFile + '.') raise Exception('Wrong HDF5 format for ' + h5parmFile + '.') if readonly: logging.debug('Reading from ' + h5parmFile + '.') self.H = tables.openFile(h5parmFile, 'r') else: logging.debug('Appending to ' + h5parmFile + '.') self.H = tables.openFile(h5parmFile, 'r+') else: if readonly: raise Exception('Missing file ' + h5parmFile + '.') else: logging.debug('Creating ' + h5parmFile + '.') # add a compression filter f = tables.Filters(complevel=complevel, complib=complib) self.H = tables.openFile(h5parmFile, filters=f, mode='w') self.fileName = h5parmFile
def __init__(self, filename, model_state, proposal_state): """ Initialize the object. """ self.filename = filename self.ChainRecordDType = state_to_table_dtype(model_state) self.ChainRecordDType['step'] = pt.UInt32Col() self.ChainRecordDType['accepted'] = pt.UInt32Col() self.ChainRecordDType['proposal'] = pt.UInt16Col() self.ProposalRecordDType = state_to_table_dtype(proposal_state) self.ChainCounterDType = { 'id': pt.UInt16Col(), 'name': pt.StringCol(itemsize=32), 'date': pt.StringCol(itemsize=26) } if os.path.exists(filename) and pt.is_pytables_file(filename): self.fd = pt.open_file(filename, mode='a') else: self.fd = pt.open_file(filename, mode='w') self.fd.create_group('/', 'mcmc', 'Metropolis-Hastings Algorithm Data') self.fd.create_table('/mcmc', 'proposals', self.ProposalRecordDType, 'MCMC Proposals') self.fd.create_table('/mcmc', 'chain_counter', self.ChainCounterDType, 'Chain Counter') self.fd.create_group('/mcmc', 'data', 'Collection of Chains')
def load_file(path, fileformat, mode='a'): '''Validates and loads a spectral document''' try: cache = path + CACHE[fileformat].suffix assert os.path.exists(cache) and tb.is_pytables_file(cache) return DOCUMENTS[fileformat].open(path, mode=mode) except IOError: raise AssertionError(exception.CODES['013']) except (ValueError, AssertionError, KeyError): raise AssertionError(exception.CODES['023'])
def load_file(path, fileformat, mode='a'): '''Validates and loads a spectral document''' try: cache = path + CACHE[fileformat].suffix assert os.path.exists(cache) and tb.is_pytables_file(cache) return DOCUMENTS[fileformat].open(path, mode=mode) except IOError: raise AssertionError(exception.CODES['013']) except (ValueError, AssertionError, KeyError): raise AssertionError(exception.CODES['023'])
def request(): f = tempfile.NamedTemporaryFile(suffix='.h5', dir=tempfile.gettempdir(), delete=False) f.close() attempts = 0 while True: try: path = super(ParallelFeature, self).fetch(session, name, partial_resource_list, path=f.name) except socket.error as e: #if connection fails if attempts > MAX_ATTEMPTS: log.debug('Connection fail: Reached max attempts') break if e.errno == errno.WSAECONNRESET: #pylint: disable=no-member attempts += 1 log.debug( 'Connection fail: Attempting to reconnect (try: %s)' % attempts) try: tables.is_pytables_file(path) except tables.HDF5ExtError: #if fail gets corrupts during download if attempts > MAX_ATTEMPTS: log.debug( 'Failed to open hdf5 file: Reached max attempts' ) break attempts += 1 log.debug( 'HDF5 file may be corrupted: Attempted to redownload (try: %s)' % attempts) if os.path.exists(path): os.remove(path) write_queue.put(path) break
def start(self): if not os.path.exists(self.input): self.error('No such file or directory: %s' % self.input) if not tables.is_pytables_file(self.input): self.error('Unrecognized format') try: ds = DataSet(self.input, 'r') self.print_dataset(ds) ds.close() except UnrecognizedFormatError: handle = tables.open_file(self.input) self.print_model(handle) handle.close()
def _open_database(self): test = tables.is_pytables_file(str(self.path)) if test <= 0: raise LineageError("File is not a pytables") if self.readonly: mode = 'r' else: mode = 'r+' log.debug("Opening {} in mode {}".format(self.path.name, mode)) h5 = tables.open_file(str(self.path), mode=mode) attrs = h5.root._v_attrs assert attrs.storage_class == self.__class__.__name__ # Assign the arrays ------------ self._h5 = h5 self._attrs = attrs self._networks = h5.root.networks self._generations = h5.root.generations
def __init__(self, store): """Load fragment database view over store.""" self.store = None if isinstance(store, basestring): self.logger.info("Opening file: %s", store) try: is_store = tables.is_pytables_file(store) except tables.HDF5ExtError: is_store = False if is_store: if os.access(store, os.W_OK): self.store = tables.open_file(store, "r+") else: self.store = tables.open_file(store, "r") else: raise ValueError("Store is not a PyTables file: %s" % store) elif isinstance(store, tables.file.File): self.logger.info("Opening : %s", store) self.store = store else: raise ValueError("Unable to open store: %s" % store)
def load_log(self, path, overwrite='raise', template=False): logfile = path filename = os.path.split(os.path.splitext(os.path.normpath(path))[0])[1] path = os.path.split(os.path.splitext(os.path.normpath(path))[0])[0] autosave_writer = self.autosave_writer try: log_temp = self.log_temp log_temp.copy_file(logfile, overwrite=(overwrite=='overwrite')) if not template: self.log = logfile if self.log_auto_dur: temp_file = tempfile.NamedTemporaryFile(dir=path, prefix=filename+'_', suffix='.autosave', delete=False) temp_name = temp_file.name temp_file.close() log_temp.copy_file(temp_name, overwrite=True) autosave_writer(temp_name) self.log_auto = temp_name self.unsaved_changes = False self.status_func(unsaved=False) if overwrite == 'overwrite': return [] score_groups = self.score_groups res = [None, ] * len(score_groups) for col in range(len(score_groups)): settings = {} score_group = score_groups[col] for setting in score_group._v_attrs._f_list('user'): settings[setting] = score_group._v_attrs[setting] res[col] = (col, settings) return res else: template_file = None try: template_file = tb.open_file(logfile, 'a') raw_group = template_file.root.raw_data for row in raw_group.pts._f_list_nodes(): template_file.remove_node(row, recursive=True) for group in raw_group._f_iter_nodes(): if group._v_name == 'pts' or group._v_attrs.score_type != 'xy': for row in group._f_list_nodes(): template_file.remove_node(row, recursive=True) return [] finally: if template_file: template_file.close() except IOError as e: # file exits and we don't overwrite if overwrite == 'raise' and not len(self.score_groups): overwrite = 'load' if overwrite == 'raise': log = None try: log = tb.open_file(logfile) cols = [node._v_name for node in log.root.raw_data._f_iter_nodes() if node._v_name != 'pts'] finally: if log: log.close() raise PyTrackException('log_res', logfile + ' already exists '+ 'with the following columns: [' + ' | '.join(cols) + ']. \n\nWhat would you like to do?\n-Load the file and overwrite the current '+ 'channels.\n-Merge the current channels with the file channels.\n-Overwrite the file with '+ 'the current channels (overwrites the file on disk).\n-Browse for another file.\n-Cancel.' + '\n\n Module error message:\n' + str(e)) if overwrite == 'merge' or template: if (not template) and not self.pts_list_complete: raise PyTrackException('error', 'Not all frames have been seen yet. You cannot '+ 'merge a log until the full video has been seen.') if tb.is_pytables_file(logfile): log = None try: log = tb.open_file(logfile) raw_group = log.root.raw_data video_group = log.root.video_info pts_group = raw_group.pts score_groups = [None,] * (raw_group._v_nchildren - 1) settings_list = list(score_groups) res = list(score_groups) for group in raw_group._f_iter_nodes(): if group._v_name != 'pts': score_groups[int(group._v_name.rpartition('_')[2])] = group for chan in range(len(score_groups)): settings = {} for setting in score_groups[chan]._v_attrs._f_list('user'): settings[setting] = score_groups[chan]._v_attrs[setting] settings_list[chan] = settings n_pts = len(pts_group._f_list_nodes()) data = [[None for j in range(len(score_groups)+1)] for i in range(max(1, n_pts))] for chan in range(len(score_groups)): for row in score_groups[chan]._f_iter_nodes(): data[int(row._v_name.rpartition('_')[2])][chan + 1] = row for row in pts_group._f_iter_nodes(): data[int(row._v_name.rpartition('_')[2])][0] = row dist_max = 0 sorted_pts_list = [] pts_list = self.pts_list self.get_closest_pts(sorted_pts_list) for chan in range(len(score_groups)): count = len(self.score_groups) settings = settings_list[chan] self.add_header(count, settings) if settings['score_type'] == 'xy' and data[0][chan+1] and len(data[0][chan+1]): self.add_data_range(count, list(data[0][chan+1])) elif not template: for i in range(len(data)): for j in range(len(data[i][chan+1]) if data[i][chan+1] else 0): pts = self.get_closest_pts(sorted_pts_list, data[i][0][j]) dist_max = max(dist_max, abs(pts-data[i][0][j])) matched_list, matched_idx = pts_list[pts] matched_list[1+count][matched_idx] = data[i][1+chan][j] res[chan] = count, settings finally: if log: log.close() else: with open(logfile, 'r') as csvfile: csvlog = [row for row in csv.reader(csvfile)] if not len(csvlog) or len(csvlog[0]) <= 1: raise Exception("There's nothing to read in the file.") if csvlog[0][0] != 'Time': raise Exception("The first column is time and its title must be Time.") pts = [float(t[0]) for t in csvlog[1:] if t[0]] i = 1 channels = [] while i < len(csvlog[0]): if csvlog[0][i].startswith('xyt'): if (i+1 >= len(csvlog[0]) or (not csvlog[0][i].startswith('xyt_x_')) or (not csvlog[0][i+1].startswith('xyt_y_'))): raise Exception("A xyt score must be a two column series, with the first "+ "column starting with xyt_x_, and the next column starting with "+ "xyt_y_. Text following xyt_x_ is the column name.") pos = [(int(t[i]), int(t[i+1])) for t in csvlog[1:] if t[i] and t[i+1]] if len(pos) != len(pts): raise Exception("Number of data points in a xyt channel must match the number of time points.") sett = TrackLog.get_default_settings('xyt') sett.update({'name':csvlog[0][i][6:]}) channels.append((pos, sett)) i += 2 continue elif csvlog[0][i].startswith('xy'): if (i+1 >= len(csvlog[0]) or (not csvlog[0][i].startswith('xy_x_')) or (not csvlog[0][i+1].startswith('xy_y_'))): raise Exception("A xy score must be a two column series, with the first "+ "column starting with xy_x_, and the next column starting with "+ "xy_y_. Text following xy_x_ is the column name.") pos = [(int(t[i]), int(t[i+1])) for t in csvlog[1:] if t[i] and t[i+1]] sett = TrackLog.get_default_settings('xy') sett.update({'name':csvlog[0][i][5:]}) channels.append((pos, sett)) i += 2 continue elif csvlog[0][i].startswith('t'): if (not csvlog[0][i].startswith('t_')): raise Exception("A t score is a column starting with t_. "+ "Text following t_ is the column name.") vals = [bool(t[i]) for t in csvlog[1:] if t[i]] if len(vals) != len(pts): raise Exception("Number of data points in a t channel must match the number of time points.") sett = TrackLog.get_default_settings('t') sett.update({'name':csvlog[0][i][2:]}) channels.append((vals, sett)) i += 1 continue else: raise Exception('Column "'+csvlog[0][i]+'" does not match a channel type.') dist_max = 0 sorted_pts_list = [] res = [] pts_list = self.pts_list self.get_closest_pts(sorted_pts_list) for d, settings in channels: count = len(self.score_groups) self.add_header(count, settings) if settings['score_type'] == 'xy': self.add_data_range(count, d) else: for j in range(len(d)): closest_pts = self.get_closest_pts(sorted_pts_list, pts[j]) dist_max = max(dist_max, abs(closest_pts-pts[j])) matched_list, matched_idx = pts_list[closest_pts] matched_list[1+count][matched_idx] = d[j] res.append((count, settings)) logging.warning('The maximum time between the read timestamp and'+ 'the closest timestamp of this video is %d.' % dist_max) return res elif overwrite == 'load': log = self.log_temp # overwrite temp file with new file log_temp = tb.open_file(logfile) res = '' #if log_temp.root._v_attrs.Glitter_version != log.root._v_attrs.Glitter_version: # res = "%s version is different than the current program's version" %filename if log_temp.root.video_info._v_attrs.file_name != log.root.video_info._v_attrs.file_name: res = "%s video file's name is different than the current video file." %filename if log_temp.root.video_info._v_attrs.video_params != log.root.video_info._v_attrs.video_params: res = "%s video file's parameters is different than the current video file's." %filename log_temp.close() if res: res += ' You might want to merge the file instead.' raise PyTrackException('log_res', res+'\n\n Error message: ' + str(e)) temp_filename = log.filename log.close() log = tb.open_file(logfile) log.copy_file(temp_filename, overwrite=True) log.close() self.log = logfile log = tb.open_file(temp_filename, 'a') # open temp self.log_temp = log if self.log_auto_dur: # maybe do autosave if self.log_auto: os.remove(self.log_auto) auto_file = tempfile.NamedTemporaryFile(dir=path, prefix=filename+'_', suffix='.autosave', delete=False) auto_name = auto_file.name auto_file.close() log.copy_file(auto_name, overwrite=True) self.log_auto = auto_name self.jumped = True # make sure info is up to date log.root._v_attrs.Glitter_version = glitter.__version__ log.root._v_attrs.Glitter_description = glitter.__description__ #log.root._v_attrs.username = '' #log.root._v_attrs.user_comment = '' #log.root._v_attrs.video_id = '' log.root._v_attrs.logID = self.filetail raw_group = log.root.raw_data self.raw_group = raw_group video_group = log.root.video_info self.video_info_group = video_group video_group._v_attrs.file_path = self.filehead video_group._v_attrs.file_name = self.filetail self.pts_list_complete = video_group._v_attrs.seen_all_frames self.pts_group = raw_group.pts score_groups = [None,] * (raw_group._v_nchildren - 1) for group in raw_group._f_iter_nodes(): if group._v_name != 'pts': score_groups[int(group._v_name.rpartition('_')[2])] = group self.score_groups = score_groups n_pts = len(self.pts_group._f_list_nodes()) data = [[None for j in range(len(score_groups)+1)] for i in range(n_pts)] for col in range(len(score_groups)): for row in score_groups[col]._f_iter_nodes(): data[int(row._v_name.rpartition('_')[2])][col + 1] = row pts_list = {} for row in self.pts_group._f_iter_nodes(): segment = int(row._v_name.rpartition('_')[2]) data[segment][0] = row for i in range(len(row)): pts_list[row[i]] = [data[segment], i] self.data = data self.last_list = data[0] self.pts_list = pts_list res = [None, ] * len(score_groups) for col in range(len(score_groups)): settings = {} score_group = score_groups[col] for setting in score_group._v_attrs._f_list('user'): settings[setting] = score_group._v_attrs[setting] res[col] = (col, settings) self.unsaved_changes = False self.status_func(complete=self.pts_list_complete, unsaved=False) if self.log_auto_dur: autosave_writer(auto_name) # res must be in increasing chan order return res