def request():
f = tempfile.NamedTemporaryFile(suffix='.h5', dir=tempfile.gettempdir(), delete=False)
f.close()
attempts = 0
while True:
try:
path = super(ParallelFeature, self).fetch(session, name, partial_resource_list, path=f.name)
except socket.error as e: #if connection fails
if attempts>MAX_ATTEMPTS:
log.debug('Connection fail: Reached max attempts')
break
if e.errno == errno.WSAECONNRESET:
attempts+=1
log.debug('Connection fail: Attempting to reconnect (try: %s)' % attempts)
try:
tables.is_pytables_file(path)
except tables.HDF5ExtError: #if fail gets corrupts during download
if attempts>MAX_ATTEMPTS:
log.debug('Failed to open hdf5 file: Reached max attempts')
break
attempts+=1
log.debug('HDF5 file may be corrupted: Attempted to redownload (try: %s)' % attempts)
if os.path.exists(path):
os.remove(path)
write_queue.put(path)
break
def __init__(self, h5parmFile, readonly = True, complevel = 5, complib='zlib'):
"""
Keyword arguments:
h5parmFile -- H5parm filename
readonly -- if True the table is open in readonly mode (default=True)
complevel -- compression level from 0 to 9 (default=5) when creating the file
complib -- library for compression: lzo, zlib, bzip2 (default=zlib)
"""
if os.path.isfile(h5parmFile):
if tables.is_pytables_file(h5parmFile) == None:
logging.critical('Wrong HDF5 format for '+h5parmFile+'.')
raise Exception('Wrong HDF5 format for '+h5parmFile+'.')
if readonly:
logging.debug('Reading from '+h5parmFile+'.')
self.H = tables.openFile(h5parmFile, 'r')
else:
logging.debug('Appending to '+h5parmFile+'.')
self.H = tables.openFile(h5parmFile, 'r+')
else:
if readonly:
raise Exception('Missing file '+h5parmFile+'.')
else:
logging.debug('Creating '+h5parmFile+'.')
# add a compression filter
f = tables.Filters(complevel=complevel, complib=complib)
self.H = tables.openFile(h5parmFile, filters=f, mode='w')
self.fileName = h5parmFile
def __init__(self, filename, model_state, proposal_state):
"""
Initialize the object.
"""
self.filename = filename
self.ChainRecordDType = state_to_table_dtype(model_state)
self.ChainRecordDType['step'] = pt.UInt32Col()
self.ChainRecordDType['accepted'] = pt.UInt32Col()
self.ChainRecordDType['proposal'] = pt.UInt16Col()
self.ProposalRecordDType = state_to_table_dtype(proposal_state)
self.ChainCounterDType = {'id': pt.UInt16Col(),
'name': pt.StringCol(itemsize=32),
'date': pt.StringCol(itemsize=26)
}
if os.path.exists(filename) and pt.is_pytables_file(filename):
self.fd = pt.open_file(filename, mode='a')
else:
self.fd = pt.open_file(filename, mode='w')
self.fd.create_group('/', 'mcmc',
'Metropolis-Hastings Algorithm Data')
self.fd.create_table('/mcmc', 'proposals',
self.ProposalRecordDType,
'MCMC Proposals')
self.fd.create_table('/mcmc', 'chain_counter',
self.ChainCounterDType, 'Chain Counter')
self.fd.create_group('/mcmc', 'data', 'Collection of Chains')
def __init__(self, h5parmFile, readonly=True, complevel=5, complib='lzo'):
"""
Keyword arguments:
h5parmFile -- H5parm filename
readonly -- if True the table is open in readonly mode (default=True)
complevel -- compression level from 0 to 9 (default=5) when creating the file
complib -- library for compression: lzo, zlib, bzip2 (default=lzo)
"""
if os.path.isfile(h5parmFile):
if tables.is_pytables_file(h5parmFile) == None:
logging.critical('Wrong HDF5 format for ' + h5parmFile + '.')
raise Exception('Wrong HDF5 format for ' + h5parmFile + '.')
if readonly:
logging.debug('Reading from ' + h5parmFile + '.')
self.H = tables.openFile(h5parmFile, 'r')
else:
logging.debug('Appending to ' + h5parmFile + '.')
self.H = tables.openFile(h5parmFile, 'r+')
else:
if readonly:
raise Exception('Missing file ' + h5parmFile + '.')
else:
logging.debug('Creating ' + h5parmFile + '.')
# add a compression filter
f = tables.Filters(complevel=complevel, complib=complib)
self.H = tables.openFile(h5parmFile, filters=f, mode='w')
self.fileName = h5parmFile
def __init__(self, filename, model_state, proposal_state):
"""
Initialize the object.
"""
self.filename = filename
self.ChainRecordDType = state_to_table_dtype(model_state)
self.ChainRecordDType['step'] = pt.UInt32Col()
self.ChainRecordDType['accepted'] = pt.UInt32Col()
self.ChainRecordDType['proposal'] = pt.UInt16Col()
self.ProposalRecordDType = state_to_table_dtype(proposal_state)
self.ChainCounterDType = {
'id': pt.UInt16Col(),
'name': pt.StringCol(itemsize=32),
'date': pt.StringCol(itemsize=26)
}
if os.path.exists(filename) and pt.is_pytables_file(filename):
self.fd = pt.open_file(filename, mode='a')
else:
self.fd = pt.open_file(filename, mode='w')
self.fd.create_group('/', 'mcmc',
'Metropolis-Hastings Algorithm Data')
self.fd.create_table('/mcmc', 'proposals',
self.ProposalRecordDType, 'MCMC Proposals')
self.fd.create_table('/mcmc', 'chain_counter',
self.ChainCounterDType, 'Chain Counter')
self.fd.create_group('/mcmc', 'data', 'Collection of Chains')
def load_file(path, fileformat, mode='a'):
'''Validates and loads a spectral document'''
try:
cache = path + CACHE[fileformat].suffix
assert os.path.exists(cache) and tb.is_pytables_file(cache)
return DOCUMENTS[fileformat].open(path, mode=mode)
except IOError:
raise AssertionError(exception.CODES['013'])
except (ValueError, AssertionError, KeyError):
raise AssertionError(exception.CODES['023'])
def load_file(path, fileformat, mode='a'):
'''Validates and loads a spectral document'''
try:
cache = path + CACHE[fileformat].suffix
assert os.path.exists(cache) and tb.is_pytables_file(cache)
return DOCUMENTS[fileformat].open(path, mode=mode)
except IOError:
raise AssertionError(exception.CODES['013'])
except (ValueError, AssertionError, KeyError):
raise AssertionError(exception.CODES['023'])
def request():
f = tempfile.NamedTemporaryFile(suffix='.h5',
dir=tempfile.gettempdir(),
delete=False)
f.close()
attempts = 0
while True:
try:
path = super(ParallelFeature,
self).fetch(session,
name,
partial_resource_list,
path=f.name)
except socket.error as e: #if connection fails
if attempts > MAX_ATTEMPTS:
log.debug('Connection fail: Reached max attempts')
break
if e.errno == errno.WSAECONNRESET: #pylint: disable=no-member
attempts += 1
log.debug(
'Connection fail: Attempting to reconnect (try: %s)'
% attempts)
try:
tables.is_pytables_file(path)
except tables.HDF5ExtError: #if fail gets corrupts during download
if attempts > MAX_ATTEMPTS:
log.debug(
'Failed to open hdf5 file: Reached max attempts'
)
break
attempts += 1
log.debug(
'HDF5 file may be corrupted: Attempted to redownload (try: %s)'
% attempts)
if os.path.exists(path):
os.remove(path)
write_queue.put(path)
break
def start(self):
if not os.path.exists(self.input):
self.error('No such file or directory: %s' % self.input)
if not tables.is_pytables_file(self.input):
self.error('Unrecognized format')
try:
ds = DataSet(self.input, 'r')
self.print_dataset(ds)
ds.close()
except UnrecognizedFormatError:
handle = tables.open_file(self.input)
self.print_model(handle)
handle.close()
def _open_database(self):
test = tables.is_pytables_file(str(self.path))
if test <= 0:
raise LineageError("File is not a pytables")
if self.readonly:
mode = 'r'
else:
mode = 'r+'
log.debug("Opening {} in mode {}".format(self.path.name, mode))
h5 = tables.open_file(str(self.path), mode=mode)
attrs = h5.root._v_attrs
assert attrs.storage_class == self.__class__.__name__
# Assign the arrays ------------
self._h5 = h5
self._attrs = attrs
self._networks = h5.root.networks
self._generations = h5.root.generations
def __init__(self, store):
"""Load fragment database view over store."""
self.store = None
if isinstance(store, basestring):
self.logger.info("Opening file: %s", store)
try:
is_store = tables.is_pytables_file(store)
except tables.HDF5ExtError:
is_store = False
if is_store:
if os.access(store, os.W_OK):
self.store = tables.open_file(store, "r+")
else:
self.store = tables.open_file(store, "r")
else:
raise ValueError("Store is not a PyTables file: %s" % store)
elif isinstance(store, tables.file.File):
self.logger.info("Opening : %s", store)
self.store = store
else:
raise ValueError("Unable to open store: %s" % store)
def load_log(self, path, overwrite='raise', template=False):
logfile = path
filename = os.path.split(os.path.splitext(os.path.normpath(path))[0])[1]
path = os.path.split(os.path.splitext(os.path.normpath(path))[0])[0]
autosave_writer = self.autosave_writer
try:
log_temp = self.log_temp
log_temp.copy_file(logfile, overwrite=(overwrite=='overwrite'))
if not template:
self.log = logfile
if self.log_auto_dur:
temp_file = tempfile.NamedTemporaryFile(dir=path, prefix=filename+'_', suffix='.autosave', delete=False)
temp_name = temp_file.name
temp_file.close()
log_temp.copy_file(temp_name, overwrite=True)
autosave_writer(temp_name)
self.log_auto = temp_name
self.unsaved_changes = False
self.status_func(unsaved=False)
if overwrite == 'overwrite':
return []
score_groups = self.score_groups
res = [None, ] * len(score_groups)
for col in range(len(score_groups)):
settings = {}
score_group = score_groups[col]
for setting in score_group._v_attrs._f_list('user'):
settings[setting] = score_group._v_attrs[setting]
res[col] = (col, settings)
return res
else:
template_file = None
try:
template_file = tb.open_file(logfile, 'a')
raw_group = template_file.root.raw_data
for row in raw_group.pts._f_list_nodes():
template_file.remove_node(row, recursive=True)
for group in raw_group._f_iter_nodes():
if group._v_name == 'pts' or group._v_attrs.score_type != 'xy':
for row in group._f_list_nodes():
template_file.remove_node(row, recursive=True)
return []
finally:
if template_file:
template_file.close()
except IOError as e: # file exits and we don't overwrite
if overwrite == 'raise' and not len(self.score_groups):
overwrite = 'load'
if overwrite == 'raise':
log = None
try:
log = tb.open_file(logfile)
cols = [node._v_name for node in log.root.raw_data._f_iter_nodes() if node._v_name != 'pts']
finally:
if log:
log.close()
raise PyTrackException('log_res', logfile + ' already exists '+
'with the following columns: [' + ' | '.join(cols) +
']. \n\nWhat would you like to do?\n-Load the file and overwrite the current '+
'channels.\n-Merge the current channels with the file channels.\n-Overwrite the file with '+
'the current channels (overwrites the file on disk).\n-Browse for another file.\n-Cancel.'
+ '\n\n Module error message:\n' + str(e))
if overwrite == 'merge' or template:
if (not template) and not self.pts_list_complete:
raise PyTrackException('error',
'Not all frames have been seen yet. You cannot '+
'merge a log until the full video has been seen.')
if tb.is_pytables_file(logfile):
log = None
try:
log = tb.open_file(logfile)
raw_group = log.root.raw_data
video_group = log.root.video_info
pts_group = raw_group.pts
score_groups = [None,] * (raw_group._v_nchildren - 1)
settings_list = list(score_groups)
res = list(score_groups)
for group in raw_group._f_iter_nodes():
if group._v_name != 'pts':
score_groups[int(group._v_name.rpartition('_')[2])] = group
for chan in range(len(score_groups)):
settings = {}
for setting in score_groups[chan]._v_attrs._f_list('user'):
settings[setting] = score_groups[chan]._v_attrs[setting]
settings_list[chan] = settings
n_pts = len(pts_group._f_list_nodes())
data = [[None for j in range(len(score_groups)+1)] for i in range(max(1, n_pts))]
for chan in range(len(score_groups)):
for row in score_groups[chan]._f_iter_nodes():
data[int(row._v_name.rpartition('_')[2])][chan + 1] = row
for row in pts_group._f_iter_nodes():
data[int(row._v_name.rpartition('_')[2])][0] = row
dist_max = 0
sorted_pts_list = []
pts_list = self.pts_list
self.get_closest_pts(sorted_pts_list)
for chan in range(len(score_groups)):
count = len(self.score_groups)
settings = settings_list[chan]
self.add_header(count, settings)
if settings['score_type'] == 'xy' and data[0][chan+1] and len(data[0][chan+1]):
self.add_data_range(count, list(data[0][chan+1]))
elif not template:
for i in range(len(data)):
for j in range(len(data[i][chan+1]) if data[i][chan+1] else 0):
pts = self.get_closest_pts(sorted_pts_list, data[i][0][j])
dist_max = max(dist_max, abs(pts-data[i][0][j]))
matched_list, matched_idx = pts_list[pts]
matched_list[1+count][matched_idx] = data[i][1+chan][j]
res[chan] = count, settings
finally:
if log:
log.close()
else:
with open(logfile, 'r') as csvfile:
csvlog = [row for row in csv.reader(csvfile)]
if not len(csvlog) or len(csvlog[0]) <= 1:
raise Exception("There's nothing to read in the file.")
if csvlog[0][0] != 'Time':
raise Exception("The first column is time and its title must be Time.")
pts = [float(t[0]) for t in csvlog[1:] if t[0]]
i = 1
channels = []
while i < len(csvlog[0]):
if csvlog[0][i].startswith('xyt'):
if (i+1 >= len(csvlog[0]) or (not csvlog[0][i].startswith('xyt_x_'))
or (not csvlog[0][i+1].startswith('xyt_y_'))):
raise Exception("A xyt score must be a two column series, with the first "+
"column starting with xyt_x_, and the next column starting with "+
"xyt_y_. Text following xyt_x_ is the column name.")
pos = [(int(t[i]), int(t[i+1])) for t in csvlog[1:] if t[i] and t[i+1]]
if len(pos) != len(pts):
raise Exception("Number of data points in a xyt channel must match the number of time points.")
sett = TrackLog.get_default_settings('xyt')
sett.update({'name':csvlog[0][i][6:]})
channels.append((pos, sett))
i += 2
continue
elif csvlog[0][i].startswith('xy'):
if (i+1 >= len(csvlog[0]) or (not csvlog[0][i].startswith('xy_x_'))
or (not csvlog[0][i+1].startswith('xy_y_'))):
raise Exception("A xy score must be a two column series, with the first "+
"column starting with xy_x_, and the next column starting with "+
"xy_y_. Text following xy_x_ is the column name.")
pos = [(int(t[i]), int(t[i+1])) for t in csvlog[1:] if t[i] and t[i+1]]
sett = TrackLog.get_default_settings('xy')
sett.update({'name':csvlog[0][i][5:]})
channels.append((pos, sett))
i += 2
continue
elif csvlog[0][i].startswith('t'):
if (not csvlog[0][i].startswith('t_')):
raise Exception("A t score is a column starting with t_. "+
"Text following t_ is the column name.")
vals = [bool(t[i]) for t in csvlog[1:] if t[i]]
if len(vals) != len(pts):
raise Exception("Number of data points in a t channel must match the number of time points.")
sett = TrackLog.get_default_settings('t')
sett.update({'name':csvlog[0][i][2:]})
channels.append((vals, sett))
i += 1
continue
else:
raise Exception('Column "'+csvlog[0][i]+'" does not match a channel type.')
dist_max = 0
sorted_pts_list = []
res = []
pts_list = self.pts_list
self.get_closest_pts(sorted_pts_list)
for d, settings in channels:
count = len(self.score_groups)
self.add_header(count, settings)
if settings['score_type'] == 'xy':
self.add_data_range(count, d)
else:
for j in range(len(d)):
closest_pts = self.get_closest_pts(sorted_pts_list, pts[j])
dist_max = max(dist_max, abs(closest_pts-pts[j]))
matched_list, matched_idx = pts_list[closest_pts]
matched_list[1+count][matched_idx] = d[j]
res.append((count, settings))
logging.warning('The maximum time between the read timestamp and'+
'the closest timestamp of this video is %d.' % dist_max)
return res
elif overwrite == 'load':
log = self.log_temp # overwrite temp file with new file
log_temp = tb.open_file(logfile)
res = ''
#if log_temp.root._v_attrs.Glitter_version != log.root._v_attrs.Glitter_version:
# res = "%s version is different than the current program's version" %filename
if log_temp.root.video_info._v_attrs.file_name != log.root.video_info._v_attrs.file_name:
res = "%s video file's name is different than the current video file." %filename
if log_temp.root.video_info._v_attrs.video_params != log.root.video_info._v_attrs.video_params:
res = "%s video file's parameters is different than the current video file's." %filename
log_temp.close()
if res:
res += ' You might want to merge the file instead.'
raise PyTrackException('log_res', res+'\n\n Error message: ' + str(e))
temp_filename = log.filename
log.close()
log = tb.open_file(logfile)
log.copy_file(temp_filename, overwrite=True)
log.close()
self.log = logfile
log = tb.open_file(temp_filename, 'a') # open temp
self.log_temp = log
if self.log_auto_dur: # maybe do autosave
if self.log_auto:
os.remove(self.log_auto)
auto_file = tempfile.NamedTemporaryFile(dir=path, prefix=filename+'_', suffix='.autosave', delete=False)
auto_name = auto_file.name
auto_file.close()
log.copy_file(auto_name, overwrite=True)
self.log_auto = auto_name
self.jumped = True # make sure info is up to date
log.root._v_attrs.Glitter_version = glitter.__version__
log.root._v_attrs.Glitter_description = glitter.__description__
#log.root._v_attrs.username = ''
#log.root._v_attrs.user_comment = ''
#log.root._v_attrs.video_id = ''
log.root._v_attrs.logID = self.filetail
raw_group = log.root.raw_data
self.raw_group = raw_group
video_group = log.root.video_info
self.video_info_group = video_group
video_group._v_attrs.file_path = self.filehead
video_group._v_attrs.file_name = self.filetail
self.pts_list_complete = video_group._v_attrs.seen_all_frames
self.pts_group = raw_group.pts
score_groups = [None,] * (raw_group._v_nchildren - 1)
for group in raw_group._f_iter_nodes():
if group._v_name != 'pts':
score_groups[int(group._v_name.rpartition('_')[2])] = group
self.score_groups = score_groups
n_pts = len(self.pts_group._f_list_nodes())
data = [[None for j in range(len(score_groups)+1)] for i in range(n_pts)]
for col in range(len(score_groups)):
for row in score_groups[col]._f_iter_nodes():
data[int(row._v_name.rpartition('_')[2])][col + 1] = row
pts_list = {}
for row in self.pts_group._f_iter_nodes():
segment = int(row._v_name.rpartition('_')[2])
data[segment][0] = row
for i in range(len(row)):
pts_list[row[i]] = [data[segment], i]
self.data = data
self.last_list = data[0]
self.pts_list = pts_list
res = [None, ] * len(score_groups)
for col in range(len(score_groups)):
settings = {}
score_group = score_groups[col]
for setting in score_group._v_attrs._f_list('user'):
settings[setting] = score_group._v_attrs[setting]
res[col] = (col, settings)
self.unsaved_changes = False
self.status_func(complete=self.pts_list_complete, unsaved=False)
if self.log_auto_dur:
autosave_writer(auto_name)
# res must be in increasing chan order
return res
