def loadFile(self, file_path, arg_dict={}):
"""Loads the ISIS .DAT file.
Splits it into objects for each unit type, initial conditions etc.
This is an epic if-else section for each unit type currently
represented.
Needs cleaning up and writing with a bit more style.
Easy to add another unit type, if it's not currently covered then it
will just be collected in the universal 'UnknownUnit' and printed
back out the same as it came in.
Args:
file_path (str): path to the .dat file to load.
Returns:
units - UnitCollection containing the dat file units or False if
they couldn't be loaded.
Raises:
IOError: If the file cannot be loaded or is empty.
AttributeError: if the file is not of an expected type (.dat/.ief).
See Also:
IsisUnitCollection
FactoryClasses
TODO: Decide if the observer style calls are ever going to be needed.
If they aren't then remove them rather than have them
cluttering up the file.
"""
line = ''
# Used to populate the data for the UnknownUnit
self.unknown_data = []
# Composite for all dat units
path_holder = ftools.PathHolder(file_path)
self.units = DatCollection(path_holder)
# self.units.file_dir, self.units.filename = os.path.split(file_path)
# self.units.filename = os.path.splitext(self.units.filename)[0]
if not uf.checkFileType(file_path, ext=['.dat', '.DAT']):
if not uf.checkFileType(file_path, ext=['.ied', '.IED']):
logger.error(
'Illegal File Error: ' + file_path +
'\nDoes not have extension (.dat, .DAT, .ied, .IED)')
raise AttributeError(
'Illegal File Error: ' + file_path +
'\nDoes not have extension (.dat, .DAT, .ied, .IED)')
else:
self.is_ied = True
contents = self.__loadFile(file_path)
if (contents == False):
raise IOError('Unable to load file at: ' + file_path)
return self.buildDat(contents, arg_dict)
def initialisedDat(cls, dat_path, units=[], **kwargs):
"""Create a new ISIS .dat file with basic header info and no units.
Creates the equivelant of generating a new .dat unit in the software. The
DatCollection returned can then be manipulated in the same way that any
other one loaded from file would be.
A single comment unit will be be added to the file stating that it was
created by the SHIP library at timestamp.
Example unit_kwargs. Note that first index is a placholder for no args::
unit_kwargs== [{},
{
'ics': {rdt.FLOW: 3.0, rdt.STAGE: 15.0, rdt.ELEVATION: 14.5},
}
]
**kwargs:
unit_kwargs(list): contains a dict with the kwargs for each unit
that is included in units. The list must be the same length
as units, or not included. If no kwargs for a particular unit
are to be given an empty dict should be used as a placholder in
the list.
Args:
dat_path(str): the path to set for the newly created .dat file.
Return:
DatCollection - setup as an empty ISIS .dat file.
"""
unit_kwargs = kwargs.get('unit_kwargs', [{}] * len(units))
if not len(unit_kwargs) == len(units):
raise ValueError(
'unit_kwargs kwarg must be the same length as unit or not be given'
)
path_holder = ft.PathHolder(dat_path)
dat = cls(path_holder)
hunit = iuf.FmpUnitFactory.createUnit('header')
icunit = iuf.FmpUnitFactory.createUnit('initial_conditions')
cunit = CommentUnit(text=('Created by SHIP library on %s' %
datetime.now().strftime('%Y-%M-%d %H:%M')))
dat.addUnit(hunit, update_node_count=False)
dat.addUnit(cunit, update_node_count=False)
dat.addUnit(icunit, update_node_count=False)
for i, u in enumerate(units):
dat.addUnit(u, **unit_kwargs[i])
return dat
def readMatSubfile(main_datafile, filename, header_list, args_dict):
"""
"""
value_separator = ','
comment_types = ['#', '!']
mat_subfile_enum = dataobj.SubfileMatEnum()
path = os.path.join(main_datafile.root, filename)
root = main_datafile.root
header1 = 'None'
header2 = 'None'
if len(header_list) > 0:
header1 = header_list[0]
if len(header_list) > 1:
header2 = header_list[1]
def _scanfile(filepath):
"""Scans the file before we do any loading to identify the contents.
Need to do this because the file can be setup in so many way that it
becomes a headache to work it out in advance. Better to take a little
bit of extra processing time and do some quick checks first.
Arguments:
file_path (str): the path to the subfile.
Return:
tuple:
list: booleans with whether the column contains
data that we want or not.
int: length of the cols list.
list: containing all of the first row column data
int: first row with usable data on.
"""
logger.debug('Scanning Materials file - %s' % (filepath))
with open(filepath, 'rb') as csv_file:
csv_file = csv.reader(csv_file)
cols = []
head_list = []
start_row = -1
for i, row in enumerate(csv_file, 0):
if "".join(row).strip() == "":
break
for j, col in enumerate(row, 0):
if i == 0:
cols.append(False)
head_list = row
elif uuf.isNumeric(col):
cols[j] = True
if start_row == -1:
start_row = i
elif cols[j] == True:
break
return cols, len(cols), head_list, start_row
def _loadHeadData(row, row_collection, col_length):
"""
"""
new_row = [None] * 12
comment_indices, length = uuf.findSubstringInList('!', row)
comment_lines.append(None)
head1_location = -1
head2_location = -1
row_length = len(row)
for i in range(0, col_length):
if i < row_length:
entry = row[i].strip()
if entry == header1:
head1_location = i
if entry == header2:
head2_location = i
row_collection._addValue('actual_header', entry)
return row_collection, head1_location, head2_location
def _loadRowData(row, row_count, row_collection, comment_lines, col_length,
start_row):
"""Loads the data in a specific row of the file.
Args:
row(list): containing the row data.
row_count(int): the current row number.
required_headers(list): column names that must exist.
Return:
rowdatacollection: updated with header row details.
"""
# Any lines that aren't headers, but are above the first row to contain
# actual data will be stored as comment lines
if row_count < start_row:
comment_lines.append(row)
return row_collection, comment_lines
else:
comment_lines.append(None)
if '!' in row[-1] or '#' in row[-1]:
row_collection._addValue('comment', row[-1])
# Add the row data in the order that it appears in the file
# from left to right.
for i in range(col_length):
if i < len(row):
row_collection._addValue(i, row[i])
return row_collection, comment_lines
try:
logger.info('Loading data file contents from disc - %s' % (path))
with open(path, 'rb') as csv_file:
csv_file = csv.reader(csv_file)
# Do a quick check of the file setup
cols, col_length, head_list, start_row = _scanfile(path)
# First entry doesn't want to have a comma in front when formatting.
# but all of the others do.
row_collection = RowDataCollection()
row_collection.addToCollection(
do.FloatData(0, format_str=' {0}', default='', no_of_dps=6))
for i in range(1, len(cols)):
if cols[i] == True:
row_collection.addToCollection(
do.FloatData(i,
format_str=', {0}',
default='',
no_of_dps=6))
else:
row_collection.addToCollection(
do.StringData(i, format_str=', {0}', default=''))
row_collection.addToCollection(do.StringData('actual_header',
format_str='{0}',
default=''),
index=0)
row_collection.addToCollection(
do.IntData('row_no', format_str=None, default=''))
# Stores the comments found in the file
comment_lines = []
first_data_line = False
# Loop through the contents list loaded from file line-by-line.
for i, line in enumerate(csv_file, 0):
comment = hasCommentOnlyLine(''.join(line), comment_types)
if comment or comment == '':
comment_lines.append([comment, i])
# If we have a line that isn't a comment or a blank then it is going
# to contain materials entries.
else:
# First non-comment is the headers
if first_data_line == False:
first_data_line = True
row_collection, head1_loc, head2_loc = _loadHeadData(
line, row_collection, col_length)
else:
row_collection, comment_lines = _loadRowData(
line, i, row_collection, comment_lines, col_length,
start_row)
row_collection._addValue('row_no', i)
except IOError:
logger.warning('Cannot load file - IOError')
raise IOError('Cannot load file at: ' + path)
path_holder = filetools.PathHolder(path, root)
mat_sub = dataobj.DataFileSubfileMat(path_holder, row_collection,
comment_lines, path_holder.filename,
head1_loc, head2_loc)
return mat_sub
def loadFile(self, file_path, arg_dict={}):
"""Loads the ief file at the given path.
Args:
file_path (str): The path to the ief file.
Returns:
Ief file object containing the contents loaded from file.
"""
if not uf.checkFileType(file_path, ext=['.ief', '.IEF']):
logger.error('File: ' + file_path +
'\nDoes not match ief extension (*.ief or *.IEF)')
logger.error('Illegal File Error: %s is not of type ief' %
(file_path))
raise AttributeError('Illegal File Error: %s is not of type ief' %
(file_path))
contents = self._loadFile(file_path)
# if we couldn't load the file let them know.
if contents == False:
raise IOError('Unable to load file at: ' + file_path)
event_header = {}
event_details = {}
snapshot = []
ied_data = []
description = []
in_event_details = False
index = 0
while (index < len(contents)):
c = contents[index]
# If we're moving into the event details then mark it and skip the
# next line, which tells us so.
if c.strip('\n') == '[ISIS Event Details]':
in_event_details = True
index += 1
continue
# Collect the header data with the name as the key and the variable
# as the value.
if not in_event_details:
if not c.strip('\n') == '[ISIS Event Header]':
self._addHeaderLine(event_header, c)
else:
# If it's a snapshot then get the time. We know that the next
# line is a file reference so get that too then skip ahead by
# one line in the contents.
if c.split('=')[0] == 'SnapshotTime':
index = self._addSnapshotLine(snapshot, contents, index)
# If the line starts with a ';' then it's an IED file so grab
# the name and get the file file reference from the next line
# and skip ahead.
elif c[:1] == ';':
index = self._addIedLine(ied_data, contents, index)
# Otherwise just populate the event_details.
else:
# If the description bit is included its at the end so just grab the
# last few line and put them in it.
if c.strip() == '[Description]':
index = self._loadDescription(description, contents,
index)
break
else:
if not c.strip() == '':
self._addDetailsLine(event_details, contents,
index)
index += 1
path_holder = filetools.PathHolder(file_path)
ief_file = Ief(path_holder, event_header, event_details, snapshot,
ied_data, description)
return ief_file
def loadFile(self, file_path, arg_dict={}):
"""Loads the ISIS .DAT file.
Splits it into objects for each unit type, initial conditions etc.
This is an epic if-else section for each unit type currently
represented.
Needs cleaning up and writing with a bit more style.
Easy to add another unit type, if it's not currently covered then it
will just be collected in the universal 'UnknownSection' and printed
back out the same as it came in.
Args:
file_path (str): path to the .dat file to load.
Returns:
units - UnitCollection containing the dat file units or False if
they couldn't be loaded.
Raises:
IOError: If the file cannot be loaded or is empty.
AttributeError: if the file is not of an expected type (.dat/.ief).
See Also:
IsisUnitCollection
FactoryClasses
TODO: Decide if the observer style calls are ever going to be needed.
If they aren't then remove them rather than have them
cluttering up the file.
"""
line = ''
# Used to populate the data for the UnknownSection
self.unknown_data = []
# Composite for all dat units
path_holder = ftools.PathHolder(file_path)
self.units = DatCollection(path_holder)
self.units.file_dir, self.units.filename = os.path.split(file_path)
self.units.filename = os.path.splitext(self.units.filename)[0]
if not uf.checkFileType(file_path, ext=['.dat', '.DAT']):
if not uf.checkFileType(file_path, ext=['.ied', '.IED']):
logger.error(
'Illegal File Error: ' + file_path +
'\nDoes not have extension (.dat, .DAT, .ied, .IED)')
raise AttributeError(
'Illegal File Error: ' + file_path +
'\nDoes not have extension (.dat, .DAT, .ied, .IED)')
else:
self.is_ied = True
if (self.__loadFile(file_path) == False):
raise IOError('Unable to load file at: ' + file_path)
# Counter for the number of rows that have been read from the
# file contents list.
i = 0
# Get an instance of the unit factory with the number of nodes in the file.
unit_factory = IsisUnitFactory()
# Dictionary containing the keys to identify units in the dat file
unit_vars = unit_factory.getUnitIdentifiers()
# Create a unit from the header data in the first few lines of the dat file.
if not self.is_ied:
i, self.temp_unit = unit_factory.createUnit(
self.contents, 0, 'header', 0)
in_unknown_section = False
# Now we can update the HeaderUnit subContents
self.updateSubContents()
in_unknown_section = False
while i < len(self.contents):
# Get the line and then split it to retrieve the first word.
# Check this word against the # unit_type keys we set above to see
line = self.contents[i]
temp_line = line.strip()
if temp_line:
first_word = line.split()[0].strip()
else:
first_word = 'Nothing'
if first_word in unit_vars:
# If building an UnknownSection then create an reset
if (in_unknown_section == True):
self.createUnknownSection()
self.updateSubContents()
# Reset the reach for the UnknownSection
unit_factory.same_reach = False
'''Call the unit creator function and get back the unit and the
updated contents list index.
Most of these variables are self explanatory, but
unit_vars[first_word] is the key for the unit type to make.
'''
i, self.temp_unit = unit_factory.createUnit(
self.contents, i, unit_vars[first_word],
self.cur_no_of_units)
'''In case we got in but found something wasn't supported.
it's i-1 because we can't return onto the same line that was
read or it will loop forever, so store it here and move on
'''
if self.temp_unit == False:
self.unknown_data.append(self.contents[i].rstrip('\n'))
i += 1
self.unknown_data.append(self.contents[i].rstrip('\n'))
in_unknown_section = True
else:
self.updateSubContents()
in_unknown_section = False
else:
in_unknown_section = True
self.unknown_data.append(self.contents[i].rstrip('\n'))
i += 1
line = None
del self.unknown_data
return self.units