def _read_ascii(filename, usedatetime, nhead=5): """ Read a lsl file from ASCII """ # open the file open_file = open(filename, 'r') # get the header file_lines = open_file.readlines() nvariables = file_lines[2].strip().split() head = file_lines[0].split() # read starting time new and old formats try: starttime = datetime.datetime.strptime(head[2], '%Y%m%d_%H%M') except ValueError: try: starttime = datetime.datetime.strptime(head[2] + '_' + head[3], '%Y%m%d_%H') except: print("Warning: could not retrieve starttime from header,\ setting to default value ") starttime = datetime.datetime(1970, 1, 1) if usedatetime: dtypes = [ 'f8' if var != 'time' else datetime.datetime for var in nvariables ] else: dtypes = ['f8' for var in nvariables] # read the content as numpy array array = scipy.genfromtxt(filename, dtype=dtypes, names=nvariables, skip_header=nhead, missing_values=-999.99) # find characteristic of trajectories timestep = float(array[1][0]) - float(array[0][0]) period = float(array[-1][0]) - float(array[0][0]) # Convert minutes to decimal hours if max((array['time'].astype(float)) % 1) <= 0.60: timestep = scipy.floor(timestep) + ((timestep % 1) / .60) period = scipy.floor(period) + ((period % 1) / .60) # period/timestep gives strange offset (related to precision??) # so use scipy.around ntime = int(1 + scipy.around(period / timestep)) ntra = int(array.size / ntime) # reshape traj file array = scipy.reshape(array, (ntra, ntime)) if usedatetime: # change time into datetime object time = scipy.array([ scipy.datetime64(starttime + datetime.timedelta(hours=t)) for t in scipy.arange(0, period + timestep, timestep) ]) time.shape = (1, ) + time.shape time = time.repeat(array.shape[0], axis=0) array['time'] = 0 newdtypes = [ descr if descr[0] != 'time' else ('time', 'datetime64[s]') for descr in array.dtype.descr ] array = array.astype(newdtypes) array['time'] = time return array
def _read_ascii(filename, usedatetime, nhead=5): """ Read a lsl file from ASCII """ # open the file open_file = open(filename, 'r') # get the header file_lines = open_file.readlines() nvariables = file_lines[2].strip().split() head = file_lines[0].split() # read starting time new and old formats try: starttime = datetime.datetime.strptime(head[2], '%Y%m%d_%H%M') except ValueError: try: starttime = datetime.datetime.strptime( head[2] + '_' + head[3], '%Y%m%d_%H' ) except: print("Warning: could not retrieve starttime from header,\ setting to default value ") starttime = datetime.datetime(1970, 1, 1) if usedatetime: dtypes = ['f8' if var != 'time' else datetime.datetime for var in nvariables] else: dtypes = ['f8' for var in nvariables] # read the content as numpy array array = scipy.genfromtxt(filename, dtype=dtypes, names=nvariables, skip_header=nhead, missing_values=-999.99) # find characteristic of trajectories timestep = float(array[1][0]) - float(array[0][0]) period = float(array[-1][0]) - float(array[0][0]) # Convert minutes to decimal hours if max((array['time'].astype(float)) % 1) <= 0.60: timestep = scipy.floor(timestep) + ((timestep % 1) / .60) period = scipy.floor(period) + ((period % 1) / .60) # period/timestep gives strange offset (related to precision??) # so use scipy.around ntime = int(1 + scipy.around(period / timestep)) ntra = int(array.size / ntime) # reshape traj file array = scipy.reshape(array, (ntra, ntime)) if usedatetime: # change time into datetime object time = scipy.array( [scipy.datetime64(starttime + datetime.timedelta(hours=t)) for t in scipy.arange(0, period + timestep, timestep)] ) time.shape = (1,) + time.shape time = time.repeat(array.shape[0], axis=0) array['time'] = 0 newdtypes = [descr if descr[0] != 'time' else ('time', 'datetime64[s]') for descr in array.dtype.descr] array = array.astype(newdtypes) array['time'] = time return array
def _read_netcdf(filename, usedatetime): """ Read a netcdf lsl file """ # read the netcdf, the formats and the variables names ncfile = netCDF4.Dataset(filename) if usedatetime: formats = [ var[1].dtype if var[0] != 'time' else 'datetime64[s]' for var in list(ncfile.variables.items()) ] else: formats = [var[1].dtype for var in list(ncfile.variables.items())] variables = list(ncfile.variables.keys()) # set the numbers of trajectories and time step try: ntra = len(ncfile.dimensions['dimx_lon']) except: try: ntra = len(ncfile.dimensions['id']) except: try: ntra = len(ncfile.dimensions['ntra']) except: raise Exception('Cannot read the number of trajectories,\ not one of dimx_lon, id or ntra') try: ntime = len(ncfile.dimensions['time']) except: ntime = len(ncfile.dimensions['ntim']) # change names of the coordinates if necessary nvariables = list(variables) if "longitude" in variables: nvariables[variables.index("longitude")] = "lon" if "latitude" in variables: nvariables[variables.index("latitude")] = "lat" # create a structured array using numpy array = scipy.zeros((ntra, ntime), dtype={ 'names': nvariables, 'formats': formats }) for variable in variables: if variable == 'BASEDATE': continue nvariable = variable if variable == 'longitude': nvariable = 'lon' if variable == 'latitude': nvariable = 'lat' if len(ncfile.variables[variable].shape) == 4: array[nvariable] = ncfile.variables[variable][:, 0, 0, :][:].T else: array[nvariable] = ncfile.variables[variable][:].T # read the starting date and duration try: date = [int(i) for i in ncfile.variables['BASEDATE'][0, 0, 0, :]] starttime = datetime.datetime(date[0], date[1], date[2], date[3], date[4]) except: starttime = datetime.datetime(ncfile.ref_year, ncfile.ref_month, ncfile.ref_day, ncfile.ref_hour, ncfile.ref_min) # find characteristic of trajectories timestep = ncfile.variables['time'][1] - ncfile.variables['time'][0] period = ncfile.variables['time'][-1] - ncfile.variables['time'][0] # True if time = hh.mm cond1 = 0.00 < (ncfile.variables['time'][:] % 1).max() <= 0.60 if cond1: timestep = scipy.floor(timestep) + ((timestep % 1) / .60) timestep = scipy.around(timestep, 6) period = scipy.floor(period) + ((period % 1) / .60) period = scipy.around(period, 6) # transform the times into datetime object # special treatment for online trajectories (time given in minutes) if usedatetime: try: time = scipy.array([ scipy.datetime64(starttime + datetime.timedelta(hours=t)) for t in scipy.arange(0, period + timestep, timestep) ]) except AttributeError: time = scipy.array([ scipy.datetime64(starttime + datetime.timedelta(seconds=t)) for t in scipy.arange(0, period + timestep, timestep) ]) time.shape = (1, ) + time.shape time = time.repeat(array.shape[0], axis=0) array['time'] = time ncfile.close() return array
def _read_netcdf(filename, usedatetime): """ Read a netcdf lsl file """ # read the netcdf, the formats and the variables names ncfile = netCDF4.Dataset(filename) if usedatetime: formats = [var[1].dtype if var[0] != 'time' else 'datetime64[s]' for var in list(ncfile.variables.items())] else: formats = [var[1].dtype for var in list(ncfile.variables.items())] variables = list(ncfile.variables.keys()) # set the numbers of trajectories and time step try: ntra = len(ncfile.dimensions['dimx_lon']) except: try: ntra = len(ncfile.dimensions['id']) except: try: ntra = len(ncfile.dimensions['ntra']) except: raise Exception('Cannot read the number of trajectories,\ not one of dimx_lon, id or ntra') try: ntime = len(ncfile.dimensions['time']) except: ntime = len(ncfile.dimensions['ntim']) # change names of the coordinates if necessary nvariables = list(variables) if "longitude" in variables: nvariables[variables.index("longitude")] = "lon" if "latitude" in variables: nvariables[variables.index("latitude")] = "lat" # create a structured array using numpy array = scipy.zeros((ntra, ntime), dtype={'names': nvariables, 'formats': formats}) for variable in variables: if variable == 'BASEDATE': continue nvariable = variable if variable == 'longitude': nvariable = 'lon' if variable == 'latitude': nvariable = 'lat' if len(ncfile.variables[variable].shape) == 4: array[nvariable] = ncfile.variables[variable][:, 0, 0, :][:].T else: array[nvariable] = ncfile.variables[variable][:].T # read the starting date and duration try: date = [int(i) for i in ncfile.variables['BASEDATE'][0, 0, 0, :]] starttime = datetime.datetime(date[0], date[1], date[2], date[3], date[4]) except: starttime = datetime.datetime(ncfile.ref_year, ncfile.ref_month, ncfile.ref_day, ncfile.ref_hour, ncfile.ref_min) # find characteristic of trajectories timestep = ncfile.variables['time'][1] - ncfile.variables['time'][0] period = ncfile.variables['time'][-1] - ncfile.variables['time'][0] # True if time = hh.mm cond1 = 0.00 < (ncfile.variables['time'][:] % 1).max() <= 0.60 if cond1: timestep = scipy.floor(timestep) + ((timestep % 1) / .60) timestep = scipy.around(timestep, 6) period = scipy.floor(period) + ((period % 1) / .60) period = scipy.around(period, 6) # transform the times into datetime object # special treatment for online trajectories (time given in minutes) if usedatetime: try: time = scipy.array( [scipy.datetime64(starttime + datetime.timedelta(hours=t)) for t in scipy.arange(0, period + timestep, timestep)] ) except AttributeError: time = scipy.array( [scipy.datetime64(starttime + datetime.timedelta(seconds=t)) for t in scipy.arange(0, period + timestep, timestep)] ) time.shape = (1,) + time.shape time = time.repeat(array.shape[0], axis=0) array['time'] = time ncfile.close() return array