def from_dataframe(cls, df, output, **kwargs): axes = get_default_axes(kwargs.pop('axes', {})) daxes = axes _ = kwargs.pop('reduce_dims', False) _ = kwargs.pop('unlimited', False) unique_dims = kwargs.pop('unique_dims', False) if unique_dims is True: # Rename the dimension to avoid a dimension and coordinate having the same name # which is not support in xarray changed_axes = { k: '{}_dim'.format(v) for k, v in axes._asdict().items() } daxes = get_default_axes(changed_axes) # Downcast anything from int64 to int32 # Convert any timezone aware datetimes to native UTC times df = downcast_dataframe(nativize_times(df)) with ContiguousRaggedTrajectoryProfile(output, 'w') as nc: trajectory_groups = df.groupby(axes.trajectory) unique_trajectories = list(trajectory_groups.groups.keys()) num_trajectories = len(unique_trajectories) nc.createDimension(daxes.trajectory, num_trajectories) trajectory = nc.createVariable(axes.trajectory, get_dtype(df[axes.trajectory]), (daxes.trajectory, )) trajectory[:] = np.array(unique_trajectories) # Calculate the max number of profiles unique_profiles = df[axes.profile].unique() num_profiles = len(unique_profiles) nc.createDimension(daxes.profile, num_profiles) profile = nc.createVariable(axes.profile, get_dtype(df[axes.profile]), (daxes.profile, )) profile[:] = np.array(unique_profiles) # Get unique obs by grouping on traj and profile and getting the max size num_obs = len(df) nc.createDimension(daxes.sample, num_obs) # The trajectory this profile belongs to t_ind = nc.createVariable('trajectoryIndex', 'i4', (daxes.profile, )) # Number of observations in each profile row_size = nc.createVariable('rowSize', 'i4', (daxes.profile, )) # Create all of the axis variables time = nc.createVariable(axes.t, 'f8', (daxes.profile, ), fill_value=np.dtype('f8').type( cls.default_fill_value)) latitude = nc.createVariable( axes.y, get_dtype(df[axes.y]), (daxes.profile, ), fill_value=df[axes.y].dtype.type(cls.default_fill_value)) longitude = nc.createVariable( axes.x, get_dtype(df[axes.x]), (daxes.profile, ), fill_value=df[axes.x].dtype.type(cls.default_fill_value)) # Axes variables are already processed so skip them data_columns = [d for d in df.columns if d not in axes] attributes = dict_update(nc.nc_attributes(axes, daxes), kwargs.pop('attributes', {})) # Variables defined on only the profile axis profile_vars = kwargs.pop('profile_vars', []) profile_columns = [p for p in profile_vars if p in data_columns] for c in profile_columns: var_name = cf_safe_name(c) if var_name not in nc.variables: create_ncvar_from_series(nc, var_name, (daxes.profile, ), df[c], zlib=True, complevel=1) for i, (_, trg) in enumerate(trajectory_groups): for j, (_, pfg) in enumerate(trg.groupby(axes.profile)): time[j] = get_ncdata_from_series(pfg[axes.t], time).astype('f8')[0] latitude[j] = get_ncdata_from_series( pfg[axes.y], latitude)[0] longitude[j] = get_ncdata_from_series( pfg[axes.x], longitude)[0] row_size[j] = len(pfg) t_ind[j] = i # Save any profile variables on the "profile" index using the first value found # in the column. for c in profile_columns: var_name = cf_safe_name(c) if var_name not in nc.variables: continue v = nc.variables[var_name] vvalues = get_ncdata_from_series(pfg[c], v)[0] try: v[j] = vvalues except BaseException: L.exception('Failed to add {}'.format(c)) continue # Add back in the z axes that was removed when calculating data_columns # and ignore variables that were stored in the profile index sample_columns = [ f for f in data_columns + [axes.z] if f not in profile_columns ] skips = ['trajectoryIndex', 'rowSize'] for c in [d for d in sample_columns if d not in skips]: var_name = cf_safe_name(c) if var_name not in nc.variables: v = create_ncvar_from_series(nc, var_name, (daxes.sample, ), df[c], zlib=True, complevel=1) else: v = nc.variables[var_name] vvalues = get_ncdata_from_series(df[c], v) try: v[:] = vvalues.reshape(v.shape) except BaseException: L.exception('Failed to add {}'.format(c)) continue # Metadata variables if 'crs' not in nc.variables: nc.createVariable('crs', 'i4') # Set attributes nc.update_attributes(attributes) return ContiguousRaggedTrajectoryProfile(output, **kwargs)
def from_dataframe(cls, df, output, **kwargs): axes = get_default_axes(kwargs.pop('axes', {})) _ = kwargs.pop('reduce_dims', False) _ = kwargs.pop('unlimited', False) with ContiguousRaggedTrajectoryProfile(output, 'w') as nc: trajectory_groups = df.groupby(axes.trajectory) unique_trajectories = list(trajectory_groups.groups.keys()) num_trajectories = len(unique_trajectories) nc.createDimension(axes.trajectory, num_trajectories) trajectory = nc.createVariable(axes.trajectory, get_dtype(df[axes.trajectory]), (axes.trajectory,)) trajectory[:] = np.array(unique_trajectories) # Calculate the max number of profiles unique_profiles = df[axes.profile].unique() num_profiles = len(unique_profiles) nc.createDimension(axes.profile, num_profiles) profile = nc.createVariable(axes.profile, get_dtype(df[axes.profile]), (axes.profile,)) profile[:] = np.array(unique_profiles) # Get unique obs by grouping on traj and profile and getting the max size num_obs = len(df) nc.createDimension(axes.sample, num_obs) # The trajectory this profile belongs to t_ind = nc.createVariable('trajectoryIndex', 'i4', (axes.profile,)) # Number of observations in each profile row_size = nc.createVariable('rowSize', 'i4', (axes.profile,)) # Create all of the axis variables time = nc.createVariable(axes.t, 'f8', (axes.profile,), fill_value=np.dtype('f8').type(cls.default_fill_value)) latitude = nc.createVariable(axes.y, get_dtype(df[axes.y]), (axes.profile,), fill_value=df[axes.y].dtype.type(cls.default_fill_value)) longitude = nc.createVariable(axes.x, get_dtype(df[axes.x]), (axes.profile,), fill_value=df[axes.x].dtype.type(cls.default_fill_value)) # Axes variables are already processed so skip them data_columns = [ d for d in df.columns if d not in axes ] attributes = dict_update(nc.nc_attributes(axes), kwargs.pop('attributes', {})) for i, (_, trg) in enumerate(trajectory_groups): for j, (_, pfg) in enumerate(trg.groupby(axes.profile)): time[j] = get_ncdata_from_series(pfg[axes.t], time)[0] latitude[j] = get_ncdata_from_series(pfg[axes.y], latitude)[0] longitude[j] = get_ncdata_from_series(pfg[axes.x], longitude)[0] row_size[j] = len(pfg) t_ind[j] = i # Add back in the z axes that was removed when calculating data_columns data_columns = data_columns + [axes.z] skips = ['trajectoryIndex', 'rowSize'] for c in [ d for d in data_columns if d not in skips ]: var_name = cf_safe_name(c) if var_name not in nc.variables: v = create_ncvar_from_series( nc, var_name, (axes.sample,), df[c], zlib=True, complevel=1 ) else: v = nc.variables[var_name] vvalues = get_ncdata_from_series(df[c], v) try: v[:] = vvalues except BaseException: L.exception('Failed to add {}'.format(c)) continue # Metadata variables if 'crs' not in nc.variables: nc.createVariable('crs', 'i4') # Set attributes nc.update_attributes(attributes) return ContiguousRaggedTrajectoryProfile(output, **kwargs)
def from_dataframe(cls, df, output, **kwargs): axes = get_default_axes(kwargs.pop('axes', {})) daxes = axes data_columns = [d for d in df.columns if d not in axes] reduce_dims = kwargs.pop('reduce_dims', False) unlimited = kwargs.pop('unlimited', False) unique_dims = kwargs.pop('unique_dims', False) if unique_dims is True: # Rename the dimension to avoid a dimension and coordinate having the same name # which is not supported in xarray changed_axes = { k: '{}_dim'.format(v) for k, v in axes._asdict().items() } daxes = get_default_axes(changed_axes) # Downcast anything from int64 to int32 # Convert any timezone aware datetimes to native UTC times df = downcast_dataframe(nativize_times(df)) # Make a new index that is the Cartesian product of all of the values from all of the # values of the old index. This is so don't have to iterate over anything. The full column # of data will be able to be shaped to the size of the final unique sized dimensions. index_order = [axes.t, axes.z, axes.station] df = df.set_index(index_order) df = df.reindex( pd.MultiIndex.from_product(df.index.levels, names=index_order)) unique_z = df.index.get_level_values(axes.z).unique().values unique_t = df.index.get_level_values( axes.t).unique().tolist() # tolist converts to Timestamp all_stations = df.index.get_level_values(axes.station) unique_s = all_stations.unique() with OrthogonalMultidimensionalTimeseriesProfile(output, 'w') as nc: if reduce_dims is True and unique_s.size == 1: # If a singular trajectory, we can reduce that dimension if it is of size 1 default_dimensions = (daxes.t, daxes.z) station_dimensions = () else: default_dimensions = (daxes.t, daxes.z, daxes.station) station_dimensions = (daxes.station, ) nc.createDimension(daxes.station, unique_s.size) station = nc.createVariable(axes.station, get_dtype(unique_s), station_dimensions) latitude = nc.createVariable(axes.y, get_dtype(df[axes.y]), station_dimensions) longitude = nc.createVariable(axes.x, get_dtype(df[axes.x]), station_dimensions) # Assign over loop because VLEN variables (strings) have to be assigned by integer index # and we need to find the lat/lon based on station index for si, st in enumerate(unique_s): station[si] = st latitude[si] = df[axes.y][all_stations == st].dropna().iloc[0] longitude[si] = df[axes.x][all_stations == st].dropna().iloc[0] # Metadata variables nc.createVariable('crs', 'i4') # Create all of the variables if unlimited is True: nc.createDimension(daxes.t, None) else: nc.createDimension(daxes.t, len(unique_t)) time = nc.createVariable(axes.t, 'f8', (daxes.t, )) time[:] = date2num(unique_t, units=cls.default_time_unit).astype('f8') nc.createDimension(daxes.z, unique_z.size) z = nc.createVariable(axes.z, get_dtype(unique_z), (daxes.z, )) z[:] = unique_z attributes = dict_update(nc.nc_attributes(axes, daxes), kwargs.pop('attributes', {})) # Variables defined on only the time axis and not the depth axis detach_z_vars = kwargs.pop('detach_z', []) detach_z_columnms = [p for p in detach_z_vars if p in data_columns] for c in detach_z_columnms: var_name = cf_safe_name(c) if var_name not in nc.variables: v = create_ncvar_from_series( nc, var_name, default_dimensions[ 0::2], # this removes the second dimension (z) df[c], zlib=True, complevel=1) attributes[var_name] = dict_update( attributes.get(var_name, {}), { 'coordinates': '{} {} {}'.format(axes.t, axes.x, axes.y) }) else: v = nc.variables[var_name] # Because we need access to the fillvalues here, we ask not to return # the values with them already filled. vvalues = get_ncdata_from_series(df[c], v, fillna=False) # Reshape to the full array, with Z vvalues = vvalues.reshape(len(unique_t), unique_z.size, unique_s.size) # The Z axis is always the second axis, take the mean over that axis vvalues = np.apply_along_axis(np.nanmean, 1, vvalues).flatten() # Now reshape to the array without Z vvalues = vvalues.reshape(len(unique_t), unique_s.size) try: v[:] = vvalues.reshape(v.shape) except BaseException: L.exception('Failed to add {}'.format(c)) continue full_columns = [ f for f in data_columns if f not in detach_z_columnms ] for c in full_columns: # Create variable if it doesn't exist var_name = cf_safe_name(c) if var_name not in nc.variables: v = create_ncvar_from_series(nc, var_name, default_dimensions, df[c], zlib=True, complevel=1) attributes[var_name] = dict_update( attributes.get(var_name, {}), { 'coordinates': '{} {} {} {}'.format(axes.t, axes.z, axes.x, axes.y) }) else: v = nc.variables[var_name] vvalues = get_ncdata_from_series(df[c], v) v[:] = vvalues.reshape(v.shape) nc.update_attributes(attributes) return OrthogonalMultidimensionalTimeseriesProfile(output, **kwargs)
def from_dataframe(cls, df, output, **kwargs): axes = get_default_axes(kwargs.pop('axes', {})) daxes = axes # Should never be a CR file with one trajectory so we ignore the "reduce_dims" attribute _ = kwargs.pop('reduce_dims', False) # noqa unlimited = kwargs.pop('unlimited', False) unique_dims = kwargs.pop('unique_dims', False) if unique_dims is True: # Rename the dimension to avoid a dimension and coordinate having the same name # which is not support in xarray changed_axes = { k: '{}_dim'.format(v) for k, v in axes._asdict().items() } daxes = get_default_axes(changed_axes) # Downcast anything from int64 to int32 # Convert any timezone aware datetimes to native UTC times df = downcast_dataframe(nativize_times(df)) with ContiguousRaggedTrajectory(output, 'w') as nc: trajectory_groups = df.groupby(axes.trajectory) unique_trajectories = list(trajectory_groups.groups.keys()) num_trajectories = len(unique_trajectories) nc.createDimension(daxes.trajectory, num_trajectories) trajectory = nc.createVariable(axes.trajectory, get_dtype(df[axes.trajectory]), (daxes.trajectory, )) # Get unique obs by grouping on traj getting the max size if unlimited is True: nc.createDimension(daxes.sample, None) else: nc.createDimension(daxes.sample, len(df)) # Number of observations in each trajectory row_size = nc.createVariable('rowSize', 'i4', (daxes.trajectory, )) attributes = dict_update(nc.nc_attributes(axes, daxes), kwargs.pop('attributes', {})) # Variables defined on only the trajectory axis traj_vars = kwargs.pop('traj_vars', []) traj_columns = [p for p in traj_vars if p in df.columns] for c in traj_columns: var_name = cf_safe_name(c) if var_name not in nc.variables: create_ncvar_from_series(nc, var_name, (daxes.trajectory, ), df[c], zlib=True, complevel=1) for i, (trajid, trg) in enumerate(trajectory_groups): trajectory[i] = trajid row_size[i] = len(trg) # Save any trajectory variables using the first value found # in the column. for c in traj_columns: var_name = cf_safe_name(c) if var_name not in nc.variables: continue v = nc.variables[var_name] vvalues = get_ncdata_from_series(trg[c], v)[0] try: v[i] = vvalues except BaseException: L.exception('Failed to add {}'.format(c)) continue # Add all of the columns based on the sample dimension. Take all columns and remove the # trajectory, rowSize and other trajectory based columns. sample_columns = [ f for f in df.columns if f not in traj_columns + ['rowSize', axes.trajectory] ] for c in sample_columns: var_name = cf_safe_name(c) if var_name not in nc.variables: v = create_ncvar_from_series(nc, var_name, (daxes.sample, ), df[c], zlib=True, complevel=1) else: v = nc.variables[var_name] vvalues = get_ncdata_from_series(df[c], v) try: if unlimited is True: v[:] = vvalues else: v[:] = vvalues.reshape(v.shape) except BaseException: L.exception('Failed to add {}'.format(c)) continue # Metadata variables if 'crs' not in nc.variables: nc.createVariable('crs', 'i4') # Set attributes nc.update_attributes(attributes) return ContiguousRaggedTrajectory(output, **kwargs)
def to_dataframe(self, clean_cols=True, clean_rows=True): # Z zvar = self.z_axes()[0] z = np.ma.fix_invalid(np.ma.MaskedArray(zvar[:])) z = z.flatten().round(5) logger.debug(['z data size: ', z.size]) # T tvar = self.t_axes()[0] t = np.ma.MaskedArray(nc4.num2date(tvar[:], tvar.units, getattr(tvar, 'calendar', 'standard'))).flatten() # Patch the time variable back to its original mask, since num2date # breaks any missing/fill values if hasattr(tvar[0], 'mask'): t.mask = tvar[:].mask logger.debug(['time data size: ', t.size]) # X xvar = self.x_axes()[0] x = np.ma.fix_invalid(np.ma.MaskedArray(xvar[:])).flatten().round(5) logger.debug(['x data size: ', x.size]) # Y yvar = self.y_axes()[0] y = np.ma.fix_invalid(np.ma.MaskedArray(yvar[:])).flatten().round(5) logger.debug(['y data size: ', y.size]) # Trajectories pvar = self.filter_by_attrs(cf_role='trajectory_id')[0] try: p = normalize_array(pvar) except BaseException: logger.exception('Could not pull trajectory values from the variable, using indexes.') p = np.asarray(list(range(len(pvar))), dtype=np.integer) # The Dimension that the trajectory id variable doesn't have is what # the trajectory data needs to be repeated by dim_diff = self.dimensions[list(set(tvar.dimensions).difference(set(pvar.dimensions)))[0]] if dim_diff: p = p.repeat(dim_diff.size) logger.debug(['trajectory data size: ', p.size]) # Distance d = np.append([0], great_distance(start_latitude=y[0:-1], end_latitude=y[1:], start_longitude=x[0:-1], end_longitude=x[1:])['distance']) d = np.ma.fix_invalid(np.ma.MaskedArray(np.cumsum(d)).astype(np.float64).round(2)) logger.debug(['distance data size: ', d.size]) df_data = { 't': t, 'x': x, 'y': y, 'z': z, 'trajectory': p, 'distance': d } building_index_to_drop = np.ones(t.size, dtype=bool) extract_vars = list(set(self.data_vars() + self.ancillary_vars())) for i, dvar in enumerate(extract_vars): vdata = np.ma.fix_invalid(np.ma.MaskedArray(dvar[:].round(3).flatten())) building_index_to_drop = (building_index_to_drop == True) & (vdata.mask == True) # noqa df_data[dvar.name] = vdata df = pd.DataFrame(df_data) # Drop all data columns with no data if clean_cols: df = df.dropna(axis=1, how='all') # Drop all data rows with no data variable data if clean_rows: df = df.iloc[~building_index_to_drop] return df