def folder2catalogue(path,
granule_trunk='*',
granule_extension='*',
add_sf=False,
client=None):
term = '{path}/{granule_trunk}*[!_stare].{ext}'.format(
path=path, granule_trunk=granule_trunk, ext=granule_extension)
s3 = None
if path[0:5] != 's3://':
granule_names = glob.glob(term)
else:
granule_names, s3 = starepandas.s3_glob(
path, '.*\.{ext}$'.format(ext=granule_extension))
if not granule_names:
print('no granules in folder')
return None
df = starepandas.STAREDataFrame()
if client is None:
for granule_name in granule_names:
if s3 is not None:
granule_url = 's3://{bucket_name}/{granule}'.format(
bucket_name=s3[0]['bucket_name'], granule=granule_name)
else:
granule_url = granule_name
row = make_row(granule_url, add_sf)
df = df.append(row, ignore_index=True)
else:
pass
# client=Client()
# client.close()
df.set_stare('stare_cover', inplace=True)
if add_sf:
df.set_geometry('geom', inplace=True)
return df
def read_pods(pod_root, sids, pattern, add_podname=False):
""" Reads a STAREDataframe from a directory of STAREPods
:param add_podname: toggle if the podname should be read as a parameter
:type add_podname: bool
:param pod_root: Root directory containing the pods
:type pod_root: string
:param sids: STARE index values to read pods for
:type sids: list/array-like
:param pattern: name pattern of chunks to read
:type pattern: str
:return: starepandas.STAREDataFrame
:rtype: starepandas.STAREDataFrame
"""
df = starepandas.STAREDataFrame()
for sid in sids:
pod_path = '{pod_root}/{sid}/'.format(pod_root=pod_root, sid=sid)
if not os.path.exists(pod_path):
print('no pod exists for {}'.format(sid))
continue
pickles = sorted(glob.glob(os.path.expanduser(pod_path + '/*')))
search = '.*{pattern}.*'.format(pattern=pattern)
pods = list(filter(re.compile(search).match, pickles))
for pod in pods:
df = df.append(pandas.read_pickle(pod))
if add_podname:
df['pod'] = pod
return df
def to_df(self, xy=False):
""" Converts the granule object to a dataframe
Parameters
-----------
xy: bool
If true, add columns for the original array coordinates
Returns
--------
df: STAREDataFrame
dataframe containing lat, lon, xy, and data; one row per observation
"""
df = {}
if self.lat is not None:
df['lat'] = self.lat.flatten()
df['lon'] = self.lon.flatten()
if self.stare is not None:
df['sids'] = self.stare.flatten()
if xy:
if self.lat is not None:
indices = numpy.indices(self.lat.shape)
elif self.stare is not None:
indices = numpy.indices(self.stare.shape)
else:
ds = list(self.data.values())[0]
indices = numpy.indices(ds.shape)
df['x'] = indices[0].flatten()
df['y'] = indices[1].flatten()
for key in self.data.keys():
df[key] = self.data[key].flatten()
return starepandas.STAREDataFrame(df)
def to_df(self):
df = {}
df['lat'] = self.lat.flatten()
df['lon'] = self.lon.flatten()
df['stare'] = self.stare.flatten()
for key in self.data.keys():
df[key] = self.data[key].flatten()
return starepandas.STAREDataFrame(df)
def plot(self, trixels=False, boundary=False, *args, **kwargs):
if trixels==True:
boundary = True
df = self.set_geometry(self._trixel_column_name)
else:
df = self
if boundary:
df = df[df.geometry.is_empty==False]
df = starepandas.STAREDataFrame(df)
df = df.set_geometry(df.geometry.boundary)
return super(STAREDataFrame, df).plot(*args, **kwargs)
def read_sql_table(sql, con, stare_column='stare', **kwargs):
df = pandas.read_sql_table(sql, con)
df = starepandas.STAREDataFrame(df)
if stare_column in df.columns:
df[stare_column] = df[stare_column].apply(func=numpy.frombuffer,
args=('int64', ))
df.set_sids(stare_column, inplace=True)
else:
print('{} is not in the columns'.format(stare_column))
return df
def folder2catalog(path, granule_trunk='', granule_extension='*', add_sf=False, client=None):
""" Reads a folder of granules into a STAREDataFrame catalog
:param path: Path of the folder containing granules
:type path: str
:param granule_trunk: Granule identifier (e.g. MOD09)
:type granule_trunk: str
:param granule_extension: Extension of the granule (e.g. hdf, nc, HDF5)
:type granule_extension: str
:param add_sf: toggle creating simple feature representation of the iFOVs
:type add_sf: bool
:param client:
:type client:
:return: catalog
:rtype: starepandas.STAREDataFrame
"""
term = '{path}/{granule_trunk}*.{ext}'.format(path=path, granule_trunk=granule_trunk, ext=granule_extension)
s3 = None
if path[0:5] != 's3://':
granule_paths = glob.glob(term)
else:
granule_paths, s3 = starepandas.io.s3.s3_glob(path, '.*\.{ext}$'.format(ext=granule_extension))
if not granule_paths:
print('no granules in folder')
return None
pattern = '.*[^_stare]\.(nc|hdf|HDF5)'
granule_paths = list(filter(re.compile(pattern).match, granule_paths))
df = starepandas.STAREDataFrame()
if client is None:
for granule_path in granule_paths:
if s3 is not None:
granule_url = 's3://{bucket_name}/{granule}'.format(bucket_name=s3[0]['bucket_name'],
granule=granule_path)
else:
granule_url = granule_path
row = make_row(granule_url, add_sf)
df = df.append(row, ignore_index=True)
else:
pass
# client=Client()
# client.close()
df.set_sids('stare_cover', inplace=True)
if add_sf:
df.set_geometry('geom', inplace=True)
return df