def cacheDir(self,subdirs=None):
"""returns the cache directory of this scheme and dataset"""
if self._dbinvent.cache:
if subdirs:
return getCreateDir(os.path.join(self._dbinvent.cache,subdirs))
else:
return getCreateDir(self._dbinvent.cache)
return self.conf.getCacheDir(self.scheme, dataset=self.name,subdirs=subdirs)
def __init__(self, dbconn):
super().__init__(dbconn)
#initialize postgreslq table
GravitySHTBase.metadata.create_all(self.db.dbeng, checkfirst=True)
if not self._dbinvent.datadir:
self._dbinvent.datadir = getCreateDir(
os.path.join(self.conf.getDataDir(self.scheme), self.release,
self.subdirs))
def dataDir(self,subdirs=None):
"""Returns the specialized data directory of this scheme and dataset
The directory will be created if it does not exist"""
if self._dbinvent.datadir:
return getCreateDir(self._dbinvent.datadir)
#else try to retrieve the standard datadir from the configuration
return self.conf.getDataDir(self.scheme, dataset=self.name,subdirs=subdirs)
def pull(self, cycle=None):
"""Pulls the data from the rads server
:param cycle: only pulls data from a specific cycle
"""
cred=self.conf.authCred("rads")
url="rads.tudelft.nl::rads/data"
#pull configuration data (xml files)
rsync(url+"/conf",auth=cred).parallelDownload(self._dbinvent.datadir,True)
srcurl=os.path.join(url,self.sat,self.phase)
desturl=os.path.join(self._dbinvent.datadir,self.sat)
if cycle:
srcurl=os.path.join(srcurl,"c%03d"%(cycle))
desturl=os.path.join(desturl,self.phase)
getCreateDir(desturl)
self.updated=rsync(srcurl,auth=cred).parallelDownload(desturl,True)
def __init__(self, dbconn):
super().__init__(dbconn)
self.updated = None
#possibly use an external location for the data
if not self._dbinvent.datadir:
if 'RADSDATAROOT' in os.environ:
self._dbinvent.datadir = getCreateDir(
os.environ['RADSDATAROOT'])
else:
self._dbinvent.datadir = self.conf.getDataDir(self.scheme,
subdirs="RADS")
self.updateInvent(False)
def __init__(self,dbconn):
super().__init__(dbconn)
self.updated=None
if not self._dbinvent.datadir:
if 'RADSDATAROOT' in os.environ:
self._dbinvent.datadir=getCreateDir(os.environ['RADSDATAROOT'])
else:
self._dbinvent.datadir=self.conf.getDataDir(self.scheme,subdirs="RADS")
self.updateInvent(False)
#initialize postgreslq table
self.table.__table__.create(self.db.dbeng,checkfirst=True)
def pull(self, cycle=None, passes=None):
"""Pulls the data from the rads server
:param cycle: only pulls data from a specific cycle
"""
cred = self.conf.authCred("rads")
url = "rads.tudelft.nl::rads/data"
#note we need the / at the end so we set the upstream root after the phase
srcurl = os.path.join(url, self.sat, self.phase) + "/"
desturl = os.path.join(self._dbinvent.datadir, self.sat, self.phase)
getCreateDir(desturl)
include = None
if cycle or passes:
#set up include constraints for rsync
if cycle:
#which directories to include in rsync filter
if isinstance(cycle, list):
include = [f"/c{cyc:03d}" for cyc in cycle]
else:
include = [f"/c{cycle:03d}"]
else:
include = []
if passes:
if isinstance(passes, list):
include.extend(
[f"{self.sat}p{pss:04d}*nc" for pss in passes])
else:
include.append(f"{self.sat}p{passes:04d}*nc")
else:
include.append(f"{self.sat}*nc")
slurplogger().info(f"rsyncing rads data to {desturl}")
self.updated = rsync(srcurl, auth=cred).parallelDownload(
desturl, True, include)