def extract(self, theForceFlag=False):
"""Checking the grid.xml file in the machine, if found use it.
Else, download from the server
:param theForceFlag:
"""
myFinalGridXmlFile = os.path.join(self.extractDir(), 'grid.xml')
if not os.path.exists(self.extractDir()):
mkDir(self.extractDir())
if theForceFlag or self.forceFlag:
self.removeExtractedFiles()
elif os.path.exists(myFinalGridXmlFile):
return myFinalGridXmlFile
# download data
myLocalPath = self.fetchFile()
# move grid.xml to the correct directory
myExpectedGridXmlFile = os.path.join(myLocalPath, 'output', 'grid.xml')
if not os.path.exists(myExpectedGridXmlFile):
raise FileNotFoundError(
'The output does not contain an ''%s file.' %
myExpectedGridXmlFile)
# move the file we care about to the top of the extract dir
shutil.copyfile(os.path.join(
self.extractDir(), myExpectedGridXmlFile), myFinalGridXmlFile)
if not os.path.exists(myFinalGridXmlFile):
raise CopyError('Error copying grid.xml')
return myFinalGridXmlFile
def extract(self, theForceFlag=False):
"""Checking the grid.xml file in the machine, if found use it.
Else, download from the server
"""
myFinalGridXmlFile = os.path.join(self.extractDir(), 'grid.xml')
if not os.path.exists(self.extractDir()):
mkDir(self.extractDir())
if theForceFlag or self.forceFlag:
self.removeExtractedFiles()
elif os.path.exists(myFinalGridXmlFile):
return myFinalGridXmlFile
# download data
myLocalPath = self.fetchFile()
# move grid.xml to the correct directory
myExpectedGridXmlFile = os.path.join(myLocalPath, 'output', 'grid.xml')
if not os.path.exists(myExpectedGridXmlFile):
raise FileNotFoundError('The output does not contain an '
'%s file.' % myExpectedGridXmlFile)
# move the file we care about to the top of the extract dir
shutil.copyfile(os.path.join(self.extractDir(), myExpectedGridXmlFile),
myFinalGridXmlFile)
if (not os.path.exists(myFinalGridXmlFile)):
raise CopyError('Error copying grid.xml')
return myFinalGridXmlFile
def download_path(self, remote_path, local_path):
""" Download remote_dir to local_dir.
for example : remote_path = '20130111133900' will be download to
local_dir/remote_path
Must be in the parent directory of remote dir.
"""
# Check if remote_dir is exist
if not self.is_path_exist(remote_path):
print 'remote path is not exist %s' % remote_path
return False
if self.is_dir(remote_path):
# get directory name
dir_name = get_path_tail(remote_path)
# create directory in local machine
local_dir_path = os.path.join(local_path, dir_name)
mkDir(local_dir_path)
# list all directory in remote path
list_dir = self.sftp.listdir(remote_path)
# iterate recursive
for my_dir in list_dir:
new_remote_path = os.path.join(remote_path, my_dir)
self.download_path(new_remote_path, local_dir_path)
else:
# download file to local_path
file_name = get_path_tail(remote_path)
local_file_path = os.path.join(local_path, file_name)
LOGGER.info('file %s will be downloaded to %s' % (remote_path,
local_file_path))
self.sftp.get(remote_path, local_file_path)
def download_path(self, remote_path, local_path):
""" Download remote_dir to local_dir.
for example : remote_path = '20130111133900' will be download to
local_dir/remote_path
Must be in the parent directory of remote dir.
"""
# Check if remote_dir is exist
if not self.is_path_exist(remote_path):
print 'remote path is not exist %s' % remote_path
return False
if self.is_dir(remote_path):
# get directory name
dir_name = get_path_tail(remote_path)
# create directory in local machine
local_dir_path = os.path.join(local_path, dir_name)
mkDir(local_dir_path)
# list all directory in remote path
list_dir = self.sftp.listdir(remote_path)
# iterate recursive
for my_dir in list_dir:
new_remote_path = os.path.join(remote_path, my_dir)
self.download_path(new_remote_path, local_dir_path)
else:
# download file to local_path
file_name = get_path_tail(remote_path)
local_file_path = os.path.join(local_path, file_name)
LOGGER.info('file %s will be downloaded to %s' %
(remote_path, local_file_path))
self.sftp.get(remote_path, local_file_path)
def fetchFile(self, theRetries=3):
"""Private helper to fetch a file from the sftp site.
e.g. for event 20110413170148 this file would be fetched::
20110413170148 directory
.. note:: If a cached copy of the file exits, the path to the cache
copy will simply be returned without invoking any network requests.
Args:
* theEventFile: str - filename on server e.g.20110413170148.inp.zip
* theRetries: int - number of reattempts that should be made in
in case of network error etc.
Returns:
str: A string for the dataset path on the local storage system.
Raises:
EventUndefinedError, NetworkError
"""
myLocalPath = os.path.join(shakemapCacheDir(), self.eventId)
myLocalParentPath = os.path.join(myLocalPath, 'output')
myXMLFile = os.path.join(myLocalParentPath, self.fileName())
if os.path.exists(myXMLFile):
return myLocalPath
# fetch from sftp
trials = [i + 1 for i in xrange(theRetries)]
remote_path = os.path.join(self.sftpclient.workdir_path, self.eventId)
myXMLRemotePath = os.path.join(remote_path, 'output', self.fileName())
for my_counter in trials:
myLastError = None
try:
mkDir(myLocalPath)
mkDir(os.path.join(myLocalPath, 'output'))
self.sftpclient.download_path(myXMLRemotePath,
myLocalParentPath)
except NetworkError, e:
myLastError = e
except:
def fetchFile(self, theRetries=3):
"""Private helper to fetch a file from the sftp site.
e.g. for event 20110413170148 this file would be fetched::
20110413170148 directory
.. note:: If a cached copy of the file exits, the path to the cache
copy will simply be returned without invoking any network requests.
Args:
* theEventFile: str - filename on server e.g.20110413170148.inp.zip
* theRetries: int - number of reattempts that should be made in
in case of network error etc.
Returns:
str: A string for the dataset path on the local storage system.
Raises:
EventUndefinedError, NetworkError
"""
myLocalPath = os.path.join(shakemapCacheDir(), self.eventId)
myLocalParentPath = os.path.join(myLocalPath, 'output')
myXMLFile = os.path.join(myLocalParentPath, self.fileName())
if os.path.exists(myXMLFile):
return myLocalPath
# fetch from sftp
trials = [i + 1 for i in xrange(theRetries)]
remote_path = os.path.join(self.sftpclient.workdir_path, self.eventId)
myXMLRemotePath = os.path.join(remote_path, 'output', self.fileName())
for my_counter in trials:
myLastError = None
try:
mkDir(myLocalPath)
mkDir(os.path.join(myLocalPath, 'output'))
self.sftpclient.download_path(myXMLRemotePath,
myLocalParentPath)
except NetworkError, e:
myLastError = e
except:
#!/bin/env python
# simple test for the job submission tools
from cluster_job import (PBS_Script, runJobAndWait)
from utils import mkDir
logDirName='log'
mkDir(logDirName)
script = PBS_Script(command='echo hello world', walltime='0:1:0', name='pipeline_test_job', memInGB=1, logDir=logDirName)
print('before job has run')
runJobAndWait(script, 'test_stage', logDir=logDirName, verbose=1)
print('after job has run')