def extract(self, theForceFlag=False):
        """Checking the grid.xml file in the machine, if found use it.
        Else, download from the server
        :param theForceFlag:
        """
        myFinalGridXmlFile = os.path.join(self.extractDir(), 'grid.xml')
        if not os.path.exists(self.extractDir()):
            mkDir(self.extractDir())
        if theForceFlag or self.forceFlag:
            self.removeExtractedFiles()
        elif os.path.exists(myFinalGridXmlFile):
            return myFinalGridXmlFile

        # download data
        myLocalPath = self.fetchFile()
        # move grid.xml to the correct directory
        myExpectedGridXmlFile = os.path.join(myLocalPath, 'output', 'grid.xml')
        if not os.path.exists(myExpectedGridXmlFile):
            raise FileNotFoundError(
                'The output does not contain an ''%s file.' %
                myExpectedGridXmlFile)

        # move the file we care about to the top of the extract dir
        shutil.copyfile(os.path.join(
            self.extractDir(), myExpectedGridXmlFile), myFinalGridXmlFile)
        if not os.path.exists(myFinalGridXmlFile):
            raise CopyError('Error copying grid.xml')
        return myFinalGridXmlFile
Example #2
0
    def extract(self, theForceFlag=False):
        """Checking the grid.xml file in the machine, if found use it.
        Else, download from the server
        """
        myFinalGridXmlFile = os.path.join(self.extractDir(), 'grid.xml')
        if not os.path.exists(self.extractDir()):
            mkDir(self.extractDir())
        if theForceFlag or self.forceFlag:
            self.removeExtractedFiles()
        elif os.path.exists(myFinalGridXmlFile):
            return myFinalGridXmlFile

        # download data
        myLocalPath = self.fetchFile()
        # move grid.xml to the correct directory
        myExpectedGridXmlFile = os.path.join(myLocalPath, 'output', 'grid.xml')
        if not os.path.exists(myExpectedGridXmlFile):
            raise FileNotFoundError('The output does not contain an '
                                    '%s file.' % myExpectedGridXmlFile)

        # move the file we care about to the top of the extract dir
        shutil.copyfile(os.path.join(self.extractDir(), myExpectedGridXmlFile),
                        myFinalGridXmlFile)
        if (not os.path.exists(myFinalGridXmlFile)):
            raise CopyError('Error copying grid.xml')
        return myFinalGridXmlFile
Example #3
0
    def download_path(self, remote_path, local_path):
        """ Download remote_dir to local_dir.
        for example : remote_path = '20130111133900' will be download to
        local_dir/remote_path
        Must be in the parent directory of remote dir.
        """
        # Check if remote_dir is exist
        if not self.is_path_exist(remote_path):
            print 'remote path is not exist %s' % remote_path
            return False
        if self.is_dir(remote_path):
            # get directory name
            dir_name = get_path_tail(remote_path)
            # create directory in local machine
            local_dir_path = os.path.join(local_path, dir_name)
            mkDir(local_dir_path)
            # list all directory in remote path
            list_dir = self.sftp.listdir(remote_path)
            # iterate recursive
            for my_dir in list_dir:
                new_remote_path = os.path.join(remote_path, my_dir)
                self.download_path(new_remote_path, local_dir_path)
        else:
            # download file to local_path
            file_name = get_path_tail(remote_path)
            local_file_path = os.path.join(local_path, file_name)

            LOGGER.info('file %s will be downloaded to %s' % (remote_path,
                                                        local_file_path))
            self.sftp.get(remote_path, local_file_path)
Example #4
0
    def download_path(self, remote_path, local_path):
        """ Download remote_dir to local_dir.
        for example : remote_path = '20130111133900' will be download to
        local_dir/remote_path
        Must be in the parent directory of remote dir.
        """
        # Check if remote_dir is exist
        if not self.is_path_exist(remote_path):
            print 'remote path is not exist %s' % remote_path
            return False
        if self.is_dir(remote_path):
            # get directory name
            dir_name = get_path_tail(remote_path)
            # create directory in local machine
            local_dir_path = os.path.join(local_path, dir_name)
            mkDir(local_dir_path)
            # list all directory in remote path
            list_dir = self.sftp.listdir(remote_path)
            # iterate recursive
            for my_dir in list_dir:
                new_remote_path = os.path.join(remote_path, my_dir)
                self.download_path(new_remote_path, local_dir_path)
        else:
            # download file to local_path
            file_name = get_path_tail(remote_path)
            local_file_path = os.path.join(local_path, file_name)

            LOGGER.info('file %s will be downloaded to %s' %
                        (remote_path, local_file_path))
            self.sftp.get(remote_path, local_file_path)
Example #5
0
    def fetchFile(self, theRetries=3):
        """Private helper to fetch a file from the sftp site.

          e.g. for event 20110413170148 this file would be fetched::
                20110413170148 directory

        .. note:: If a cached copy of the file exits, the path to the cache
           copy will simply be returned without invoking any network requests.

        Args:
            * theEventFile: str - filename on server e.g.20110413170148.inp.zip
            * theRetries: int - number of reattempts that should be made in
                in case of network error etc.

        Returns:
            str: A string for the dataset path on the local storage system.

        Raises:
            EventUndefinedError, NetworkError
        """
        myLocalPath = os.path.join(shakemapCacheDir(), self.eventId)
        myLocalParentPath = os.path.join(myLocalPath, 'output')
        myXMLFile = os.path.join(myLocalParentPath, self.fileName())
        if os.path.exists(myXMLFile):
            return myLocalPath

        # fetch from sftp
        trials = [i + 1 for i in xrange(theRetries)]
        remote_path = os.path.join(self.sftpclient.workdir_path, self.eventId)
        myXMLRemotePath = os.path.join(remote_path, 'output', self.fileName())
        for my_counter in trials:
            myLastError = None
            try:
                mkDir(myLocalPath)
                mkDir(os.path.join(myLocalPath, 'output'))
                self.sftpclient.download_path(myXMLRemotePath,
                                              myLocalParentPath)
            except NetworkError, e:
                myLastError = e
            except:
Example #6
0
    def fetchFile(self, theRetries=3):
        """Private helper to fetch a file from the sftp site.

          e.g. for event 20110413170148 this file would be fetched::
                20110413170148 directory

        .. note:: If a cached copy of the file exits, the path to the cache
           copy will simply be returned without invoking any network requests.

        Args:
            * theEventFile: str - filename on server e.g.20110413170148.inp.zip
            * theRetries: int - number of reattempts that should be made in
                in case of network error etc.

        Returns:
            str: A string for the dataset path on the local storage system.

        Raises:
            EventUndefinedError, NetworkError
        """
        myLocalPath = os.path.join(shakemapCacheDir(), self.eventId)
        myLocalParentPath = os.path.join(myLocalPath, 'output')
        myXMLFile = os.path.join(myLocalParentPath, self.fileName())
        if os.path.exists(myXMLFile):
            return myLocalPath

        # fetch from sftp
        trials = [i + 1 for i in xrange(theRetries)]
        remote_path = os.path.join(self.sftpclient.workdir_path, self.eventId)
        myXMLRemotePath = os.path.join(remote_path, 'output', self.fileName())
        for my_counter in trials:
            myLastError = None
            try:
                mkDir(myLocalPath)
                mkDir(os.path.join(myLocalPath, 'output'))
                self.sftpclient.download_path(myXMLRemotePath,
                    myLocalParentPath)
            except NetworkError, e:
                myLastError = e
            except:
Example #7
0
#!/bin/env python
# simple test for the job submission tools

from cluster_job import (PBS_Script, runJobAndWait)
from utils import mkDir

logDirName='log'
mkDir(logDirName)
script = PBS_Script(command='echo hello world', walltime='0:1:0', name='pipeline_test_job', memInGB=1, logDir=logDirName)
print('before job has run')
runJobAndWait(script, 'test_stage', logDir=logDirName, verbose=1)
print('after job has run')