コード例 #1
0
ファイル: g2files.py プロジェクト: ricardogsilva/webg2system
 def _get_io_buffers(self, settings):
     specific_buffers = settings.specific_io_buffers.all()
     if len(specific_buffers) == 0:
         specific_buffers = [hs for hs in \
             ss.Host.objects.filter(role__name='io buffer', active=True)]
     hf = HostFactory()
     return [hf.create_host(hs) for hs in specific_buffers]
コード例 #2
0
ファイル: g2item.py プロジェクト: ricardogsilva/webg2system
    def __init__(self, timeslot, area, host=None, logger=None, callback=None):
        '''
        Inputs:

            timeslot - A datetime.datetime object

            area - A string with the name of the area

            host - A systemsettings.models.Host object. If None, the
                current host will be used.
        '''

        if callback is None:
            def cb(*args):
                pass
            self.callback = cb
        else:
            self.callback = callback
        self.logger = logger
        self.timeslot = timeslot
        self.source = G2Source(area, timeslot)
        #hf = HostFactory()
        hf = HostFactory(logger=self.logger)
        self.host = hf.create_host(host)
        # a random number for generating unique working dirs
        self.random = randint(0, 100)
コード例 #3
0
 def _get_io_buffers(self, settings):
     specific_buffers = settings.specific_io_buffers.all()
     if len(specific_buffers) == 0:
         specific_buffers = [hs for hs in \
             ss.Host.objects.filter(role__name='io buffer', active=True)]
     hf = HostFactory()
     return [hf.create_host(hs) for hs in specific_buffers]
コード例 #4
0
ファイル: g2files.py プロジェクト: ricardogsilva/webg2system
    def _get_archives(self, settings):
        '''
        Return a list of G2Host instances that are the archives for this file.

        The G2Host list is created from the specificArchives attribute. If
        there are no specificArchives defined, then all the hosts that have
        the 'archive' role and are 'active' will be used.
        '''

        specific_archives = settings.specificArchives.all()
        if len(specific_archives) == 0:
            specific_archives = [hs for hs in \
                ss.Host.objects.filter(role__name='archive', active=True)]
        hf = HostFactory(logger=self.logger)
        return [hf.create_host(hs) for hs in specific_archives]
コード例 #5
0
    def _get_archives(self, settings):
        '''
        Return a list of G2Host instances that are the archives for this file.

        The G2Host list is created from the specificArchives attribute. If
        there are no specificArchives defined, then all the hosts that have
        the 'archive' role and are 'active' will be used.
        '''

        specific_archives = settings.specificArchives.all()
        if len(specific_archives) == 0:
            specific_archives = [hs for hs in \
                ss.Host.objects.filter(role__name='archive', active=True)]
        hf = HostFactory(logger=self.logger)
        return [hf.create_host(hs) for hs in specific_archives]
コード例 #6
0
ファイル: g2files.py プロジェクト: ricardogsilva/webg2system
    def disseminate(self, temp_dir, dest_host=None, dest_dir='', file_path=0,
                    remote_protocol='sftp', compress=False, notify=[],
                    use_archive=False, use_io_buffer=False):
        '''
        Send file from its own host to dest_host.

        Inputs:

            temp_dir - A string specifying a temporary directory where the 
                files will be placed in case they have to be fetched from 
                other host

            dest_host - A G2Host instance. A value of None (the default)
                will be interpreted as the local host.

            dest_dir - A string with the path on the remote host where
                the file is to be sent to. If it starts with / then an
                absolute path is assumed. Otherwise the path will be
                assumed to be relative to dest_host's dataPath. A value
                of None (the default) will cause the file to be disseminated
                to the same directory structure as if has defined in its
                settings, only relative to it's target host's dataPath.

            file_path - An integer specifying which of the file's paths
                are to be used.

            remote_protocol - A string indicating the name of the remote
                protocol to use when sending the files. Currently supported
                protocols are 'sftp' and 'ftp'. Defaults to 'sftp'.

            compress - A boolean indicating if the files are to be compressed
                before sending

            notify - A list of string indicating types of notification to send
                with the sending status.

            use_archive - A boolean indicating if the files should be fetched
                from the archives in case they are not found locally

            use_io_buffer - A boolean indicating if the files should be fetched
                from the available io buffers in case they are not found 
                locally
        '''

        if dest_host is None:
            dest_host = HostFactory.get_host(logger=self.logger)
        if dest_dir is None:
            target_path = os.path.join(dest_host.dataPath,
                                    self.searchPath[file_path])
        elif dest_dir.startswith('/'):
            target_path = dest_dir
        else:
            target_path = os.path.join(dest_host.dataPath, dest_dir)
        fetched = self.fetch(temp_dir, use_archive=use_archive, 
                             use_io_buffer=use_io_buffer, decompress=True,
                             restrict_pattern=None)
        result = False
        if len(fetched) > 0:
            if compress:
                fetched = self.host.compress(fetched)
            return_code, sent_paths = self.host.send(
                fetched,
                target_path,
                dest_host,
                remoteProtocol=remote_protocol
            )
            if return_code == 0:
                result = True
            for notification_type in notify:
                # not implemented yet
                pass
        return result
コード例 #7
0
    def disseminate(self,
                    temp_dir,
                    dest_host=None,
                    dest_dir='',
                    file_path=0,
                    remote_protocol='sftp',
                    compress=False,
                    notify=[],
                    use_archive=False,
                    use_io_buffer=False):
        '''
        Send file from its own host to dest_host.

        Inputs:

            temp_dir - A string specifying a temporary directory where the 
                files will be placed in case they have to be fetched from 
                other host

            dest_host - A G2Host instance. A value of None (the default)
                will be interpreted as the local host.

            dest_dir - A string with the path on the remote host where
                the file is to be sent to. If it starts with / then an
                absolute path is assumed. Otherwise the path will be
                assumed to be relative to dest_host's dataPath. A value
                of None (the default) will cause the file to be disseminated
                to the same directory structure as if has defined in its
                settings, only relative to it's target host's dataPath.

            file_path - An integer specifying which of the file's paths
                are to be used.

            remote_protocol - A string indicating the name of the remote
                protocol to use when sending the files. Currently supported
                protocols are 'sftp' and 'ftp'. Defaults to 'sftp'.

            compress - A boolean indicating if the files are to be compressed
                before sending

            notify - A list of string indicating types of notification to send
                with the sending status.

            use_archive - A boolean indicating if the files should be fetched
                from the archives in case they are not found locally

            use_io_buffer - A boolean indicating if the files should be fetched
                from the available io buffers in case they are not found 
                locally
        '''

        if dest_host is None:
            dest_host = HostFactory.get_host(logger=self.logger)
        if dest_dir is None:
            target_path = os.path.join(dest_host.dataPath,
                                       self.searchPath[file_path])
        elif dest_dir.startswith('/'):
            target_path = dest_dir
        else:
            target_path = os.path.join(dest_host.dataPath, dest_dir)
        fetched = self.fetch(temp_dir,
                             use_archive=use_archive,
                             use_io_buffer=use_io_buffer,
                             decompress=True,
                             restrict_pattern=None)
        result = False
        if len(fetched) > 0:
            if compress:
                fetched = self.host.compress(fetched)
            return_code, sent_paths = self.host.send(
                fetched,
                target_path,
                dest_host,
                remoteProtocol=remote_protocol)
            if return_code == 0:
                result = True
            for notification_type in notify:
                # not implemented yet
                pass
        return result