示例#1
0
 def _get_remote_appliance(self, request, ip, port, client_id,
                           client_secret):
     with self._handle_exception(request):
         base_url = ('https://%s:%s' % (ip, port))
         set_token(client_id=client_id, client_secret=client_secret,
                   url=base_url)
         ad = api_call('%s/api/appliances/1' % base_url)
         logger.debug('remote appliance: %s' % ad)
         return ad['uuid']
示例#2
0
def main():
    #  bootstrap the machine. success of quit
    url = 'https://localhost/api/commands/bootstrap'
    time.sleep(10)
    try:
        set_token()
        api_call(url, calltype='post')
    except Exception, e:
        logger.error('Unable to bootstrap the machine. Moving on..')
        logger.exception(e)
示例#3
0
def main():
    #  bootstrap the machine. success of quit
    url = 'https://localhost/api/commands/bootstrap'
    time.sleep(10)
    try:
        set_token()
        api_call(url, calltype='post')
    except Exception, e:
        logger.error('Unable to bootstrap the machine. Moving on..')
        logger.exception(e)
示例#4
0
 def __init__(self):
     self.ppid = os.getpid()
     self.senders = {}
     self.receivers = {}
     self.data_port = settings.REPLICA_DATA_PORT
     self.meta_port = settings.REPLICA_META_PORT
     self.recv_meta = None
     self.pubq = Queue()
     self.uuid = None
     set_token()
     super(ReplicaScheduler, self).__init__()
示例#5
0
 def __init__(self):
     self.ppid = os.getpid()
     self.senders = {}
     self.receivers = {}
     self.data_port = settings.REPLICA_DATA_PORT
     self.meta_port = settings.REPLICA_META_PORT
     self.recv_meta = None
     self.pubq = Queue()
     self.uuid = None
     set_token()
     super(ReplicaScheduler, self).__init__()
示例#6
0
def main():
    #  bootstrap the machine. success of quit
    api_url = 'https://localhost/api'
    bootstrap_url = ('%s/commands/bootstrap' % api_url)
    diskscan_url = ('%s/disks/scan' % api_url)
    netscan_url = ('%s/network' % api_url)
    time.sleep(10)
    try:
        set_token()
        api_call(diskscan_url, calltype='post')
        api_call(netscan_url, calltype='get')
        api_call(bootstrap_url, calltype='post')
    except Exception, e:
        logger.error('Unable to bootstrap the machine. Moving on..')
        logger.exception(e)
示例#7
0
def main():
    #  bootstrap the machine. success of quit
    api_url = 'https://localhost/api'
    bootstrap_url = ('%s/commands/bootstrap' % api_url)
    diskscan_url = ('%s/disks/scan' % api_url)
    netscan_url = ('%s/network' % api_url)
    time.sleep(10)
    try:
        set_token()
        api_call(diskscan_url, calltype='post')
        api_call(netscan_url, calltype='get')
        api_call(bootstrap_url, calltype='post')
    except Exception, e:
        logger.error('Unable to bootstrap the machine. Moving on..')
        logger.exception(e)
示例#8
0
 def _get_remote_appliance(self, request, ip, port, client_id,
                           client_secret):
     try:
         base_url = ('https://%s:%s' % (ip, port))
         set_token(client_id=client_id, client_secret=client_secret,
                   url=base_url)
         ad = api_call('%s/api/appliances/1' % base_url)
         logger.debug('remote appliance: %s' % ad)
         return ad['uuid']
     except RockStorAPIException:
         raise
     except Exception, e:
         e_msg = ('Failed to get remote appliance uuid')
         logger.error(e_msg)
         logger.exception(e)
         handle_exception(e_msg, request)
示例#9
0
 def _get_remote_appliance(self, request, ip, port, client_id,
                           client_secret):
     try:
         base_url = ('https://%s:%s' % (ip, port))
         set_token(client_id=client_id,
                   client_secret=client_secret,
                   url=base_url)
         ad = api_call('%s/api/appliances/1' % base_url)
         logger.debug('remote appliance: %s' % ad)
         return ad['uuid']
     except RockStorAPIException:
         raise
     except Exception, e:
         e_msg = ('Failed to get remote appliance uuid')
         logger.error(e_msg)
         logger.exception(e)
         handle_exception(e_msg, request)
示例#10
0
 def _get_remote_appliance(self, request, ip, port, client_id,
                           client_secret):
     with self._handle_exception(request):
         base_url = ('https://%s:%s' % (ip, port))
         try:
             set_token(client_id=client_id, client_secret=client_secret,
                       url=base_url)
         except Exception, e:
             e_msg = ('Failed to authenticate on remote appliance. Verify '
                      'port number, id and secret are correct and try '
                      'again.')
             handle_exception(Exception(e_msg), request)
         try:
             ad = api_call('%s/api/appliances/1' % base_url,
                           save_error=False)
             return ad['uuid']
         except Exception, e:
             logger.exception(e)
             e_msg = ('Failed to get remote appliance information. Verify '
                      'all inputs and try again.')
             handle_exception(Exception(e_msg), request)
示例#11
0
 def _get_remote_appliance(self, request, ip, port, client_id,
                           client_secret):
     with self._handle_exception(request):
         base_url = ('https://%s:%s' % (ip, port))
         try:
             set_token(client_id=client_id, client_secret=client_secret,
                       url=base_url)
         except Exception, e:
             e_msg = ('Failed to authenticate on remote appliance. Verify '
                      'port number, id and secret are correct and try '
                      'again.')
             handle_exception(Exception(e_msg), request)
         try:
             ad = api_call('%s/api/appliances/1' % base_url,
                           save_error=False)
             return ad['uuid']
         except Exception, e:
             logger.exception(e)
             e_msg = ('Failed to get remote appliance information. Verify '
                      'all inputs and try again.')
             handle_exception(Exception(e_msg), request)
示例#12
0
    def run(self):
        set_token()
        msg = ('Failed to get the sender ip from the uuid(%s) for meta: %s' %
               (self.meta['uuid'], self.meta))
        with self._clean_exit_handler(msg):
            self.sender_ip = get_sender_ip(self.meta['uuid'], logger)
        logger.debug('sender ip: %s' % self.sender_ip)

        msg = ('Failed to connect to the sender(%s) on data_port(%s). meta: '
               '%s. Aborting.' % (self.sender_ip, self.data_port, self.meta))
        with self._clean_exit_handler(msg):
            #@todo: add validation
            recv_sub = self.ctx.socket(zmq.SUB)
            recv_sub.connect('tcp://%s:%d' % (self.sender_ip, self.data_port))
            recv_sub.RCVTIMEO = 100
            recv_sub.setsockopt(zmq.SUBSCRIBE, str(self.meta['id']))

        msg = ('Failed to connect to the sender(%s) on '
               'meta_port(%d). meta: %s. Aborting.' %
               (self.sender_ip, self.meta_port, self.meta))
        with self._clean_exit_handler(msg):
            self.meta_push = self.ctx.socket(zmq.PUSH)
            url = ('tcp://%s:%d' % (self.sender_ip, self.meta_port))
            logger.debug('meta url: %s' % url)
            self.meta_push.connect('tcp://%s:%d' %
                                   (self.sender_ip, self.meta_port))

        sname = ('%s-%s-%s' % (self.sender_id, self.sender_ip, self.src_share))
        if (not self.incremental):
            msg = ('Failed to verify/create share: %s. meta: %s. '
                   'Aborting.' % (sname, self.meta))
            with self._clean_exit_handler(msg, ack=True):
                create_share(sname, self.dest_pool, logger)

            msg = ('Failed to create the replica metadata object '
                   'for share: %s. meta: %s. Aborting.' % (sname, self.meta))
            with self._clean_exit_handler(msg, ack=True):
                data = {
                    'share': sname,
                    'appliance': self.sender_ip,
                    'src_share': self.src_share,
                    'data_port': self.data_port,
                    'meta_port': self.meta_port,
                }
                self.rid = create_rshare(data, logger)

        else:
            msg = ('Failed to retreive the replica metadata object for '
                   'share: %s. meta: %s. Aborting.' % (sname, self.meta))
            with self._clean_exit_handler(msg):
                self.rid = rshare_id(sname, logger)

        sub_vol = ('%s%s/%s' % (settings.MNT_PT, self.meta['pool'], sname))

        snap_fp = ('%s/%s_%s' %
                   (sub_vol, self.snap_name.split('_')[0], self.snap_name))
        logger.info('snap_fp: %s' % snap_fp)
        msg = ('Snaphost: %s already exists.' % snap_fp)
        with self._clean_exit_handler(msg):
            if (os.path.isdir(snap_fp)):
                ack = {
                    'msg': 'snap_exists',
                    'id': self.meta['id'],
                }
                self.meta_push.send_json(ack)
                logger.debug(msg)

        cmd = [BTRFS, 'receive', sub_vol]
        msg = ('Failed to start the low level btrfs receive command(%s)'
               '. Aborting.' % (cmd))
        with self._clean_exit_handler(msg, ack=True):
            rp = subprocess.Popen(cmd,
                                  shell=False,
                                  stdin=subprocess.PIPE,
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE)
            logger.debug('Btrfs receive started for snap: %s' % sub_vol)

        msg = ('Failed to send begin_ok to the sender for meta: %s' %
               self.meta)
        with self._clean_exit_handler(msg):
            ack = {
                'msg': 'begin_ok',
                'id': self.meta['id'],
            }
            self.meta_push.send_json(ack)
            logger.debug('begin_ok sent for meta: %s' % self.meta)
        recv_timeout_counter = 0
        while True:
            try:
                recv_data = recv_sub.recv()
                recv_data = recv_data[len(self.meta['id']):]
                recv_timeout_counter = 0
                self.kb_received = self.kb_received + len(recv_data)
                if (self.rtid is None):
                    msg = ('Failed to create snapshot: %s. Aborting.' %
                           self.snap_name)
                    with self._clean_exit_handler(msg, ack=True):
                        create_snapshot(sname,
                                        self.snap_name,
                                        logger,
                                        snap_type='receiver')

                    data = {'snap_name': self.snap_name}
                    msg = ('Failed to create receive trail for rid: %d'
                           '. meta: %s' % (self.rid, self.meta))
                    with self._clean_exit_handler(msg, ack=True):
                        self.rtid = create_receive_trail(
                            self.rid, data, logger)

                if (recv_data == 'END_SUCCESS' or recv_data == 'END_FAIL'):
                    ts = datetime.utcnow().replace(tzinfo=utc)
                    data = {
                        'kb_received': self.kb_received / 1024,
                    }
                    if (recv_data == 'END_SUCCESS'):
                        logger.debug('END_SUCCESS received for meta: %s' %
                                     self.meta)
                        data['receive_succeeded'] = ts
                    else:
                        logger.error('END_FAIL received for meta: %s. '
                                     'Terminating.' % self.meta)
                        rp.terminate()
                        data['receive_failed'] = ts
                        data['status'] = 'failed'

                    msg = ('Failed to update receive trail for rtid: %d'
                           '. meta: %s' % (self.rtid, self.meta))
                    with self._clean_exit_handler(msg, ack=True):
                        update_receive_trail(self.rtid, data, logger)
                    break
                if (rp.poll() is None):
                    rp.stdin.write(recv_data)
                    rp.stdin.flush()
                else:
                    logger.error('It seems the btrfs receive process died'
                                 ' unexpectedly.')
                    out, err = rp.communicate()
                    logger.debug('btrfs receive out: %s err: %s' % (out, err))
                    msg = ('Low level system error from btrfs receive '
                           'command. out: %s err: %s for rtid: %s meta: %s' %
                           (out, err, self.rtid, self.meta))
                    with self._clean_exit_handler(msg, ack=True):
                        data = {
                            'receive_failed':
                            (datetime.utcnow().replace(tzinfo=utc)),
                            'status':
                            'failed',
                            'error':
                            msg,
                        }
                        update_receive_trail(self.rtid, data, logger)
            except zmq.error.Again:
                recv_timeout_counter = recv_timeout_counter + 1
                if (recv_timeout_counter > 300):
                    logger.error('Nothing received in the last 30 seconds '
                                 'from the sender for meta: %s. Aborting.' %
                                 self.meta)
                    raise
            except Exception, e:
                msg = ('Exception occured while receiving fsdata')
                logger.error(msg)
                logger.exception(e)
                rp.terminate()
                out, err = rp.communicate()
                logger.debug('rc: %d out: %s err: %s' %
                             (rp.returncode, out, err))
                data['receive_failed'] = datetime.utcnow().replace(tzinfo=utc)
                data['status'] = 'failed'
                data['error'] = msg

                msg = ('Failed to update receive trail for rtid: %d'
                       '. meta: %s' % (self.rtid, self.meta))
                with self._clean_exit_handler(msg, ack=True):
                    update_receive_trail(self.rtid, data, logger)
                self._sys_exit(3)
            finally:
示例#13
0
    def run(self):
        set_token()
        msg = ('Failed to get the sender ip from the uuid(%s) for meta: %s' %
               (self.meta['uuid'], self.meta))
        with self._clean_exit_handler(msg):
            self.sender_ip = get_sender_ip(self.meta['uuid'], logger)
        logger.debug('sender ip: %s' % self.sender_ip)

        msg = ('Failed to connect to the sender(%s) on data_port(%s). meta: '
               '%s. Aborting.' % (self.sender_ip, self.data_port, self.meta))
        with self._clean_exit_handler(msg):
            #@todo: add validation
            recv_sub = self.ctx.socket(zmq.SUB)
            recv_sub.connect('tcp://%s:%d' % (self.sender_ip, self.data_port))
            recv_sub.RCVTIMEO = 100
            recv_sub.setsockopt(zmq.SUBSCRIBE, str(self.meta['id']))

        msg = ('Failed to connect to the sender(%s) on '
               'meta_port(%d). meta: %s. Aborting.' %
               (self.sender_ip, self.meta_port, self.meta))
        with self._clean_exit_handler(msg):
            self.meta_push = self.ctx.socket(zmq.PUSH)
            url = ('tcp://%s:%d' % (self.sender_ip, self.meta_port))
            logger.debug('meta url: %s' % url)
            self.meta_push.connect('tcp://%s:%d' % (self.sender_ip,
                                                    self.meta_port))

        sname = ('%s-%s-%s' % (self.sender_id, self.sender_ip, self.src_share))
        if (not self.incremental):
            msg = ('Failed to verify/create share: %s. meta: %s. '
                   'Aborting.' % (sname, self.meta))
            with self._clean_exit_handler(msg, ack=True):
                create_share(sname, self.dest_pool, logger)

            msg = ('Failed to create the replica metadata object '
                   'for share: %s. meta: %s. Aborting.' %
                   (sname, self.meta))
            with self._clean_exit_handler(msg, ack=True):
                data = {'share': sname,
                        'appliance': self.sender_ip,
                        'src_share': self.src_share,
                        'data_port': self.data_port,
                        'meta_port': self.meta_port, }
                self.rid = create_rshare(data, logger)

        else:
            msg = ('Failed to retreive the replica metadata object for '
                   'share: %s. meta: %s. Aborting.' % (sname, self.meta))
            with self._clean_exit_handler(msg):
                self.rid = rshare_id(sname, logger)

        sub_vol = ('%s%s/%s' % (settings.MNT_PT, self.meta['pool'],
                                sname))

        snap_fp = ('%s/%s_%s' % (sub_vol, self.snap_name.split('_')[0],
                                 self.snap_name))
        logger.info('snap_fp: %s' % snap_fp)
        msg = ('Snaphost: %s already exists.' % snap_fp)
        with self._clean_exit_handler(msg):
            if (os.path.isdir(snap_fp)):
                ack = {'msg': 'snap_exists',
                       'id': self.meta['id'],}
                self.meta_push.send_json(ack)
                logger.debug(msg)

        cmd = [BTRFS, 'receive', sub_vol]
        msg = ('Failed to start the low level btrfs receive command(%s)'
               '. Aborting.' % (cmd))
        with self._clean_exit_handler(msg, ack=True):
            rp = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE,
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE)
            logger.debug('Btrfs receive started for snap: %s' % sub_vol)

        msg = ('Failed to send begin_ok to the sender for meta: %s' %
               self.meta)
        with self._clean_exit_handler(msg):
            ack = {'msg': 'begin_ok',
                   'id': self.meta['id'], }
            self.meta_push.send_json(ack)
            logger.debug('begin_ok sent for meta: %s' % self.meta)
        recv_timeout_counter = 0
        while True:
            try:
                recv_data = recv_sub.recv()
                recv_data = recv_data[len(self.meta['id']):]
                recv_timeout_counter = 0
                self.kb_received = self.kb_received + len(recv_data)
                if (self.rtid is None):
                    msg = ('Failed to create snapshot: %s. Aborting.' %
                           self.snap_name)
                    with self._clean_exit_handler(msg, ack=True):
                        create_snapshot(sname, self.snap_name, logger,
                                        snap_type='receiver')

                    data = {'snap_name': self.snap_name}
                    msg = ('Failed to create receive trail for rid: %d'
                           '. meta: %s' % (self.rid, self.meta))
                    with self._clean_exit_handler(msg, ack=True):
                        self.rtid = create_receive_trail(self.rid, data,
                                                         logger)

                if (recv_data == 'END_SUCCESS' or recv_data == 'END_FAIL'):
                    ts = datetime.utcnow().replace(tzinfo=utc)
                    data = {'kb_received': self.kb_received / 1024,}
                    if (recv_data == 'END_SUCCESS'):
                        logger.debug('END_SUCCESS received for meta: %s' %
                                     self.meta)
                        data['receive_succeeded'] = ts
                    else:
                        logger.error('END_FAIL received for meta: %s. '
                                     'Terminating.' % self.meta)
                        rp.terminate()
                        data['receive_failed'] = ts
                        data['status'] = 'failed'

                    msg = ('Failed to update receive trail for rtid: %d'
                               '. meta: %s' % (self.rtid, self.meta))
                    with self._clean_exit_handler(msg, ack=True):
                        update_receive_trail(self.rtid, data, logger)
                    break
                if (rp.poll() is None):
                    rp.stdin.write(recv_data)
                    rp.stdin.flush()
                else:
                    logger.error('It seems the btrfs receive process died'
                                 ' unexpectedly.')
                    out, err = rp.communicate()
                    logger.debug('btrfs receive out: %s err: %s' % (out, err))
                    msg = ('Low level system error from btrfs receive '
                           'command. out: %s err: %s for rtid: %s meta: %s'
                           % (out, err, self.rtid, self.meta))
                    with self._clean_exit_handler(msg, ack=True):
                        data = {'receive_failed': (
                            datetime.utcnow().replace(tzinfo=utc)),
                                'status': 'failed',
                                'error': msg,}
                        update_receive_trail(self.rtid, data, logger)
            except zmq.error.Again:
                recv_timeout_counter = recv_timeout_counter + 1
                if (recv_timeout_counter > 300):
                    logger.error('Nothing received in the last 30 seconds '
                                 'from the sender for meta: %s. Aborting.'
                                 % self.meta)
                    raise
            except Exception, e:
                msg = ('Exception occured while receiving fsdata')
                logger.error(msg)
                logger.exception(e)
                rp.terminate()
                out, err = rp.communicate()
                logger.debug('rc: %d out: %s err: %s' % (rp.returncode, out,
                                                         err))
                data['receive_failed'] = datetime.utcnow().replace(tzinfo=utc)
                data['status'] = 'failed'
                data['error'] = msg

                msg = ('Failed to update receive trail for rtid: %d'
                       '. meta: %s' % (self.rtid, self.meta))
                with self._clean_exit_handler(msg, ack=True):
                    update_receive_trail(self.rtid, data, logger)
                self._sys_exit(3)
            finally:
示例#14
0
def validate_src_share(sender_uuid, sname):
    #do a simple get on the share of the sender.
    a = Appliance.objects.get(uuid=sender_uuid)
    url = ('https://%s:%s' % (a.ip, a.mgmt_port))
    set_token(client_id=a.client_id, client_secret=a.client_secret, url=url)
    api_call(url='%s/api/shares/%s' % (url, sname))
示例#15
0
    def run(self):
        set_token()
        msg = ('Failed to connect to receiver(%s) on meta port'
               '(%d) for snap_name: %s. Aborting.' %
               (self.receiver_ip, self.rmeta_port, self.snap_name))
        with self._clean_exit_handler(msg):
            meta_push = self.ctx.socket(zmq.PUSH)
            meta_push.connect('tcp://%s:%d' % (self.receiver_ip,
                                               self.rmeta_port))

        #  1. create a new replica trail if it's the very first time
        # of if the last one succeeded
        msg = ('Failed to create local replica trail for snap_name:'
               ' %s. Aborting.' % self.snap_name)
        with self._clean_exit_handler(msg):
            self.rt2 = create_replica_trail(self.replica.id,
                                            self.snap_name, logger)
            self.rt2_id = self.rt2['id']

        #  2. create a snapshot only if it's not already from a previous
        #  failed attempt.
        if (not is_snapshot(self.replica.share, self.snap_name, logger)):
            msg = ('Failed to create snapshot: %s. Aborting.' % self.snap_name)
            with self._clean_exit_handler(msg):
                create_snapshot(self.replica.share, self.snap_name, logger)

        #  let the receiver know that following diff is coming
        msg = ('Failed to send initial metadata communication to the '
               'receiver(%s), most likely due to a network error. Aborting.'
               % self.receiver_ip)
        with self._update_trail_and_quit(msg):
            logger.debug('sending meta_begin: %s' % self.meta_begin)
            meta_push.send_json(self.meta_begin)
            logger.debug('meta_begin sent. waiting on get')

        msg = ('Timeout occured(60 seconds) while waiting for OK '
               'from the receiver(%s) to start sending data. Aborting.'
               % self.receiver_ip)
        with self._update_trail_and_quit(msg):
            ack = self._process_q()
            logger.info('suman ack = %s' % ack)
            if (ack['msg'] == 'snap_exists'):
                data = {'status': 'succeeded',
                        'end_ts': datetime.utcnow().replace(tzinfo=utc),
                        'error': 'snapshot already exists on the receiver', }
                msg = ('Failed to update replica status for snap_name: %s. '
                       'Aborting.' % self.snap_name)
                with self._clean_exit_handler(msg):
                    update_replica_status(self.rt2_id, data, logger)
                    self._sys_exit(0)

        snap_path = ('%s%s/%s_%s' % (settings.MNT_PT, self.replica.pool,
                                     self.replica.share, self.snap_name))
        logger.debug('current snap: %s' % snap_path)
        cmd = [BTRFS, 'send', snap_path]
        if (self.rt is not None):
            prev_snap = ('%s%s/%s_%s' % (settings.MNT_PT, self.replica.pool,
                                         self.replica.share,
                                         self.rt.snap_name))
            logger.info('Sending incremental replica between %s -- %s' %
                        (prev_snap, snap_path))
            cmd = [BTRFS, 'send', '-p', prev_snap, snap_path]
        else:
            logger.info('Sending full replica: %s' % snap_path)

        try:
            sp = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE)
            fcntl.fcntl(sp.stdout.fileno(), fcntl.F_SETFL, os.O_NONBLOCK)
            logger.debug('send started. snap: %s' % snap_path)
        except Exception, e:
            msg = ('Failed to start the low level btrfs send '
                   'command(%s). Aborting' % cmd)
            logger.error(msg)
            logger.exception(e)
            with self._update_trail_and_quit(msg):
                self.pub.put('%sEND_FAIL' % self.snap_id)
            self._sys_exit(3)
示例#16
0
    def run(self):
        set_token()
        msg = ('Failed to connect to receiver(%s) on meta port'
               '(%d) for snap_name: %s. Aborting.' %
               (self.receiver_ip, self.rmeta_port, self.snap_name))
        with self._clean_exit_handler(msg):
            meta_push = self.ctx.socket(zmq.PUSH)
            meta_push.connect('tcp://%s:%d' %
                              (self.receiver_ip, self.rmeta_port))

        #  1. create a new replica trail if it's the very first time
        # of if the last one succeeded
        msg = ('Failed to create local replica trail for snap_name:'
               ' %s. Aborting.' % self.snap_name)
        with self._clean_exit_handler(msg):
            self.rt2 = create_replica_trail(self.replica.id, self.snap_name,
                                            logger)
            self.rt2_id = self.rt2['id']

        #  2. create a snapshot only if it's not already from a previous
        #  failed attempt.
        if (not is_snapshot(self.replica.share, self.snap_name, logger)):
            msg = ('Failed to create snapshot: %s. Aborting.' % self.snap_name)
            with self._clean_exit_handler(msg):
                create_snapshot(self.replica.share, self.snap_name, logger)

        #  let the receiver know that following diff is coming
        msg = ('Failed to send initial metadata communication to the '
               'receiver(%s), most likely due to a network error. Aborting.' %
               self.receiver_ip)
        with self._update_trail_and_quit(msg):
            logger.debug('sending meta_begin: %s' % self.meta_begin)
            meta_push.send_json(self.meta_begin)
            logger.debug('meta_begin sent. waiting on get')

        msg = ('Timeout occured(60 seconds) while waiting for OK '
               'from the receiver(%s) to start sending data. Aborting.' %
               self.receiver_ip)
        with self._update_trail_and_quit(msg):
            ack = self._process_q()
            logger.info('suman ack = %s' % ack)
            if (ack['msg'] == 'snap_exists'):
                data = {
                    'status': 'succeeded',
                    'end_ts': datetime.utcnow().replace(tzinfo=utc),
                    'error': 'snapshot already exists on the receiver',
                }
                msg = ('Failed to update replica status for snap_name: %s. '
                       'Aborting.' % self.snap_name)
                with self._clean_exit_handler(msg):
                    update_replica_status(self.rt2_id, data, logger)
                    self._sys_exit(0)

        snap_path = ('%s%s/%s_%s' % (settings.MNT_PT, self.replica.pool,
                                     self.replica.share, self.snap_name))
        logger.debug('current snap: %s' % snap_path)
        cmd = [BTRFS, 'send', snap_path]
        if (self.rt is not None):
            prev_snap = ('%s%s/%s_%s' %
                         (settings.MNT_PT, self.replica.pool,
                          self.replica.share, self.rt.snap_name))
            logger.info('Sending incremental replica between %s -- %s' %
                        (prev_snap, snap_path))
            cmd = [BTRFS, 'send', '-p', prev_snap, snap_path]
        else:
            logger.info('Sending full replica: %s' % snap_path)

        try:
            sp = subprocess.Popen(cmd,
                                  shell=False,
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE)
            fcntl.fcntl(sp.stdout.fileno(), fcntl.F_SETFL, os.O_NONBLOCK)
            logger.debug('send started. snap: %s' % snap_path)
        except Exception, e:
            msg = ('Failed to start the low level btrfs send '
                   'command(%s). Aborting' % cmd)
            logger.error(msg)
            logger.exception(e)
            with self._update_trail_and_quit(msg):
                self.pub.put('%sEND_FAIL' % self.snap_id)
            self._sys_exit(3)