def _get_and_process_entries(self, marker, shard_num): """ sync up to self.max_entries entries, returning number of entries processed and the last marker of the entries processed. """ log_entries = client.get_meta_log(self.source_conn, shard_num, marker, self.max_entries) log.info('shard %d has %d entries after %r', shard_num, len(log_entries), marker) try: entries = [_meta_entry_from_json(entry) for entry in log_entries] except KeyError: log.error('log conting bad key is: %s', log_entries) raise mentioned = set([(entry.section, entry.name) for entry in entries]) for section, name in mentioned: self.sync_meta(section, name) if entries: try: client.set_worker_bound(self.source_conn, 'metadata', shard_num, entries[-1].marker, entries[-1].timestamp, self.daemon_id) return len(entries), entries[-1].marker except: log.exception('error setting worker bound, may duplicate some work later') return 0, ''
def set_bound(self, key, marker, retries, type_=None): # api doesn't allow setting a bound with a blank marker if marker != " ": if type_ is None: type_ = self.type try: data = [obj_.to_dict(item, time=DEFAULT_TIME) for item in retries] client.set_worker_bound(self.dest_conn, type_, marker, DEFAULT_TIME, self.daemon_id, key, data=data) return RESULT_SUCCESS except Exception: log.warn( 'error setting worker bound for key "%s",' " may duplicate some work later. Traceback:", key, exc_info=True, ) return RESULT_ERROR
def complete_item(self, shard_num, retries): """Called when syncing a single item completes successfully""" marker = self.shard_info.get(shard_num) if not marker: return try: data = [ dict(name=retry, time=worker.DEFAULT_TIME) for retry in retries ] client.set_worker_bound(self.dest_conn, self.type, marker, worker.DEFAULT_TIME, self.daemon_id, shard_num, data) except Exception: log.warn( 'could not set worker bounds, may repeat some work.' 'Traceback:', exc_info=True)
def complete_item(self, shard_num, retries): """Called when syncing a single item completes successfully""" marker = self.shard_info.get(shard_num) if not marker: return try: data = [dict(name=retry, time=worker.DEFAULT_TIME) for retry in retries] client.set_worker_bound(self.dest_conn, self.type, marker, worker.DEFAULT_TIME, self.daemon_id, shard_num, data) except Exception: log.warn('could not set worker bounds, may repeat some work.' 'Traceback:', exc_info=True)
def _get_and_process_entries(self, marker, shard_num): """ sync up to self.max_entries entries, returning number of entries processed and the last marker of the entries processed. """ log_entries = client.get_meta_log(self.source_conn, shard_num, marker, self.max_entries) log.info('shard %d has %d entries after %r', shard_num, len(log_entries), marker) try: entries = [_meta_entry_from_json(entry) for entry in log_entries] except KeyError: log.error('log conting bad key is: %s', log_entries) raise error_encountered = False mentioned = set([(entry.section, entry.name) for entry in entries]) for section, name in mentioned: sync_result = self.sync_meta(section, name) if sync_result == RESULT_ERROR: error_encountered = True # Only set worker bounds if there was data synced and no # errors were encountered if entries and not error_encountered: try: client.set_worker_bound(self.dest_conn, 'metadata', shard_num, entries[-1].marker, entries[-1].timestamp, self.daemon_id) return len(entries), entries[-1].marker except: log.exception('error setting worker bound for shard {shard_num},' ' may duplicate some work later'.format(shard_num=shard_num)) elif entries and error_encountered: log.error('Error encountered while syncing shard {shard_num}.' 'Not setting worker bound, may duplicate some work later'.format(shard_num=shard_num)) return 0, ''
def set_bound(self, key, marker, retries, type_=None): # api doesn't allow setting a bound with a blank marker if marker: if type_ is None: type_ = self.type try: data = [ obj_.to_dict(item, time=DEFAULT_TIME) for item in retries ] client.set_worker_bound(self.dest_conn, type_, marker, DEFAULT_TIME, self.daemon_id, key, data=data) return RESULT_SUCCESS except Exception: log.warn('error setting worker bound for key "%s",' ' may duplicate some work later. Traceback:', key, exc_info=True) return RESULT_ERROR
def sync_full(self, num_workers, log_lock_time): try: sections = client.get_metadata_sections(self.src_conn) except client.HttpError as e: log.error('Error listing metadata sections: %s', e) raise # grab the lastest shard markers and timestamps before we sync shard_info = [] num_shards = client.num_log_shards(self.src_conn, 'metadata') for shard_num in xrange(num_shards): info = client.get_log_info(self.src_conn, 'metadata', shard_num) # setting an empty marker returns an error if info['marker']: shard_info.append((shard_num, info['marker'], info['last_update'])) meta_keys = [] for section in sections: try: meta_keys += [(section, key) for key in client.list_metadata_keys(self.src_conn, section)] except client.NotFound: # no keys of this type exist continue except client.HttpError as e: log.error('Error listing metadata for section %s: %s', section, e) raise # create the work and results Queue workQueue = multiprocessing.Queue() resultQueue = multiprocessing.Queue() # create the worker processes if self._type == 'data': worker_cls = worker.DataWorkerFull else: worker_cls = worker.MetadataWorkerFull processes = [worker_cls(workQueue, resultQueue, log_lock_time, self.src, self.dest) for i in xrange(num_workers)] for process in processes: process.daemon = True process.start() start_time = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%SZ") log.info('Starting full sync at %s', start_time) # enqueue the shards to be synced for meta in meta_keys: workQueue.put(meta) # add a poison pill for each worker for i in xrange(num_workers): workQueue.put(None) # pull the results out as they are produced errors = [] for i in xrange(len(meta_keys)): log.info('%d/%d items synced', i, len(meta_keys)) result, section, name = resultQueue.get() if result != worker.RESULT_SUCCESS: log.error('error syncing %s %r', section, name) errors.append((section, name)) else: log.debug('synced %s %r', section, name) for process in processes: process.join() if errors: log.error('Encountered errors syncing these %d entries: %s', len(errors), errors) else: for shard_num, marker, timestamp in shard_info: client.set_worker_bound(self.src_conn, 'metadata', shard_num, marker, timestamp, self.daemon_id) client.del_worker_bound(self.src_conn, 'metadata', shard_num, self.daemon_id)