def sync_from_rhic_serve_blocking(): _LOG.info("Attempting to synchronize RHIC data from configured rhic_serve") current_time = datetime.now(tzutc()) cfg = config.get_rhic_serve_config_info() # Lookup last time we synced from this host # Sync records from that point in time. # If we haven't synced before we get back None and proceed with a full sync server_hostname = cfg["host"] last_sync = get_last_sync_timestamp(server_hostname) current_offset=0 current_limit=cfg["sync_all_rhics_pagination_limit_per_call"] sync_loop = True while sync_loop: data, meta = rhic_serve_client.get_all_rhics(host=server_hostname, port=cfg["port"], url=cfg["rhics_url"], last_sync=last_sync, offset=current_offset, limit=current_limit) if not data: _LOG.info("Received no data from %s:%s%s" % (cfg["host"], cfg["port"], cfg["rhics_url"])) return True _LOG.info("Fetched %s RHICs from %s:%s%s with last_sync=%s, offset=%s, limit=%s" % (len(data), cfg["host"], cfg["port"], cfg["rhics_url"], last_sync, current_offset, current_limit)) syncd_uuids = process_data(data) current_offset = current_offset + len(syncd_uuids) if current_offset >= meta["total_count"]: break if not save_last_sync(server_hostname, current_time): _LOG.info("Unable to update last sync for: %s at %s" % (server_hostname, current_time)) return False return True
def test_get_all_rhics(self): rhics, meta = rhic_serve_client.get_all_rhics(host="localhost", port=0, url="mocked") self.assertEquals(len(rhics), 3)
def fetch(hostname, num_rhics, gzip_support=True, offset=0, port=443, url="/splice/api/v1/rhicrcs/"): data, meta = rhic_serve_client.get_all_rhics(host=hostname, port=port, url=url, offset=offset, limit=num_rhics, debug=False, accept_gzip=gzip_support) return data, meta