def sync_from_rhic_serve_blocking():
_LOG.info("Attempting to synchronize RHIC data from configured rhic_serve")
current_time = datetime.now(tzutc())
cfg = config.get_rhic_serve_config_info()
# Lookup last time we synced from this host
# Sync records from that point in time.
# If we haven't synced before we get back None and proceed with a full sync
server_hostname = cfg["host"]
last_sync = get_last_sync_timestamp(server_hostname)
current_offset=0
current_limit=cfg["sync_all_rhics_pagination_limit_per_call"]
sync_loop = True
while sync_loop:
data, meta = rhic_serve_client.get_all_rhics(host=server_hostname, port=cfg["port"],
url=cfg["rhics_url"], last_sync=last_sync, offset=current_offset, limit=current_limit)
if not data:
_LOG.info("Received no data from %s:%s%s" % (cfg["host"], cfg["port"], cfg["rhics_url"]))
return True
_LOG.info("Fetched %s RHICs from %s:%s%s with last_sync=%s, offset=%s, limit=%s" % (len(data),
cfg["host"], cfg["port"], cfg["rhics_url"], last_sync, current_offset, current_limit))
syncd_uuids = process_data(data)
current_offset = current_offset + len(syncd_uuids)
if current_offset >= meta["total_count"]:
break
if not save_last_sync(server_hostname, current_time):
_LOG.info("Unable to update last sync for: %s at %s" % (server_hostname, current_time))
return False
return True
def test_get_all_rhics(self):
rhics, meta = rhic_serve_client.get_all_rhics(host="localhost", port=0, url="mocked")
self.assertEquals(len(rhics), 3)
def fetch(hostname, num_rhics, gzip_support=True, offset=0, port=443, url="/splice/api/v1/rhicrcs/"):
data, meta = rhic_serve_client.get_all_rhics(host=hostname, port=port, url=url,
offset=offset, limit=num_rhics, debug=False, accept_gzip=gzip_support)
return data, meta