def test_iterate_tarfile(alert_tarball):
iterator = iterate_tarfile(alert_tarball)
i = 0
for a in iterator:
i += 1
assert i == 2567
def read_alerts(filepath: str, q: queue.Queue, read_max:int=-1) -> None:
iterator = iterate_tarfile(filepath)
n_read = 0
for alert in iterator:
q.put(alert)
logging.debug(f"putting into q: {alert.candidate_id}")
n_read += 1
if read_max > 0 and n_read >= read_max:
break
logging.debug(f"file queue complete, exiting")
q.close()
async def read_alerts(filepath: str,
q: asyncio.Queue,
waiter: asyncio.Event,
read_max: int = -1) -> None:
iterator = iterate_tarfile(filepath)
n_read = 0
for alert in iterator:
await q.put(alert)
n_read += 1
if read_max > 0 and n_read >= read_max:
break
waiter.set()
async def tarfile_to_queue() -> None:
n = 0
for alert in iterate_tarfile(tarfile_path):
logger.info("scanned alert %s", alert.candidate_id)
if skip_existing:
if self.index.get_url(alert.candidate_id) is not None:
logger.info("alert is already stored, skipping it")
continue
n += 1
if limit is not None and n > limit:
logger.info("tarfile limit reached")
break
await upload_queue.put(alert)
await asyncio.sleep(0) # Yield to the scheduler.
tarfile_read_done.set()
logger.info("done processing tarfile")
async def upload_and_download():
filepath = "testdata/alertfiles/ztf_public_20210120.tar.gz"
bucket = "ztf-alert-archive-prototyping-tmp"
bs = Blobstore(bucket)
iterator = iterate_tarfile(filepath)
tasks = []
for alert in iterator:
print(f"file read: {alert.candidate_id}")
tasks.append(bs.upload_alert_async(alert))
urls = await asyncio.gather(*tasks)
tasks = []
for url in urls:
print(f"uploaded to: {url}")
tasks.append(bs.download_alert_async(url))
downloads = await asyncio.gather(*tasks)
for download in downloads:
print(f"downloaded: {download.candidate_id}")