class SnowpipeApi:
def __init__(self, config: Config, pipe: str) -> None:
self.logger = logging.getLogger(self.__class__.__name__)
self.config = config
self.ingest_manager = SimpleIngestManager(
account=self.config.account,
host=self.config.url,
user=self.config.user,
private_key=self.config.private_key_pem,
pipe=pipe,
)
def report(self, recent_seconds: Optional[int] = None) -> None:
request_id = uuid.uuid4()
self.logger.debug(f'request_id: {request_id}')
body = self.ingest_manager.get_history(recent_seconds=recent_seconds,
request_id=request_id)
print(json.dumps(body, indent=4))
def history(self,
start_time: datetime,
end_time: Optional[datetime] = None) -> None:
if not start_time:
raise ValueError('start_time must be defined')
request_id = uuid.uuid4()
self.logger.debug(f'request_id: {request_id}')
body = self.ingest_manager.get_history_range(
start_time_inclusive=start_time.isoformat(),
end_time_exclusive=end_time.isoformat() if end_time else None,
request_id=request_id)
print(json.dumps(body, indent=4))
def ingest(self, files: Iterable[str]) -> None:
if not files:
raise ValueError('files must be defined')
request_id = uuid.uuid4()
self.logger.debug(f'request_id: {request_id}')
staged_files = [StagedFile(name, None) for name in files]
body = self.ingest_manager.ingest_files(
staged_files=staged_files,
request_id=request_id,
)
print(json.dumps(body, indent=4))
try:
resp = ingest_manager.ingest_files(staged_file_list)
print(resp)
except HTTPError as e:
# HTTP error, may need to retry
logger.error(e)
exit(1)
# This means Snowflake has received file and will start loading
assert (resp['responseCode'] == 'SUCCESS')
# Needs to wait for a while to get result in history
while True:
history_resp = ingest_manager.get_history()
if len(history_resp['files']) > 0:
print('Ingest Report:\n')
print(history_resp)
break
else:
# wait for 20 seconds
time.sleep(20)
hour = timedelta(hours=1)
date = datetime.datetime.utcnow() - hour
history_range_resp = ingest_manager.get_history_range(date.isoformat() +
'Z')
print('\nHistory scan report: \n')
print(history_range_resp)