def download_file(self, mongo_id: str) -> None:
"""
Downloads files and store them to {config.storage_vol}
Tracking: {db.record.is_downloaded} indicates success
NOTE/TODO: is run as a task.
"""
record = read_record(mongo_id)
# If pipeline is rerun after an error
if record.is_downloaded:
log.debug("Data file already downloaded. Skipping.")
return
is_downloaded_success = dreem_api.download_file(
self.session, record.download_folder(), record.manufacturer_ref)
if is_downloaded_success:
# Useful metadata for performing pre-processing.
downloaded_file = Path(record.download_folder() /
f"{record.manufacturer_ref}.h5")
record.meta["filesize"] = downloaded_file.stat().st_size
record.is_downloaded = is_downloaded_success
update_record(record)
log.debug(f"Download SUCCESS for:\n {record}")
else:
log.debug(f"Download FAILED for:\n {record}")
def task_preprocess_data(mongoid: str) -> str:
"""
Preprocessing tasks on dreem data.
"""
record = read_record(mongoid)
if record.is_downloaded and not record.is_processed:
record.is_processed = True
update_record(record)
return mongoid
def task_preprocess_data(mongoid: str) -> str:
"""
Preprocessing tasks on thinkFAST and CANTAB data.
"""
record = read_record(mongoid)
if not record.is_processed:
record.is_processed = True
update_record(record)
return mongoid
def download_file(self, mongo_id: str) -> None:
"""
Downloads files and store them to {config.storage_vol}
Tracking: {db.record.is_downloaded} indicates success
NOTE/TODO: is run as a task.
"""
record = read_record(mongo_id)
is_downloaded_success = vttsma_api.download_files(
self.bucket, record.filename, record.vttsma_export_date)
if is_downloaded_success:
record.is_downloaded = is_downloaded_success
update_record(record)