def download_file(self, mongo_id: str) -> None: """ Downloads files and store them to {config.storage_vol} Tracking: {db.record.is_downloaded} indicates success NOTE/TODO: is run as a task. """ record = read_record(mongo_id) # If pipeline is rerun after an error if record.is_downloaded: log.debug("Data file already downloaded. Skipping.") return is_downloaded_success = dreem_api.download_file( self.session, record.download_folder(), record.manufacturer_ref) if is_downloaded_success: # Useful metadata for performing pre-processing. downloaded_file = Path(record.download_folder() / f"{record.manufacturer_ref}.h5") record.meta["filesize"] = downloaded_file.stat().st_size record.is_downloaded = is_downloaded_success update_record(record) log.debug(f"Download SUCCESS for:\n {record}") else: log.debug(f"Download FAILED for:\n {record}")
def task_preprocess_data(mongoid: str) -> str: """ Preprocessing tasks on dreem data. """ record = read_record(mongoid) if record.is_downloaded and not record.is_processed: record.is_processed = True update_record(record) return mongoid
def task_preprocess_data(mongoid: str) -> str: """ Preprocessing tasks on thinkFAST and CANTAB data. """ record = read_record(mongoid) if not record.is_processed: record.is_processed = True update_record(record) return mongoid
def download_file(self, mongo_id: str) -> None: """ Downloads files and store them to {config.storage_vol} Tracking: {db.record.is_downloaded} indicates success NOTE/TODO: is run as a task. """ record = read_record(mongo_id) is_downloaded_success = vttsma_api.download_files( self.bucket, record.filename, record.vttsma_export_date) if is_downloaded_success: record.is_downloaded = is_downloaded_success update_record(record)