def run_dropbox_retrieval(self): if self.err_found: return False # Reset log attributes and mark the retrieval start time # self.dbox_log.set_retrieval_start_time(with_reset=True) self.dbox_log.save() # Update PredictDataset status to 'file retrieval started' # self.dbox_log.dataset.set_status_file_retrieval_started() dr = DropboxRetriever( self.predict_dataset.dropbox_url, self.predict_dataset.file_directory, file_patterns=self.predict_dataset.get_file_patterns(), ) if dr.err_found: self.record_retrieval_error(dr.err_msg) return False # Get the metadata # if not dr.step1_retrieve_metadata(): self.record_retrieval_error(dr.err_msg) return False # Does it have what we want? # if not dr.step2_check_file_matches(): self.record_retrieval_error(dr.err_msg) return False # Download the files # if not dr.step3_retrieve_files(): self.record_retrieval_error(dr.err_msg) return False # ---------------------- # Success! # ---------------------- self.predict_dataset.set_status_file_retrieval_complete() self.dbox_log.selected_files = dr.final_file_paths self.dbox_log.set_retrieval_end_time(files_retrieved=True) self.dbox_log.save()
def run_dropbox_retrieval(self): if self.err_found: return False # Reset log attributes and mark the retrieval start time # self.dbox_log.set_retrieval_start_time(with_reset=True) self.dbox_log.save() # Update PredictDataset status to 'file retrieval started' # self.dbox_log.dataset.set_status_file_retrieval_started() dr = DropboxRetriever( self.predict_dataset.dropbox_url, self.predict_dataset.file_directory, file_patterns=self.predict_dataset.get_file_patterns()) if dr.err_found: self.record_retrieval_error(dr.err_msg) return False # Get the metadata # if not dr.step1_retrieve_metadata(): self.record_retrieval_error(dr.err_msg) return False # Does it have what we want? # if not dr.step2_check_file_matches(): self.record_retrieval_error(dr.err_msg) return False # Download the files # if not dr.step3_retrieve_files(): self.record_retrieval_error(dr.err_msg) return False # ---------------------- # Success! # ---------------------- self.predict_dataset.set_status_file_retrieval_complete() self.dbox_log.selected_files = dr.final_file_paths self.dbox_log.set_retrieval_end_time(files_retrieved=True) self.dbox_log.save()
def get_dropbox_metadata(predict_dataset): """ Wrap the DropboxRetriever function - (True, DropboxRetrievalLog object) - (False, error message string) """ if predict_dataset is None: return (False, "The dataset was not found.") # Initialize # dr = DropboxRetriever(predict_dataset.dropbox_url, destination_dir=predict_dataset.file_directory, file_patterns=self.predict_dataset.get_file_patterns()) db_log = DropboxRetrievalLog(dataset=predict_dataset) if dr.err_found: db_log.file_metadata_err_msg = dr.err_msg db_log.save() return (False, dr.err_msg) # Get the metadata # if not dr.step1_retrieve_metadata(): db_log.file_metadata_err_msg = dr.err_msg db_log.save() return (False, dr.err_msg) # Does it have what we want? # if not dr.step2_check_file_matches(): db_log.file_metadata_err_msg = dr.err_msg db_log.save() return (False, dr.err_msg) # Yes! db_log.file_metadata = dr.dropbox_link_metadata db_log.selected_files = dr.matching_files_metadata db_log.save() return (True, dr)
def get_dropbox_metadata(predict_dataset): """ Wrap the DropboxRetriever function - (True, DropboxRetrievalLog object) - (False, error message string) """ if predict_dataset is None: return (False, "The dataset was not found.") # Initialize # dr = DropboxRetriever( predict_dataset.dropbox_url, destination_dir=predict_dataset.file_directory, file_patterns=self.predict_dataset.get_file_patterns()) db_log = DropboxRetrievalLog(dataset=predict_dataset) if dr.err_found: db_log.file_metadata_err_msg = dr.err_msg db_log.save() return (False, dr.err_msg) # Get the metadata # if not dr.step1_retrieve_metadata(): db_log.file_metadata_err_msg = dr.err_msg db_log.save() return (False, dr.err_msg) # Does it have what we want? # if not dr.step2_check_file_matches(): db_log.file_metadata_err_msg = dr.err_msg db_log.save() return (False, dr.err_msg) # Yes! db_log.file_metadata = dr.dropbox_link_metadata db_log.selected_files = dr.matching_files_metadata db_log.save() return (True, dr)
def get_dropbox_metadata_from_link(dropbox_link, file_patterns=None): """ Wrap the DropboxRetriever function - (True, DropboxRetriever object) - (False, error message string) """ if dropbox_link is None: return (False, "The dataset was not found.") # This directory doesn't actually get used # tmp_dir = join(settings.TB_SHARED_DATAFILE_DIRECTORY, 'tmp') if not isdir(tmp_dir): os.makedirs(tmp_dir) # Initialize # if file_patterns: dr = DropboxRetriever(dropbox_link,\ destination_dir=tmp_dir,\ file_patterns=file_patterns) else: dr = DropboxRetriever(dropbox_link,\ destination_dir=tmp_dir) if dr.err_found: return (False, dr.err_msg) # Get the metadata # if not dr.step1_retrieve_metadata(): return (False, dr.err_msg) # Does it have what we want? # if not dr.step2_check_file_matches(): return (False, dr.err_msg) # Yes! return (True, dr)