def run_dropbox_retrieval(self):
if self.err_found:
return False
# Reset log attributes and mark the retrieval start time
#
self.dbox_log.set_retrieval_start_time(with_reset=True)
self.dbox_log.save()
# Update PredictDataset status to 'file retrieval started'
#
self.dbox_log.dataset.set_status_file_retrieval_started()
dr = DropboxRetriever(
self.predict_dataset.dropbox_url,
self.predict_dataset.file_directory,
file_patterns=self.predict_dataset.get_file_patterns(),
)
if dr.err_found:
self.record_retrieval_error(dr.err_msg)
return False
# Get the metadata
#
if not dr.step1_retrieve_metadata():
self.record_retrieval_error(dr.err_msg)
return False
# Does it have what we want?
#
if not dr.step2_check_file_matches():
self.record_retrieval_error(dr.err_msg)
return False
# Download the files
#
if not dr.step3_retrieve_files():
self.record_retrieval_error(dr.err_msg)
return False
# ----------------------
# Success!
# ----------------------
self.predict_dataset.set_status_file_retrieval_complete()
self.dbox_log.selected_files = dr.final_file_paths
self.dbox_log.set_retrieval_end_time(files_retrieved=True)
self.dbox_log.save()
def run_dropbox_retrieval(self):
if self.err_found:
return False
# Reset log attributes and mark the retrieval start time
#
self.dbox_log.set_retrieval_start_time(with_reset=True)
self.dbox_log.save()
# Update PredictDataset status to 'file retrieval started'
#
self.dbox_log.dataset.set_status_file_retrieval_started()
dr = DropboxRetriever(
self.predict_dataset.dropbox_url,
self.predict_dataset.file_directory,
file_patterns=self.predict_dataset.get_file_patterns())
if dr.err_found:
self.record_retrieval_error(dr.err_msg)
return False
# Get the metadata
#
if not dr.step1_retrieve_metadata():
self.record_retrieval_error(dr.err_msg)
return False
# Does it have what we want?
#
if not dr.step2_check_file_matches():
self.record_retrieval_error(dr.err_msg)
return False
# Download the files
#
if not dr.step3_retrieve_files():
self.record_retrieval_error(dr.err_msg)
return False
# ----------------------
# Success!
# ----------------------
self.predict_dataset.set_status_file_retrieval_complete()
self.dbox_log.selected_files = dr.final_file_paths
self.dbox_log.set_retrieval_end_time(files_retrieved=True)
self.dbox_log.save()
def get_dropbox_metadata(predict_dataset):
"""
Wrap the DropboxRetriever function
- (True, DropboxRetrievalLog object)
- (False, error message string)
"""
if predict_dataset is None:
return (False, "The dataset was not found.")
# Initialize
#
dr = DropboxRetriever(predict_dataset.dropbox_url,
destination_dir=predict_dataset.file_directory,
file_patterns=self.predict_dataset.get_file_patterns())
db_log = DropboxRetrievalLog(dataset=predict_dataset)
if dr.err_found:
db_log.file_metadata_err_msg = dr.err_msg
db_log.save()
return (False, dr.err_msg)
# Get the metadata
#
if not dr.step1_retrieve_metadata():
db_log.file_metadata_err_msg = dr.err_msg
db_log.save()
return (False, dr.err_msg)
# Does it have what we want?
#
if not dr.step2_check_file_matches():
db_log.file_metadata_err_msg = dr.err_msg
db_log.save()
return (False, dr.err_msg)
# Yes!
db_log.file_metadata = dr.dropbox_link_metadata
db_log.selected_files = dr.matching_files_metadata
db_log.save()
return (True, dr)
def get_dropbox_metadata(predict_dataset):
"""
Wrap the DropboxRetriever function
- (True, DropboxRetrievalLog object)
- (False, error message string)
"""
if predict_dataset is None:
return (False, "The dataset was not found.")
# Initialize
#
dr = DropboxRetriever(
predict_dataset.dropbox_url,
destination_dir=predict_dataset.file_directory,
file_patterns=self.predict_dataset.get_file_patterns())
db_log = DropboxRetrievalLog(dataset=predict_dataset)
if dr.err_found:
db_log.file_metadata_err_msg = dr.err_msg
db_log.save()
return (False, dr.err_msg)
# Get the metadata
#
if not dr.step1_retrieve_metadata():
db_log.file_metadata_err_msg = dr.err_msg
db_log.save()
return (False, dr.err_msg)
# Does it have what we want?
#
if not dr.step2_check_file_matches():
db_log.file_metadata_err_msg = dr.err_msg
db_log.save()
return (False, dr.err_msg)
# Yes!
db_log.file_metadata = dr.dropbox_link_metadata
db_log.selected_files = dr.matching_files_metadata
db_log.save()
return (True, dr)
def get_dropbox_metadata_from_link(dropbox_link, file_patterns=None):
"""
Wrap the DropboxRetriever function
- (True, DropboxRetriever object)
- (False, error message string)
"""
if dropbox_link is None:
return (False, "The dataset was not found.")
# This directory doesn't actually get used
#
tmp_dir = join(settings.TB_SHARED_DATAFILE_DIRECTORY, 'tmp')
if not isdir(tmp_dir):
os.makedirs(tmp_dir)
# Initialize
#
if file_patterns:
dr = DropboxRetriever(dropbox_link,\
destination_dir=tmp_dir,\
file_patterns=file_patterns)
else:
dr = DropboxRetriever(dropbox_link,\
destination_dir=tmp_dir)
if dr.err_found:
return (False, dr.err_msg)
# Get the metadata
#
if not dr.step1_retrieve_metadata():
return (False, dr.err_msg)
# Does it have what we want?
#
if not dr.step2_check_file_matches():
return (False, dr.err_msg)
# Yes!
return (True, dr)
def get_dropbox_metadata_from_link(dropbox_link, file_patterns=None):
"""
Wrap the DropboxRetriever function
- (True, DropboxRetriever object)
- (False, error message string)
"""
if dropbox_link is None:
return (False, "The dataset was not found.")
# This directory doesn't actually get used
#
tmp_dir = join(settings.TB_SHARED_DATAFILE_DIRECTORY, 'tmp')
if not isdir(tmp_dir):
os.makedirs(tmp_dir)
# Initialize
#
if file_patterns:
dr = DropboxRetriever(dropbox_link,\
destination_dir=tmp_dir,\
file_patterns=file_patterns)
else:
dr = DropboxRetriever(dropbox_link,\
destination_dir=tmp_dir)
if dr.err_found:
return (False, dr.err_msg)
# Get the metadata
#
if not dr.step1_retrieve_metadata():
return (False, dr.err_msg)
# Does it have what we want?
#
if not dr.step2_check_file_matches():
return (False, dr.err_msg)
# Yes!
return (True, dr)
