def download_data(group): service = gen.authenticate_drive_api() kml_file_id = gen.get_tableId_by_table_name("SSURGO_" + str(group) + ".kml", service, google_service='drive') bytes_data = download_file(service, group, kml_file_id) out_file = write_download_to_file(bytes_data, group) delete_files_on_drive(service, group, kml_file_id) return out_file
def delete_temp_tables(): for file_str in os.listdir(dirs.intermediate_directory): if 'SSURGO_' in file_str: os.remove(dirs.intermediate_directory + "\\" + file_str) service = gen.authenticate_drive_api() ids = gen.get_tableId_by_table_name('SSURGO_', service, exact=False, google_service='drive') for file_id in ids: service.files().trash(fileId=file_id).execute() service.files().emptyTrash()
def parralelize_points(feature_list, out_file_name, simultaneous_downloads, group_table_ID, min_date, max_date, start_index, end_index): class EE_manual_error(Exception): def __init__(self, value): self.value = value def __str__(self): return repr(self.value) def wait_on_task_list(task_list, run, simultaneous_downloads, allowed_failures, failed_groups): def wait_on_task(task, failed_groups, allowed_failures, fl_index): failure_counter = 0 while(True): try: status = task.status()['state'] except urllib2.HTTPError as err: failure_counter = exponential_backoff(failure_counter) if(failure_counter == allowed_failures): failed_groups, failure_counter = log_error(failed_groups, allowed_failures, failure_counter, fl_index) if status in ['READY', 'RUNNING']: time.sleep(3 + np.random.uniform()) else: break def failed_task(task): if(task.status()['state'] == 'FAILED'): raise EE_manual_error(task.status()['error_message']) n_tasks = len(task_list) fl_index = run*simultaneous_downloads while(n_tasks != 0): task = task_list[0] wait_on_task(task, failed_groups, allowed_failures, fl_index) try: failed_task(task) except: failed_groups, counter = log_error(failed_groups, 10, 0, fl_index) if(n_tasks>1): task_list = task_list[1:] n_tasks = len(task_list) fl_index = fl_index+1 else: break def get_runs(n_features, simultaneous_downloads): runs = int(math.floor(n_features / simultaneous_downloads) + 1) return runs def get_feature_list_index(run, simultaneous_downloads, dl): fl_index = run*simultaneous_downloads + dl return fl_index def start_task(download_index, run, simultaneous_downloads, task_list, out_drive_names, feature_list, n_features, group_table_ID, allowed_failures, failed_groups): failure_counter = 0 while(failure_counter < allowed_failures): fl_index = get_feature_list_index(run, simultaneous_downloads, download_index) if(fl_index > n_features-1): break row = feature_list[fl_index] try: out_name, task = export_image_from_point(row, out_file_name, group_table_ID, min_date, max_date) if out_name == 'outside date range': break else: out_drive_names.append(out_name) task_list.append(task) break except urllib2.HTTPError as err: failure_counter = exponential_backoff(failure_counter) if(failure_counter == allowed_failures): failed_groups, failure_counter = log_error(failed_groups, allowed_failures, failure_counter, fl_index) except: failed_groups, failure_counter = log_error(failed_groups, allowed_failures, failure_counter, fl_index) return(failed_groups, task_list, out_drive_names) n_features = len(feature_list) runs = get_runs(n_features, simultaneous_downloads) failed_groups = [] service = gen.authenticate_drive_api() for run in xrange(runs): task_list = [] out_drive_names = [] for download_index in xrange(simultaneous_downloads): failed_groups, task_list, out_drive_names = start_task(download_index, run, simultaneous_downloads, task_list, out_drive_names, feature_list, n_features, group_table_ID, allowed_failures, failed_groups) wait_on_task_list(task_list, run, simultaneous_downloads, allowed_failures, failed_groups) for out_name in out_drive_names: download_data(out_name + '.tif', dirs.intermediate_directory + '\\' + out_name + '.tif', service)