def download_data(group):
service = gen.authenticate_drive_api()
kml_file_id = gen.get_tableId_by_table_name("SSURGO_" + str(group) + ".kml",
service, google_service='drive')
bytes_data = download_file(service, group, kml_file_id)
out_file = write_download_to_file(bytes_data, group)
delete_files_on_drive(service, group, kml_file_id)
return out_file
def delete_temp_tables():
for file_str in os.listdir(dirs.intermediate_directory):
if 'SSURGO_' in file_str:
os.remove(dirs.intermediate_directory + "\\" + file_str)
service = gen.authenticate_drive_api()
ids = gen.get_tableId_by_table_name('SSURGO_', service,
exact=False, google_service='drive')
for file_id in ids:
service.files().trash(fileId=file_id).execute()
service.files().emptyTrash()
def parralelize_points(feature_list, out_file_name, simultaneous_downloads, group_table_ID,
min_date, max_date, start_index, end_index):
class EE_manual_error(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr(self.value)
def wait_on_task_list(task_list, run, simultaneous_downloads, allowed_failures, failed_groups):
def wait_on_task(task, failed_groups, allowed_failures, fl_index):
failure_counter = 0
while(True):
try:
status = task.status()['state']
except urllib2.HTTPError as err:
failure_counter = exponential_backoff(failure_counter)
if(failure_counter == allowed_failures):
failed_groups, failure_counter = log_error(failed_groups,
allowed_failures, failure_counter, fl_index)
if status in ['READY', 'RUNNING']: time.sleep(3 + np.random.uniform())
else: break
def failed_task(task):
if(task.status()['state'] == 'FAILED'):
raise EE_manual_error(task.status()['error_message'])
n_tasks = len(task_list)
fl_index = run*simultaneous_downloads
while(n_tasks != 0):
task = task_list[0]
wait_on_task(task, failed_groups, allowed_failures, fl_index)
try: failed_task(task)
except:
failed_groups, counter = log_error(failed_groups, 10, 0, fl_index)
if(n_tasks>1):
task_list = task_list[1:]
n_tasks = len(task_list)
fl_index = fl_index+1
else: break
def get_runs(n_features, simultaneous_downloads):
runs = int(math.floor(n_features / simultaneous_downloads) + 1)
return runs
def get_feature_list_index(run, simultaneous_downloads, dl):
fl_index = run*simultaneous_downloads + dl
return fl_index
def start_task(download_index, run, simultaneous_downloads, task_list, out_drive_names,
feature_list, n_features, group_table_ID, allowed_failures, failed_groups):
failure_counter = 0
while(failure_counter < allowed_failures):
fl_index = get_feature_list_index(run, simultaneous_downloads, download_index)
if(fl_index > n_features-1): break
row = feature_list[fl_index]
try:
out_name, task = export_image_from_point(row, out_file_name, group_table_ID, min_date, max_date)
if out_name == 'outside date range': break
else:
out_drive_names.append(out_name)
task_list.append(task)
break
except urllib2.HTTPError as err:
failure_counter = exponential_backoff(failure_counter)
if(failure_counter == allowed_failures):
failed_groups, failure_counter = log_error(failed_groups,
allowed_failures, failure_counter, fl_index)
except:
failed_groups, failure_counter = log_error(failed_groups,
allowed_failures, failure_counter, fl_index)
return(failed_groups, task_list, out_drive_names)
n_features = len(feature_list)
runs = get_runs(n_features, simultaneous_downloads)
failed_groups = []
service = gen.authenticate_drive_api()
for run in xrange(runs):
task_list = []
out_drive_names = []
for download_index in xrange(simultaneous_downloads):
failed_groups, task_list, out_drive_names = start_task(download_index, run, simultaneous_downloads,
task_list, out_drive_names, feature_list, n_features, group_table_ID, allowed_failures, failed_groups)
wait_on_task_list(task_list, run, simultaneous_downloads, allowed_failures, failed_groups)
for out_name in out_drive_names:
download_data(out_name + '.tif', dirs.intermediate_directory +
'\\' + out_name + '.tif', service)
