def upload():
print('in upload')
# download() delete all the images from the prev instance of running the website
# get the function running
msg = requests.get('https://azunzipmangroves.azurewebsites.net/')
if msg.status_code == 200:
# check if the post request has the file part
if 'file' not in request.files:
print('No file part')
return redirect(request.url)
file = request.files['file']
# if user does not select file, browser also
# submit an empty part without filename
if file.filename == '':
html = render_template('index.html')
response = make_response(html)
return response
elif file and allowed_file(file.filename):
filename = secure_filename(file.filename)
print('filename:', filename)
input_container = 'input-files'
client = azure_blob.DirectoryClient(CONNECTION_STRING,
input_container)
print('created client')
client.create_blob_from_stream(blob_name=filename, stream=file)
print('completed file upload')
# file.save(os.path.join(server.config['UPLOAD_FOLDER'], filename))
else:
print('error occured HANDLE THIS!')
html = render_template('index.html')
response = make_response(html)
return response
def download_result_df():
PRED_CONTAINER_NAME = 'prediction-results'
client_pred = azure_blob.DirectoryClient(CONNECTION_STRING,
PRED_CONTAINER_NAME)
client_pred.download_file('content.csv', MAIN_DIRECTORY)
try:
client_pred.rmdir('')
except:
print("error when deleting from blob storage")
return
def searchResults():
print('search results is called')
output_container_name = 'output-files'
client = azure_blob.DirectoryClient(CONNECTION_STRING,
output_container_name)
list_of_files = list(client.ls_files('', recursive=False))
html = ''
for filename in list_of_files:
html += filename + '<br>'
response = make_response(html)
return response
def classificationfin():
print('in classification fin')
container_name = 'prediction-results'
client = azure_blob.DirectoryClient(CONNECTION_STRING, container_name)
list_of_files = list(client.ls_files('', recursive=False))
print(list_of_files)
if list_of_files != []:
if list_of_files[0] == 'content.csv':
html = 'Classification finished.'
else:
html = ''
response = make_response(html)
return response
def classify():
output_container_name = 'output-files'
client = azure_blob.DirectoryClient(CONNECTION_STRING,
output_container_name)
list_of_files = list(client.ls_files('', recursive=False))
print("number of tif files in output-files: ", len(list_of_files))
# KEEP THIS
BATCH_SIZE = 10
BIG_BATCH_SIZE = 10
batch_list = get_batch_list(list_of_files, BATCH_SIZE)
n_batches = len(batch_list)
print('number of batches:' + str(n_batches))
big_batches = math.floor(n_batches / BIG_BATCH_SIZE)
json_msg_final = []
for bigbatch in range(big_batches):
start = str(bigbatch * BATCH_SIZE * BIG_BATCH_SIZE)
print('start:' + start)
msg = requests.get(
'https://predict-mangroves.azurewebsites.net/api/classify?method=predict&start='
+ start)
print(msg)
# try again
while (msg.status_code != 200):
# restart the function
msg = requests.get(
'https://predict-mangroves.azurewebsites.net/api/classify?method=predict&start='
+ start)
print(msg)
# print(msg_get)
json_msg = msg.json()
json_msg = json.loads(json_msg)
# print('json 1st row: ', json_msg[0])
print('entire: ', json_msg)
json_msg_final.append(json_msg)
json_msg_final = list(itertools.chain.from_iterable(json_msg_final))
result_df = pd.DataFrame.from_records(json_msg_final)
print(result_df.head())
for n, batch in enumerate(batch_list):
# Download all tifs in the batch
# Memory: 0.16015625
for i in range(len(batch)):
client.download_file(batch[i],
str(MAIN_DIRECTORY + "images/images/"))
print('downsampling images')
ds_factor = 1 / 10
for rel_filename in batch:
FILENAME = UPLOAD_FOLDER + rel_filename
img, _ = raster.load_image(FILENAME)
_, _ = raster.downsample_raster(img, ds_factor, FILENAME)
# REUPLOAD DOWNSAMPLE TIFS TO DATABASE
# memory: 0Mb
print('reuploading batch')
for rel_filename in batch:
FILENAME = UPLOAD_FOLDER + rel_filename
client.upload_file(FILENAME, rel_filename)
# DELETE ALL TIFS IN images/images to prepare for the next batch
# Memory: 54.1953125 Mb
print('deleting images in folder')
delete_files_in_dir(UPLOAD_FOLDER)
# gc.get_stats()
gc.collect()
result_df.to_csv(r'content.csv')
filename_result_df = 'content.csv'
save_result_df(filename_result_df)
return
def post_classify():
download_result_df()
output_container_name = 'output-files'
client = azure_blob.DirectoryClient(CONNECTION_STRING,
output_container_name)
# KEEP THIS
gc.collect()
blobs = client.ls_files(path='')
for blob in blobs:
client.download_file(source=blob, dest=IMAGE_DIRECTORY + '/images/')
result_df = pd.read_csv('content.csv')
# KEEP THIS
dest_folders = []
# Organize tiles into folders
for index, row in tqdm(result_df.iterrows()):
cur_file = UPLOAD_FOLDER + row['filename']
# cur_file = cur_file.replace("jpg","tif",2)
classification = row['prediction']
#set destination folder, and creates the folder if it doesn't exist
dest_folder = os.path.join(os.path.abspath(IMAGE_DIRECTORY),
str(classification))
dest_folders.append(dest_folder)
if os.path.exists(dest_folder) == False:
os.mkdir(dest_folder)
dest = os.path.join(dest_folder, os.path.basename(cur_file))
#moves file
src = cur_file
os.rename(src, dest)
print('organized into folders')
nonmangrove_exists = False
mangrove_exists = False
if (os.path.isdir(IMAGE_DIRECTORY + '/1/')):
nonmangrove_exists = True
if (os.path.isdir(IMAGE_DIRECTORY + '/0/')):
mangrove_exists = True
if nonmangrove_exists:
nm_img_list = list(os.listdir(IMAGE_DIRECTORY + '/1/'))
nm_img_path = IMAGE_DIRECTORY + '/1/'
nm_img_list = prepend(nm_img_list, nm_img_path)
raster.merge_raster(nm_img_list,
output_file=MAIN_DIRECTORY + nm_filename + ".tif")
print('created ' + nm_filename + '.tif')
os.system('gdal_polygonize.py ' + nm_filename +
'.tif -f "ESRI Shapefile" -b 4 ' + nm_filename + '.shp')
tif_to_jpg(MAIN_DIRECTORY + nm_filename + ".tif")
delete_files_in_dir(IMAGE_DIRECTORY + '/1/')
if mangrove_exists:
m_img_list = list(os.listdir(IMAGE_DIRECTORY + '/0/'))
m_img_path = IMAGE_DIRECTORY + '/0/'
m_img_list = prepend(m_img_list, m_img_path)
raster.merge_raster(m_img_list,
output_file=MAIN_DIRECTORY + m_filename + ".tif")
print('created ' + m_filename + '.tif')
os.system('gdal_polygonize.py ' + m_filename +
'.tif -f "ESRI Shapefile" -b 4 ' + m_filename + '.shp')
# store tif as jpg for visualization
tif_to_jpg(MAIN_DIRECTORY + m_filename + ".tif")
# Delete files in images/images
delete_files_in_dir(IMAGE_DIRECTORY + '/0/')
# fix shape files
if mangrove_exists:
fix_shp(m_filename + '.shp')
if nonmangrove_exists:
fix_shp(nm_filename + '.shp')
delete_files_in_dir(IMAGE_DIRECTORY + '/images/')
# Delete the files in the blob containers
# remove files in output-files container
try:
client.rmdir('')
except:
print("error when deleting from blob storage")
# remove files in input-files container
input_container_name = 'input-files'
client = azure_blob.DirectoryClient(CONNECTION_STRING,
input_container_name)
try:
client.rmdir('')
except:
print("error when deleting from blob storage")
print("classification finished")
return
def save_result_df(filename):
PRED_CONTAINER_NAME = 'prediction-results'
client_pred = azure_blob.DirectoryClient(CONNECTION_STRING,
PRED_CONTAINER_NAME)
client_pred.upload_file(filename, filename)
return