def get_training_data(location_id):
remote_dir = storage_manager.generate_storage_path(STORAGE_TYPE, BUCKET_NAME, REMOTE_TRAINING_DIR)
fl = storage_manager.get_list_of_remote_path(remote_dir)
# TODO: choose training data based on time
for f in fl:
fn = os.path.basename(f)
if fn.startswith(str(location_id)):
tmp_local_dir = '/tmp/fusion_training/'
if not os.path.exists(tmp_local_dir):
os.makedirs(tmp_local_dir)
tmp_path = os.path.join(tmp_local_dir, fn)
storage_manager.download_to_local_path(f, tmp_path)
local_f_handler = open(tmp_path, 'r')
csv_r = csv.reader(local_f_handler)
training_data = []
first_line_flag = True
for r in csv_r:
if first_line_flag:
first_line_flag = False
continue
try:
training_data.append(map(int,r[-6:]))
except:
pass
local_f_handler.close()
os.remove(tmp_path)
return training_data
def download_dir(bucket_name, directory, save_dir):
# Usage: python download_dir.py <bucket_name> <directory> <save_dir>\n\neg. python download_dir.py percolata-data software/fixed software
save_dir = os.path.expanduser(save_dir)
storage_type = 2
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
storage_path = storage_manager.generate_storage_path(storage_type, bucket_name, directory)
print storage_path
storage_manager.download_dir_to_local(storage_path, save_dir)
print "download %s to %s ok" % (directory, save_dir)
def upload_file(bucket_name, directory, local_file):
#Usage: python upload_file.py <bucket_name> <directory> <local_file>\n\neg. python upload_file.py percolata-data software/fixed ./test/test.json
local_file = os.path.expanduser(local_file)
storage_type = 2
if os.path.exists(local_file):
fp = open(local_file)
else:
print "file not found: %s" % local_file
storage_path = storage_manager.generate_storage_path(storage_type, bucket_name, directory)
print storage_path
storage_manager.upload_to_container_from_file(storage_path, fp)
print "upload %s to %s ok" % (local_file, directory)
import os
from backend_common.storage import storage_manager
import sys
import shutil
import sys
# need install boto and azure
args = sys.argv
try:
bucket_name, directory, local_file = args[1:]
except Exception as e:
print "Usage: python upload_file.py <bucket_name> <directory> <local_file>\n\neg. python upload_file.py percolata-data software/fixed ./test/test.json"
sys.exit()
local_file = os.path.expanduser(local_file)
storage_type = 1
if os.path.exists(local_file):
fp = open(local_file)
else:
print "file not found: %s" % local_file
storage_path = storage_manager.generate_storage_path(storage_type, bucket_name, directory)
print storage_path
storage_manager.upload_to_container_from_file(storage_path, fp)
print "upload %s to %s ok" % (local_file, directory)
