def get_training_data(location_id): remote_dir = storage_manager.generate_storage_path(STORAGE_TYPE, BUCKET_NAME, REMOTE_TRAINING_DIR) fl = storage_manager.get_list_of_remote_path(remote_dir) # TODO: choose training data based on time for f in fl: fn = os.path.basename(f) if fn.startswith(str(location_id)): tmp_local_dir = '/tmp/fusion_training/' if not os.path.exists(tmp_local_dir): os.makedirs(tmp_local_dir) tmp_path = os.path.join(tmp_local_dir, fn) storage_manager.download_to_local_path(f, tmp_path) local_f_handler = open(tmp_path, 'r') csv_r = csv.reader(local_f_handler) training_data = [] first_line_flag = True for r in csv_r: if first_line_flag: first_line_flag = False continue try: training_data.append(map(int,r[-6:])) except: pass local_f_handler.close() os.remove(tmp_path) return training_data
def download_dir(bucket_name, directory, save_dir): # Usage: python download_dir.py <bucket_name> <directory> <save_dir>\n\neg. python download_dir.py percolata-data software/fixed software save_dir = os.path.expanduser(save_dir) storage_type = 2 if not os.path.isdir(save_dir): os.makedirs(save_dir) storage_path = storage_manager.generate_storage_path(storage_type, bucket_name, directory) print storage_path storage_manager.download_dir_to_local(storage_path, save_dir) print "download %s to %s ok" % (directory, save_dir)
def upload_file(bucket_name, directory, local_file): #Usage: python upload_file.py <bucket_name> <directory> <local_file>\n\neg. python upload_file.py percolata-data software/fixed ./test/test.json local_file = os.path.expanduser(local_file) storage_type = 2 if os.path.exists(local_file): fp = open(local_file) else: print "file not found: %s" % local_file storage_path = storage_manager.generate_storage_path(storage_type, bucket_name, directory) print storage_path storage_manager.upload_to_container_from_file(storage_path, fp) print "upload %s to %s ok" % (local_file, directory)
import os from backend_common.storage import storage_manager import sys import shutil import sys # need install boto and azure args = sys.argv try: bucket_name, directory, local_file = args[1:] except Exception as e: print "Usage: python upload_file.py <bucket_name> <directory> <local_file>\n\neg. python upload_file.py percolata-data software/fixed ./test/test.json" sys.exit() local_file = os.path.expanduser(local_file) storage_type = 1 if os.path.exists(local_file): fp = open(local_file) else: print "file not found: %s" % local_file storage_path = storage_manager.generate_storage_path(storage_type, bucket_name, directory) print storage_path storage_manager.upload_to_container_from_file(storage_path, fp) print "upload %s to %s ok" % (local_file, directory)