def connection(self): if self._connection is None: self._connection = object_storage.get_client(self.username, self.api_key, self.auth_url, datacenter=self.cluster) return self._connection
def send_to_storage(maindir = 'data'): """ sends """ sl_storage = object_storage.get_client(sl_user_name, sl_api_key, datacenter = sl_data_center) here = [f for f in os.walk(maindir).next()[2] if f.endswith(".txt")] #print here today = (datetime.datetime.today()).date() fetchtoday = [f for f in os.walk(maindir).next()[2] if f == 'fetch_times_' + str(today) + '.txt'] #print fetchtoday move = list(set(here) - set(fetchtoday)) #print move #send here for filetosend in here: fullfilename = 'data/' + filetosend with open(fullfilename, 'r') as readfile: print "Sending " + fullfilename sl_storage['bart_dump'][readfile.name.split('/')[-1]].send(readfile) print 'All data files sent to object store.' #move move to sentdata for filetomove in move: os.rename(maindir+'/'+filetomove, maindir+'/sentdata/'+filetomove) print "Last hour's data moved."
def index_object_store(): sl_storage = object_storage.get_client( 'SLOS1356065-2:SL1356065', os.environ['SL_API_KEY'], datacenter='sjc01') containers = sl_storage.containers() auth = ('daniel', 'welcome1111') for container in containers: datasets_url = 'http://localhost:8000/datasets/' headers = {'Content-Type': 'application/json'} body = json.dumps({'name': str(container.name)}) print datasets_url print body print headers r = requests.post(datasets_url, data=body, headers=headers, auth=auth) print r objects = container.objects() for object in objects: raw_url = 'http://localhost:8000/raw/' location = str(object.url) if not (location.endswith(".fsa")): continue body = json.dumps({'dataset': container.name, 'location': str(location)}) headers = {'Content-Type': 'application/json'} requests.post(raw_url, data=body, headers=headers, auth=auth)
def _setup_client(self): use_network = "private" if self.use_private else "public" object_storage.consts.USER_AGENT = __agent__ self.client = object_storage.get_client( self.username, self.apikey, datacenter=self.dc, network=use_network, auth_url=self.auth_url )
def __init__(self, args, extra): self.args = args self.extra = extra self.client = object_storage.get_client( self.args.username, self.args.api_key, datacenter=self.args.datacenter, network=self.args.network)
def get_container(app, name=None): if name is None: name = app.container object_storage.consts.USER_AGENT = __agent__ client = object_storage.get_client( app.username, app.apikey, auth_token=app.token, auth_url=app.auth_url) client.set_storage_url(app.url) return client[name]
def authenticate(self): use_network = 'private' if self.use_private else 'public' object_storage.consts.USER_AGENT = __agent__ client = object_storage.get_client( self.username, self.apikey, datacenter=self.dc, network=use_network, auth_url=self.auth_url) logging.info("Logging in as %s in %s", self.username, self.dc) client.conn.auth.authenticate() self.url = client.get_url() self.token = copy(client.conn.auth.auth_token) del client
def setUp(self): for key in 'account username password'.split(): if key not in config: raise unittest.SkipTest( "%s not set in the test config file" % key) protocol = 'http' if config.get('auth_ssl', 'no').lower() in ('yes', 'true', 'on', '1'): protocol = 'https' host = config.get('auth_host', '127.0.0.1') port = config.get('auth_port', '8080') auth_prefix = config.get('auth_prefix', '/auth/') auth_url = '%s://%s:%s%sv1.0' % (protocol, host, port, auth_prefix) username = "******" % (config['account'], config['username']) api_key = config['password'] self.client = object_storage.get_client( username, api_key, auth_url=auth_url) self.sleep_factor = int(config.get('sleep_factor', 2)) self.time_out = int(config.get('time_out',2))
def main(): args = docopt(__doc__, version='kleiber cluster director 0.1') try: configuration = yaml.load( open(os.environ['HOME'] + '/.kleiber', 'r').read()) except IOError: error('kleiber configuration file {} not found'.format( os.environ['HOME'] + '/.kleiber')) client = SoftLayer.create_client_from_env( username=configuration['username'], api_key=configuration['api_key']) if 'obj_store' not in configuration: if 'local_store' in configuration: directory = configuration['local_store'] else: directory = os.path.expanduser("~/.kleiber-data") sl_storage = { 'type': 'local', 'directory': os.path.expanduser(directory + "/containers/") } else: sl_storage = { 'type': 'swift', 'client': object_storage.get_client( configuration['obj_store']['name'] + ':' + configuration['username'], configuration['api_key'], datacenter=configuration['obj_store']['datacenter']) } if args['create']: do_create(args, client, sl_storage, configuration) elif args['status']: do_status(args, client, sl_storage, configuration) elif args['delete']: do_delete(args, client, sl_storage, configuration) elif args['list']: print clusters(sl_storage)
def trans(): sl_storage = object_storage.get_client(sl_user_name, sl_api_key, datacenter = sl_data_center) dump_container = 'weather_dump' archive_container = 'weather_archive' #grab all files fts_pre = sl_storage[archive_container].objects() fts = [] print "Grabbing list of fetch_time files..." for pre in fts_pre: prestr = pre.__str__() try: match = re.search(archive_container + ', (.*)\.txt', prestr) fts.append(match.group(1) + '.txt') except: pass print fts #for each fetchtime for ft in fts: print "Reading " + ft #read fetchtime file from object_store and write to to_parse newLocalFile = open('to_parse/'+ft, "w") swiftObject = sl_storage[archive_container][ft].read() newLocalFile.write(swiftObject) newLocalFile.close() to_send = fts print "Sending files for dump container..." for i, item in enumerate(to_send): print "Sending file " + str(i + 1) + ": " + item fullfilename = 'to_parse/' + item with open(fullfilename, 'r') as readfile: sl_storage[dump_container][readfile.name.split('/')[-1]].send(readfile) print 'Data files sent to object store.'
def main(): args = docopt(__doc__, version='kleiber cluster director 0.1') try: configuration = yaml.load( open(os.environ['HOME']+'/.kleiber', 'r').read()) except IOError: error('kleiber configuration file {} not found'.format( os.environ['HOME']+'/.kleiber')) client = SoftLayer.create_client_from_env( username=configuration['username'], api_key=configuration['api_key']) if 'obj_store' not in configuration: if 'local_store' in configuration: directory = configuration['local_store'] else: directory = os.path.expanduser("~/.kleiber-data") sl_storage = { 'type': 'local', 'directory': os.path.expanduser(directory+"/containers/") } else: sl_storage = { 'type': 'swift', 'client': object_storage.get_client( configuration['obj_store']['name']+':'+configuration['username'], configuration['api_key'], datacenter=configuration['obj_store']['datacenter']) } if args['create']: do_create(args, client, sl_storage, configuration) elif args['status']: do_status(args, client, sl_storage, configuration) elif args['delete']: do_delete(args, client, sl_storage, configuration) elif args['list']: print clusters(sl_storage)
import object_storage import os import time DATAFILE = os.path.expanduser("~/onedrive/github/Berkeley_W251/googlebooks.csv") googlefilesize = 1596 # connect to storage account sl_storage = object_storage.get_client('SLOS856755-2:SL856755', 'cdc24208f9bb1aff86c22d9474b79b5333c654d96503ae9761979d5eb9650ae8', datacenter='dal05') try: sl_storage['week7']['googlebooks3.csv'].create() except ResponseError: print ResponseError try: uploadstart = time.time() fileUpload = sl_storage['week7']['googlebooks3.csv'].load_from_filename(DATAFILE) except object_storage.errors.ResponseError: print ResponseError uploadcomplete = time.time() uploadtime = uploadcomplete - uploadstart print "Total upload time " + str(uploadtime) + " seconds" uploadspeed = googlefilesize/uploadtime print "Upload speed = " + str(uploadspeed) + " MB/sec"
cred = json.load(f) f.close() # コンテナとオブジェクト名 cnt = '' obj = '' ### # # メイン # if __name__ == '__main__': argvs = sys.argv # コマンドライン引数のリスト argc = len(argvs) # 個数 if (argc != 3): print 'Usage: os_delete.py container object_name' quit() cnt = argvs[1] obj = argvs[2] oos = object_storage.get_client(cred['username'], cred['password'], datacenter=cred['data_center']) res = {} res = oos[cnt][obj].search(obj) for rec in res['results']: print "Delete %s" % rec rec.delete() oos[cnt][obj].delete()
from timeit import Timer # Initialize the authentication constants user_id = 'SLOS1726245-2:SL1726245' api_key = 'be50d25997da7f6065de2d22ea7bc8640c79936611ee34852a8d70f73540b944' data_center = 'dal05' # List to hold the runtimes runtime_list = [] # Number of runs number_of_runs = 5 # Create the storage object sl_storage = object_storage.get_client(user_id, api_key, datacenter=data_center) # To create a container with in the storage def create_container(storage_obj, container_name): storage_obj[container_name].create() # To load the file in the container def load_file(storage_obj, container_name, file_location, dest_file): file_obj = open(file_location) storage_obj[container_name][dest_file].send(file_obj) def list_objects(storage_obj, container_name):
def weather_pipeline(col_names, values): #create storage client object sl_storage = object_storage.get_client(sl_user_name, sl_api_key, datacenter = sl_data_center) dump_container = 'weather_dump' archive_container = 'weather_archive' #grab all fetchtimes w_files_pre = sl_storage[dump_container].objects() w_files = [] if len(w_files_pre) > 2: w_files_pre = w_files_pre[:2] print "Grabbing list of files..." for pre in w_files_pre: prestr = pre.__str__() try: match = re.search(dump_container + ', (.*)\.txt', prestr) w_files.append(match.group(1) + '.txt') except: pass #for each file for w_file in w_files: print "Reading " + w_file #read file from object_store and write to to_parse newLocalFile = open('to_parse/'+w_file, "w") swiftObject = sl_storage[dump_container][w_file].read() newLocalFile.write(swiftObject) newLocalFile.close() print "Parsing..." parse_weather.main(w_file, col_names, values) to_send = w_files print "Sending files for archive container..." for i, item in enumerate(to_send): print "Sending file " + str(i + 1) + ": " + item fullfilename = 'to_parse/' + item with open(fullfilename, 'r') as readfile: sl_storage[archive_container][readfile.name.split('/')[-1]].send(readfile) print 'Data files sent to object store.' #if they got there safely... #delete them from local disk and remove them from dump_container sentfiles_pre = sl_storage[archive_container].objects() sentfiles = [] for pre in sentfiles_pre: prestr = pre.__str__() try: match = re.search(archive_container + ', (.*)\.txt', prestr) sentfiles.append(match.group(1) + '.txt') except: pass print sentfiles success = True for t in to_send: if t not in sentfiles: success = False if success: #delete locally for f in to_send: os.remove('to_parse/'+f) print 'Deleted local files corresponding to ' + date + '.' #delete from dump_container for f in to_send: sl_storage[dump_container][f].delete() print 'Deleted dump files corresponding to ' + date + '.' else: print 'Error: all files corresponding to ' + date + ' did not get to ' + archive_container + '. Try again later.'
import object_storage import time parser = argparse.ArgumentParser() parser.add_argument("--objectStorageAccount", default="") parser.add_argument("--username", default="") parser.add_argument("--apiKey", default="") parser.add_argument("--datacenter", default="dal05") args = parser.parse_args() username = args.objectStorageAccount + ":" + args.username apiKey = args.apiKey sl_storage = object_storage.get_client(username= username, password=apiKey, datacenter=datacenter) names = ['testContainer99'] #Create containers in array at root level def createContainers(): for name in names: sl_storage[name].create() #Get all top level containers def getContainers(): containers = sl_storage.containers() #List containers at root level print ("containers:") for container in containers:
import object_storage from django.conf import settings sl_storage = object_storage.get_client( settings.SOFTLAYER_OS['username'], settings.SOFTLAYER_OS['api_key'], datacenter='sjc01') sl_storage.containers() # [] sl_storage['foo'].create() # Container(foo) sl_storage.containers() # [Container(foo)] sl_storage['foo'].properties # {'count': 0, 'object_count': 0, 'name': 'foo', 'size': 0.0} sl_storage['foo']['bar.txt'].create() # StorageObject(foo, sample_object.txt) sl_storage['foo']['bar.txt'].send('Plain-Text Content') # True sl_storage['foo']['bar.txt'].read() # 'Plain-Text Content' sl_storage['foo'].objects() # [StorageObject(foo, bar.txt)]
import object_storage import datetime sl_storage = object_storage.get_client( 'SLOS1011257-2:SL1011257', '73b75a2cd3b2345903da72d33c6465a01c1dbf9e25acb635b79b6965ba67607c', datacenter='dal05') sl_storage.containers() sl_storage['foo'].properties start = datetime.datetime.now() print 'Started Upload' end = datetime.datetime.now() print start, end read_start = datetime.datetime.now() sl_storage['foo']['google-0.csv'].save_to_filename('google-0.csv') read_end = datetime.datetime.now() print read_start, read_end
def _get_connection(self): if not hasattr(self, '_connection'): self._connection = object_storage.get_client(self.username, self.api_key, datacenter='dal05') return self._connection
import json import object_storage import re from elasticsearch import Elasticsearch from elasticsearch.client import IndicesClient es = Elasticsearch("http://elasticm1:9200") ic = IndicesClient(es) mapping = '{"mappings": {"comment": {"properties": {"created_utc": {"type": "date","format": "epoch_second"}, "body": {"type": "string", "analyzer": "english"}}}}}' ic.create(index="2007", body=mapping) sl_storage = object_storage.get_client("username", "key", datacenter="sjc01") def iterload(src): buffer = "" dec = json.JSONDecoder() for chunk in src: buffer = buffer + chunk while True: try: r = dec.raw_decode(buffer) except: break yield r[0] buffer = buffer[r[1] :].strip(" \n\r\t") chunk_size = 512 * 1024
import json import object_storage import re from elasticsearch import Elasticsearch from elasticsearch.client import IndicesClient es = Elasticsearch("http://elasticm1:9200") ic = IndicesClient(es) mapping = '{"mappings": {"comment": {"properties": {"created_utc": {"type": "date","format": "epoch_second"}, "body": {"type": "string", "analyzer": "english"}}}}}' ic.create(index='2007', body=mapping) sl_storage = object_storage.get_client('username', 'key', datacenter='sjc01') def iterload(src): buffer = "" dec = json.JSONDecoder() for chunk in src: buffer = buffer + chunk while (True): try: r = dec.raw_decode(buffer) except: break yield r[0] buffer = buffer[r[1]:].strip(" \n\r\t") chunk_size = 512 * 1024
import object_storage import datetime sl_storage = object_storage.get_client('SLOS1011257-2:SL1011257','73b75a2cd3b2345903da72d33c6465a01c1dbf9e25acb635b79b6965ba67607c', datacenter='dal05') sl_storage.containers() sl_storage['foo'].properties start = datetime.datetime.now() print 'Started Upload' end = datetime.datetime.now() print start, end read_start = datetime.datetime.now() sl_storage['foo']['google-0.csv'].save_to_filename('google-0.csv') read_end = datetime.datetime.now() print read_start, read_end