def connection(self):
if self._connection is None:
self._connection = object_storage.get_client(self.username,
self.api_key,
self.auth_url,
datacenter=self.cluster)
return self._connection
def send_to_storage(maindir = 'data'):
"""
sends
"""
sl_storage = object_storage.get_client(sl_user_name, sl_api_key, datacenter = sl_data_center)
here = [f for f in os.walk(maindir).next()[2] if f.endswith(".txt")]
#print here
today = (datetime.datetime.today()).date()
fetchtoday = [f for f in os.walk(maindir).next()[2] if f == 'fetch_times_' + str(today) + '.txt']
#print fetchtoday
move = list(set(here) - set(fetchtoday))
#print move
#send here
for filetosend in here:
fullfilename = 'data/' + filetosend
with open(fullfilename, 'r') as readfile:
print "Sending " + fullfilename
sl_storage['bart_dump'][readfile.name.split('/')[-1]].send(readfile)
print 'All data files sent to object store.'
#move move to sentdata
for filetomove in move:
os.rename(maindir+'/'+filetomove, maindir+'/sentdata/'+filetomove)
print "Last hour's data moved."
def index_object_store():
sl_storage = object_storage.get_client(
'SLOS1356065-2:SL1356065',
os.environ['SL_API_KEY'],
datacenter='sjc01')
containers = sl_storage.containers()
auth = ('daniel', 'welcome1111')
for container in containers:
datasets_url = 'http://localhost:8000/datasets/'
headers = {'Content-Type': 'application/json'}
body = json.dumps({'name': str(container.name)})
print datasets_url
print body
print headers
r = requests.post(datasets_url, data=body, headers=headers, auth=auth)
print r
objects = container.objects()
for object in objects:
raw_url = 'http://localhost:8000/raw/'
location = str(object.url)
if not (location.endswith(".fsa")):
continue
body = json.dumps({'dataset': container.name, 'location': str(location)})
headers = {'Content-Type': 'application/json'}
requests.post(raw_url, data=body, headers=headers, auth=auth)
def _setup_client(self):
use_network = "private" if self.use_private else "public"
object_storage.consts.USER_AGENT = __agent__
self.client = object_storage.get_client(
self.username, self.apikey, datacenter=self.dc, network=use_network, auth_url=self.auth_url
)
def __init__(self, args, extra):
self.args = args
self.extra = extra
self.client = object_storage.get_client(
self.args.username,
self.args.api_key,
datacenter=self.args.datacenter,
network=self.args.network)
def get_container(app, name=None):
if name is None:
name = app.container
object_storage.consts.USER_AGENT = __agent__
client = object_storage.get_client(
app.username,
app.apikey,
auth_token=app.token,
auth_url=app.auth_url)
client.set_storage_url(app.url)
return client[name]
def authenticate(self):
use_network = 'private' if self.use_private else 'public'
object_storage.consts.USER_AGENT = __agent__
client = object_storage.get_client(
self.username,
self.apikey,
datacenter=self.dc,
network=use_network,
auth_url=self.auth_url)
logging.info("Logging in as %s in %s",
self.username, self.dc)
client.conn.auth.authenticate()
self.url = client.get_url()
self.token = copy(client.conn.auth.auth_token)
del client
def setUp(self):
for key in 'account username password'.split():
if key not in config:
raise unittest.SkipTest(
"%s not set in the test config file" % key)
protocol = 'http'
if config.get('auth_ssl', 'no').lower() in ('yes', 'true', 'on', '1'):
protocol = 'https'
host = config.get('auth_host', '127.0.0.1')
port = config.get('auth_port', '8080')
auth_prefix = config.get('auth_prefix', '/auth/')
auth_url = '%s://%s:%s%sv1.0' % (protocol, host, port, auth_prefix)
username = "******" % (config['account'], config['username'])
api_key = config['password']
self.client = object_storage.get_client(
username, api_key, auth_url=auth_url)
self.sleep_factor = int(config.get('sleep_factor', 2))
self.time_out = int(config.get('time_out',2))
def main():
args = docopt(__doc__, version='kleiber cluster director 0.1')
try:
configuration = yaml.load(
open(os.environ['HOME'] + '/.kleiber', 'r').read())
except IOError:
error('kleiber configuration file {} not found'.format(
os.environ['HOME'] + '/.kleiber'))
client = SoftLayer.create_client_from_env(
username=configuration['username'], api_key=configuration['api_key'])
if 'obj_store' not in configuration:
if 'local_store' in configuration:
directory = configuration['local_store']
else:
directory = os.path.expanduser("~/.kleiber-data")
sl_storage = {
'type': 'local',
'directory': os.path.expanduser(directory + "/containers/")
}
else:
sl_storage = {
'type':
'swift',
'client':
object_storage.get_client(
configuration['obj_store']['name'] + ':' +
configuration['username'],
configuration['api_key'],
datacenter=configuration['obj_store']['datacenter'])
}
if args['create']:
do_create(args, client, sl_storage, configuration)
elif args['status']:
do_status(args, client, sl_storage, configuration)
elif args['delete']:
do_delete(args, client, sl_storage, configuration)
elif args['list']:
print clusters(sl_storage)
def trans():
sl_storage = object_storage.get_client(sl_user_name, sl_api_key, datacenter = sl_data_center)
dump_container = 'weather_dump'
archive_container = 'weather_archive'
#grab all files
fts_pre = sl_storage[archive_container].objects()
fts = []
print "Grabbing list of fetch_time files..."
for pre in fts_pre:
prestr = pre.__str__()
try:
match = re.search(archive_container + ', (.*)\.txt', prestr)
fts.append(match.group(1) + '.txt')
except:
pass
print fts
#for each fetchtime
for ft in fts:
print "Reading " + ft
#read fetchtime file from object_store and write to to_parse
newLocalFile = open('to_parse/'+ft, "w")
swiftObject = sl_storage[archive_container][ft].read()
newLocalFile.write(swiftObject)
newLocalFile.close()
to_send = fts
print "Sending files for dump container..."
for i, item in enumerate(to_send):
print "Sending file " + str(i + 1) + ": " + item
fullfilename = 'to_parse/' + item
with open(fullfilename, 'r') as readfile:
sl_storage[dump_container][readfile.name.split('/')[-1]].send(readfile)
print 'Data files sent to object store.'
def main():
args = docopt(__doc__, version='kleiber cluster director 0.1')
try:
configuration = yaml.load(
open(os.environ['HOME']+'/.kleiber', 'r').read())
except IOError:
error('kleiber configuration file {} not found'.format(
os.environ['HOME']+'/.kleiber'))
client = SoftLayer.create_client_from_env(
username=configuration['username'],
api_key=configuration['api_key'])
if 'obj_store' not in configuration:
if 'local_store' in configuration:
directory = configuration['local_store']
else:
directory = os.path.expanduser("~/.kleiber-data")
sl_storage = {
'type': 'local',
'directory': os.path.expanduser(directory+"/containers/")
}
else:
sl_storage = {
'type': 'swift',
'client': object_storage.get_client(
configuration['obj_store']['name']+':'+configuration['username'],
configuration['api_key'],
datacenter=configuration['obj_store']['datacenter'])
}
if args['create']:
do_create(args, client, sl_storage, configuration)
elif args['status']:
do_status(args, client, sl_storage, configuration)
elif args['delete']:
do_delete(args, client, sl_storage, configuration)
elif args['list']:
print clusters(sl_storage)
import object_storage
import os
import time
DATAFILE = os.path.expanduser("~/onedrive/github/Berkeley_W251/googlebooks.csv")
googlefilesize = 1596
# connect to storage account
sl_storage = object_storage.get_client('SLOS856755-2:SL856755',
'cdc24208f9bb1aff86c22d9474b79b5333c654d96503ae9761979d5eb9650ae8',
datacenter='dal05')
try:
sl_storage['week7']['googlebooks3.csv'].create()
except ResponseError:
print ResponseError
try:
uploadstart = time.time()
fileUpload = sl_storage['week7']['googlebooks3.csv'].load_from_filename(DATAFILE)
except object_storage.errors.ResponseError:
print ResponseError
uploadcomplete = time.time()
uploadtime = uploadcomplete - uploadstart
print "Total upload time " + str(uploadtime) + " seconds"
uploadspeed = googlefilesize/uploadtime
print "Upload speed = " + str(uploadspeed) + " MB/sec"
cred = json.load(f)
f.close()
# コンテナとオブジェクト名
cnt = ''
obj = ''
###
#
# メイン
#
if __name__ == '__main__':
argvs = sys.argv # コマンドライン引数のリスト
argc = len(argvs) # 個数
if (argc != 3):
print 'Usage: os_delete.py container object_name'
quit()
cnt = argvs[1]
obj = argvs[2]
oos = object_storage.get_client(cred['username'], cred['password'], datacenter=cred['data_center'])
res = {}
res = oos[cnt][obj].search(obj)
for rec in res['results']:
print "Delete %s" % rec
rec.delete()
oos[cnt][obj].delete()
from timeit import Timer
# Initialize the authentication constants
user_id = 'SLOS1726245-2:SL1726245'
api_key = 'be50d25997da7f6065de2d22ea7bc8640c79936611ee34852a8d70f73540b944'
data_center = 'dal05'
# List to hold the runtimes
runtime_list = []
# Number of runs
number_of_runs = 5
# Create the storage object
sl_storage = object_storage.get_client(user_id,
api_key,
datacenter=data_center)
# To create a container with in the storage
def create_container(storage_obj, container_name):
storage_obj[container_name].create()
# To load the file in the container
def load_file(storage_obj, container_name, file_location, dest_file):
file_obj = open(file_location)
storage_obj[container_name][dest_file].send(file_obj)
def list_objects(storage_obj, container_name):
def weather_pipeline(col_names, values):
#create storage client object
sl_storage = object_storage.get_client(sl_user_name, sl_api_key, datacenter = sl_data_center)
dump_container = 'weather_dump'
archive_container = 'weather_archive'
#grab all fetchtimes
w_files_pre = sl_storage[dump_container].objects()
w_files = []
if len(w_files_pre) > 2:
w_files_pre = w_files_pre[:2]
print "Grabbing list of files..."
for pre in w_files_pre:
prestr = pre.__str__()
try:
match = re.search(dump_container + ', (.*)\.txt', prestr)
w_files.append(match.group(1) + '.txt')
except:
pass
#for each file
for w_file in w_files:
print "Reading " + w_file
#read file from object_store and write to to_parse
newLocalFile = open('to_parse/'+w_file, "w")
swiftObject = sl_storage[dump_container][w_file].read()
newLocalFile.write(swiftObject)
newLocalFile.close()
print "Parsing..."
parse_weather.main(w_file, col_names, values)
to_send = w_files
print "Sending files for archive container..."
for i, item in enumerate(to_send):
print "Sending file " + str(i + 1) + ": " + item
fullfilename = 'to_parse/' + item
with open(fullfilename, 'r') as readfile:
sl_storage[archive_container][readfile.name.split('/')[-1]].send(readfile)
print 'Data files sent to object store.'
#if they got there safely...
#delete them from local disk and remove them from dump_container
sentfiles_pre = sl_storage[archive_container].objects()
sentfiles = []
for pre in sentfiles_pre:
prestr = pre.__str__()
try:
match = re.search(archive_container + ', (.*)\.txt', prestr)
sentfiles.append(match.group(1) + '.txt')
except:
pass
print sentfiles
success = True
for t in to_send:
if t not in sentfiles:
success = False
if success:
#delete locally
for f in to_send:
os.remove('to_parse/'+f)
print 'Deleted local files corresponding to ' + date + '.'
#delete from dump_container
for f in to_send:
sl_storage[dump_container][f].delete()
print 'Deleted dump files corresponding to ' + date + '.'
else:
print 'Error: all files corresponding to ' + date + ' did not get to ' + archive_container + '. Try again later.'
import object_storage
import time
parser = argparse.ArgumentParser()
parser.add_argument("--objectStorageAccount", default="")
parser.add_argument("--username", default="")
parser.add_argument("--apiKey", default="")
parser.add_argument("--datacenter", default="dal05")
args = parser.parse_args()
username = args.objectStorageAccount + ":" + args.username
apiKey = args.apiKey
sl_storage = object_storage.get_client(username= username, password=apiKey, datacenter=datacenter)
names = ['testContainer99']
#Create containers in array at root level
def createContainers():
for name in names:
sl_storage[name].create()
#Get all top level containers
def getContainers():
containers = sl_storage.containers()
#List containers at root level
print ("containers:")
for container in containers:
import object_storage
from django.conf import settings
sl_storage = object_storage.get_client(
settings.SOFTLAYER_OS['username'],
settings.SOFTLAYER_OS['api_key'],
datacenter='sjc01')
sl_storage.containers()
# []
sl_storage['foo'].create()
# Container(foo)
sl_storage.containers()
# [Container(foo)]
sl_storage['foo'].properties
# {'count': 0, 'object_count': 0, 'name': 'foo', 'size': 0.0}
sl_storage['foo']['bar.txt'].create()
# StorageObject(foo, sample_object.txt)
sl_storage['foo']['bar.txt'].send('Plain-Text Content')
# True
sl_storage['foo']['bar.txt'].read()
# 'Plain-Text Content'
sl_storage['foo'].objects()
# [StorageObject(foo, bar.txt)]
import object_storage
import datetime
sl_storage = object_storage.get_client(
'SLOS1011257-2:SL1011257',
'73b75a2cd3b2345903da72d33c6465a01c1dbf9e25acb635b79b6965ba67607c',
datacenter='dal05')
sl_storage.containers()
sl_storage['foo'].properties
start = datetime.datetime.now()
print 'Started Upload'
end = datetime.datetime.now()
print start, end
read_start = datetime.datetime.now()
sl_storage['foo']['google-0.csv'].save_to_filename('google-0.csv')
read_end = datetime.datetime.now()
print read_start, read_end
def _get_connection(self):
if not hasattr(self, '_connection'):
self._connection = object_storage.get_client(self.username, self.api_key, datacenter='dal05')
return self._connection
import json
import object_storage
import re
from elasticsearch import Elasticsearch
from elasticsearch.client import IndicesClient
es = Elasticsearch("http://elasticm1:9200")
ic = IndicesClient(es)
mapping = '{"mappings": {"comment": {"properties": {"created_utc": {"type": "date","format": "epoch_second"}, "body": {"type": "string", "analyzer": "english"}}}}}'
ic.create(index="2007", body=mapping)
sl_storage = object_storage.get_client("username", "key", datacenter="sjc01")
def iterload(src):
buffer = ""
dec = json.JSONDecoder()
for chunk in src:
buffer = buffer + chunk
while True:
try:
r = dec.raw_decode(buffer)
except:
break
yield r[0]
buffer = buffer[r[1] :].strip(" \n\r\t")
chunk_size = 512 * 1024
import json
import object_storage
import re
from elasticsearch import Elasticsearch
from elasticsearch.client import IndicesClient
es = Elasticsearch("http://elasticm1:9200")
ic = IndicesClient(es)
mapping = '{"mappings": {"comment": {"properties": {"created_utc": {"type": "date","format": "epoch_second"}, "body": {"type": "string", "analyzer": "english"}}}}}'
ic.create(index='2007', body=mapping)
sl_storage = object_storage.get_client('username', 'key', datacenter='sjc01')
def iterload(src):
buffer = ""
dec = json.JSONDecoder()
for chunk in src:
buffer = buffer + chunk
while (True):
try:
r = dec.raw_decode(buffer)
except:
break
yield r[0]
buffer = buffer[r[1]:].strip(" \n\r\t")
chunk_size = 512 * 1024
import object_storage
import datetime
sl_storage = object_storage.get_client('SLOS1011257-2:SL1011257','73b75a2cd3b2345903da72d33c6465a01c1dbf9e25acb635b79b6965ba67607c', datacenter='dal05')
sl_storage.containers()
sl_storage['foo'].properties
start = datetime.datetime.now()
print 'Started Upload'
end = datetime.datetime.now()
print start, end
read_start = datetime.datetime.now()
sl_storage['foo']['google-0.csv'].save_to_filename('google-0.csv')
read_end = datetime.datetime.now()
print read_start, read_end
