def _run(self, thread):
if time.time() - self.heartbeat >= 15:
self.heartbeat = time.time()
self._log_status("GETS")
device, partition, name, container_name = random.choice(self.names)
with self.connection() as conn:
try:
if self.use_proxy:
client.get_object(self.url, self.token, container_name, name, http_conn=conn)
else:
node = {"ip": self.ip, "port": self.port, "device": device}
direct_client.direct_get_object(node, partition, self.account, container_name, name)
except client.ClientException, e:
self.logger.debug(str(e))
self.failures += 1
def _open(self, name, mode='rb'):
headers, content = swiftclient.get_object(self.storage_url, self.token,
self.container_name, name,
http_conn=self.http_conn)
buf = StringIO(content)
buf.name = os.path.basename(name)
buf.mode = mode
return File(buf)
def _run(self, thread):
if time.time() - self.heartbeat >= 15:
self.heartbeat = time.time()
self._log_status('GETS')
device, partition, name, container_name = random.choice(self.names)
with self.connection() as conn:
try:
if self.use_proxy:
client.get_object(self.url, self.token,
container_name, name, http_conn=conn)
else:
node = {'ip': self.ip, 'port': self.port, 'device': device}
direct_client.direct_get_object(node, partition,
self.account, container_name, name)
except client.ClientException, e:
self.logger.debug(str(e))
self.failures += 1
def open_on_swift(name, mode="rb"):
headers, content = swiftclient.get_object(
PRE_AUTH_URL, PRE_AUTH_TOKEN, CONTAINER_NAME, name, http_conn=connect_swift()
)
buf = BytesIO(content)
buf.name = basename(name)
buf.mode = mode
return File(buf)
def agent():
keystone = client.Client(token=ca.creds['token'], tenant_id=ca.creds['tenantId'],
auth_url=ca.creds['identity_url'])
object_store_catalog = keystone.service_catalog.get_endpoints()['object-store']
region_endpoints = None
for endpoints in object_store_catalog:
if endpoints['region'] == ca.conf['region']:
region_endpoints = endpoints
if not region_endpoints:
ca.log_fail("Failing, region not found in endpoint list.")
exit()
t = Twython()
# Figure out what files already exist, and what our latest tweet is.
files = []
try:
(headers,files) = swiftclient.get_container(region_endpoints['publicURL'],ca.creds['token'],
ca.conf['container'],full_listing=True, prefix=ca.conf['path'])
except swiftclient.client.ClientException:
pass
files = sorted(files, key=itemgetter('name'))
last_tweet = 0
last_file = ''
tweet_list = []
if files:
(headers,last_file) = swiftclient.get_object(region_endpoints['publicURL'],ca.creds['token'],
ca.conf['container'],files[-1]['name'])
headers = swiftclient.head_object(region_endpoints['publicURL'],ca.creds['token'],
ca.conf['container'],files[-1]['name'])
last_tweet = headers.get('x-object-meta-last-tweet-id',0)
tweet_list = json.loads(last_file)
# Grab our tweet list (tweets since last tweet up to 3200), optimized for
# fewest requests.
try:
if last_tweet:
tweets = t.getUserTimeline(screen_name=ca.conf['screen_name'], count=200, since_id=last_tweet, include_rts=True)
else:
tweets = t.getUserTimeline(screen_name=ca.conf['screen_name'], count=200, include_rts=True)
except TwythonError, e:
ca.log_fail("Error accessing twitter stream. User not found or twitter down.")
exit()
def _open(self, name, mode='rb'):
headers, content = swiftclient.get_object(self.storage_url,
self.token,
self.container_name,
name,
http_conn=self.http_conn)
buf = StringIO(content)
buf.name = os.path.basename(name)
buf.mode = mode
return File(buf)
def get_object(self, container_name, object_name):
token = self.fetch_token()
url = self.auth['endpoint_url']
try:
response = get_object(url=url,
token=token,
container=container_name,
name=object_name)
except ClientException:
response = None
return response
def download(self, file):
object = files.get_object(self.auth.url,self.auth.token,self.container,file)
headers = object [0] # The first element in tuple is the headers
object = object[1] # File data.
encrypted_out = tempfile.NamedTemporaryFile()
encrypted_out.write(object) # Write file data to temp file.
encrypted_out.flush()
encrypted_out.seek(0)
return encrypted_out
def _open(self, name, mode='rb'):
original_name = name
name = self.name_prefix + name
headers, content = swiftclient.get_object(self.storage_url,
self.token,
self.container_name,
name,
http_conn=self.http_conn)
buf = BytesIO(content)
buf.name = os.path.basename(original_name)
buf.mode = mode
return File(buf)
def _open(self, name, mode='rb'):
original_name = name
name = self.name_prefix + name
headers, content = swiftclient.get_object(self.storage_url,
self.token,
self.container_name,
name,
http_conn=self.http_conn)
buf = BytesIO(content)
buf.name = os.path.basename(original_name)
buf.mode = mode
return File(buf)
def rescale(source,largethumb,smallthumb,token):
"""
Resize a given graphic from a source size to a smaller size.
"""
(headers,source_file) = swiftclient.get_object(source['url'],token,source['container'],source['name'])
large_image_file = StringIO.StringIO(source_file)
small_image_file = StringIO.StringIO(source_file)
if source['name'].endswith("jpg") or source['name'].endswith("JPG") or source['name'].endswith("jpeg") or source['name'].endswith("JPEG"):
type = "JPEG"
elif source['name'].endswith("png") or source['name'].endswith("PNG"):
type = "PNG"
large = Image.open(large_image_file)
small = Image.open(small_image_file)
# This chunk rotates images if they have exif data that indicates that they need rotation.
for orientation in ExifTags.TAGS.keys():
if ExifTags.TAGS[orientation]=='Orientation':
break
if hasattr(large, '_getexif'):
e = large._getexif()
if e is not None:
exif=dict(e.items())
orientation = exif[orientation]
if orientation == 3 :
large=large.transpose(Image.ROTATE_180)
small=small.transpose(Image.ROTATE_180)
elif orientation == 6 :
large=large.transpose(Image.ROTATE_270)
small=small.transpose(Image.ROTATE_270)
elif orientation == 8 :
large=large.transpose(Image.ROTATE_90)
small=small.transpose(Image.ROTATE_90)
large.thumbnail((800,800), Image.ANTIALIAS)
small.thumbnail((150,150), Image.ANTIALIAS)
largeoutput = StringIO.StringIO()
large.save(largeoutput, format=type)
largedata = largeoutput.getvalue()
largeoutput.close()
smalloutput = StringIO.StringIO()
small.save(smalloutput, format=type)
smalldata = smalloutput.getvalue()
smalloutput.close()
swiftclient.put_object(str(largethumb['url']+"/"+largethumb['container']+"/"+urllib2.quote(largethumb['name'])).encode('utf-8'),token=token,contents=largedata)
swiftclient.put_object(str(smallthumb['url']+"/"+smallthumb['container']+"/"+urllib2.quote(smallthumb['name'])).encode('utf-8'),token=token,contents=smalldata)
def agent():
ca.log("Starting!")
keystone = client.Client(token=ca.creds['token'],
tenant_id=ca.creds['tenantId'],
auth_url=ca.creds['identity_url'])
object_store_catalog = keystone.service_catalog.get_endpoints(
)['object-store']
region_endpoints = None
for endpoints in object_store_catalog:
if endpoints['region'] == ca.conf['region']:
region_endpoints = endpoints
if not region_endpoints:
ca.log_fail("Failing, region not found in endpoint list.")
exit()
if ca.conf.get('date'):
p = pdt.Calendar()
result = p.parse(ca.conf['date'])
dt = datetime.datetime.fromtimestamp(mktime(result[0]))
path = dt.strftime(ca.conf['name'])
else:
path = ca.conf['name']
try:
headers, contents = swiftclient.get_object(
region_endpoints['publicURL'], ca.creds['token'],
ca.conf['container'], path)
if headers['content-length'] >= 0:
ca.log("Emailing file!")
file_name = path.split("/")[-1]
ca.email(
path, '''
Here is the file '%s' from the container '%s'.
''' % (ca.conf['container'], path), None, [{
"file-name": file_name,
"contents": contents
}])
except swiftclient.client.ClientException, e:
ca.log("File doesn't exist!")
ca.email(
"File missing: " + ca.conf['container'] + "/" + path, '''
The container '%s' appears to be missing the file '%s'.
''' % (ca.conf['container'], path))
def _open(self, name, mode='rb'):
if self.name_prefix:
name = self.name_prefix + name
print("_OPEN METHOD")
print(self.storage_url)
print(self.token)
print(self.container_name)
print(name)
headers, content = swiftclient.get_object(self.storage_url, self.token,
self.container_name, name,
http_conn=self.http_conn)
# buf = BytesIO(content)
# print("::_OPEN %s" % sys.getsizeof(buf))
# buf.name = os.path.basename(name)
# print(buf.name)
# buf.mode = mode
# print("::END_OPEN")
return ContentFile(content)
def agent():
ca.log("Starting!")
keystone = client.Client(token=ca.creds['token'], tenant_id=ca.creds['tenantId'],
auth_url=ca.creds['identity_url'])
object_store_catalog = keystone.service_catalog.get_endpoints()['object-store']
region_endpoints = None
for endpoints in object_store_catalog:
if endpoints['region'] == ca.conf['region']:
region_endpoints = endpoints
if not region_endpoints:
ca.log_fail("Failing, region not found in endpoint list.")
exit()
if ca.conf.get('date'):
p = pdt.Calendar()
result = p.parse(ca.conf['date'])
dt = datetime.datetime.fromtimestamp(mktime(result[0]))
path = dt.strftime(ca.conf['name'])
else:
path = ca.conf['name']
try:
headers, contents = swiftclient.get_object(region_endpoints['publicURL'],ca.creds['token'],
ca.conf['container'],path)
if headers['content-length'] >= 0:
ca.log("Emailing file!")
file_name = path.split("/")[-1]
ca.email(path,'''
Here is the file '%s' from the container '%s'.
''' % (ca.conf['container'], path), None, [{"file-name":file_name,"contents":contents}])
except swiftclient.client.ClientException, e:
ca.log("File doesn't exist!")
ca.email("File missing: "+ca.conf['container']+"/"+path,'''
The container '%s' appears to be missing the file '%s'.
''' % (ca.conf['container'], path))
def page_main(self, req, storage_url, token):
""" main page container list or object list """
path = urlparse(self.del_prefix(req.url)).path
if len(path.split('/')) <= 2:
path = urlparse(storage_url).path
vrs, acc, cont, obj = split_path(path, 1, 4, True)
path_type = len([i for i in [vrs, acc, cont, obj] if i])
if path_type == 2: # account
return self.page_cont_list(req, storage_url, token)
if path_type == 3: # container
return self.page_obj_list(req, storage_url, token)
if path_type == 4: # object
try:
(obj_status, objct) = get_object(storage_url, token, cont, obj)
except ClientException, e:
resp = Response(charset='utf8')
resp.status = e.http_status
return resp
except err:
pass
def page_main(self, req, storage_url, token):
""" main page container list or object list """
path = urlparse(self.del_prefix(req.url)).path
if len(path.split('/')) <= 2:
path = urlparse(storage_url).path
vrs, acc, cont, obj = split_path(path, 1, 4, True)
path_type = len([i for i in [vrs, acc, cont, obj] if i])
if path_type == 2: # account
return self.page_cont_list(req, storage_url, token)
if path_type == 3: # container
return self.page_obj_list(req, storage_url, token)
if path_type == 4: # object
try:
(obj_status, objct) = get_object(storage_url, token, cont, obj)
except ClientException, e:
resp = Response(charset='utf8')
resp.status = e.http_status
return resp
except err:
pass
except IOError, err:
pass
self.logger.debug('Upload obj size: %s' % obj_size)
try:
put_object(storage_url,
token,
cont,
obj,
obj_fp,
content_length=obj_size)
except ClientException, err:
return err.http_status
return HTTP_CREATED
return HTTP_BAD_REQUEST
if action == 'obj_get':
(obj_status, hunk) = get_object(storage_url, token, cont, obj)
#resp.headerlist = obj_status.items()
#resp.body_file = hunk
if action == 'obj_delete':
try:
delete_object(storage_url, token, cont, obj)
except ClientException, err:
return err.http_status
return HTTP_NO_CONTENT
if action == 'obj_metadata':
if meta_headers:
try:
headers = head_object(storage_url, token, cont, obj)
except ClientException, err:
return err.http_status
headers = self.get_current_meta(headers)
elif json_data['agent_url'][0:5] == 'swift':
(method,trash,region,container,object_name) = json_data['agent_url'].split("/",4)
keystone = client.Client(token=json_data['token'], tenant_id=json_data['tenantId'], auth_url=json_data['identity_url'])
object_store_catalog = keystone.service_catalog.get_endpoints()['object-store']
region_endpoints = None
for endpoints in object_store_catalog:
if endpoints['region'] == region:
region_endpoints = endpoints
if not region_endpoints:
exit("Failing, region not found in endpoint list.")
try:
file = open(json_data['filename'],"w")
headers, body = swiftclient.get_object(region_endpoints['publicURL'],json_data['token'],container,object_name,resp_chunk_size=65000)
for chunk in body:
file.write(chunk)
file.close()
print "Success."
sys.exit(0)
except swiftclient.client.ClientException, err:
sys.exit("Failed to download agent file: "+err)
sys.exit("Failure.")
def rescale(source, largethumb, smallthumb, token):
"""
Resize a given graphic from a source size to a smaller size.
"""
(headers, source_file) = swiftclient.get_object(source['url'], token,
source['container'],
source['name'])
large_image_file = StringIO.StringIO(source_file)
small_image_file = StringIO.StringIO(source_file)
if source['name'].endswith("jpg") or source['name'].endswith(
"JPG") or source['name'].endswith(
"jpeg") or source['name'].endswith("JPEG"):
type = "JPEG"
elif source['name'].endswith("png") or source['name'].endswith("PNG"):
type = "PNG"
large = Image.open(large_image_file)
small = Image.open(small_image_file)
# This chunk rotates images if they have exif data that indicates that they need rotation.
for orientation in ExifTags.TAGS.keys():
if ExifTags.TAGS[orientation] == 'Orientation':
break
if hasattr(large, '_getexif'):
e = large._getexif()
if e is not None:
exif = dict(e.items())
orientation = exif[orientation]
if orientation == 3:
large = large.transpose(Image.ROTATE_180)
small = small.transpose(Image.ROTATE_180)
elif orientation == 6:
large = large.transpose(Image.ROTATE_270)
small = small.transpose(Image.ROTATE_270)
elif orientation == 8:
large = large.transpose(Image.ROTATE_90)
small = small.transpose(Image.ROTATE_90)
large.thumbnail((800, 800), Image.ANTIALIAS)
small.thumbnail((150, 150), Image.ANTIALIAS)
largeoutput = StringIO.StringIO()
large.save(largeoutput, format=type)
largedata = largeoutput.getvalue()
largeoutput.close()
smalloutput = StringIO.StringIO()
small.save(smalloutput, format=type)
smalldata = smalloutput.getvalue()
smalloutput.close()
swiftclient.put_object(
str(largethumb['url'] + "/" + largethumb['container'] + "/" +
urllib2.quote(largethumb['name'])).encode('utf-8'),
token=token,
contents=largedata)
swiftclient.put_object(
str(smallthumb['url'] + "/" + smallthumb['container'] + "/" +
urllib2.quote(smallthumb['name'])).encode('utf-8'),
token=token,
contents=smalldata)
headers = head_object(storage_url, token, cont, obj)
except ClientException, err:
return err.http_status, headers
return HTTP_OK, headers
if action == 'obj_create':
if obj:
if len(obj) > 1024:
return HTTP_PRECONDITION_FAILED
try:
put_object(storage_url, token, cont, obj, obj_fp)
except ClientException, err:
return err.http_status
return HTTP_CREATED
return HTTP_BAD_REQUEST
if action == 'obj_get':
(obj_status, hunk) = get_object(storage_url, token, cont, obj)
#resp.headerlist = obj_status.items()
#resp.body_file = hunk
if action == 'obj_delete':
try:
delete_object(storage_url, token, cont, obj)
except ClientException, err:
return err.http_status
return HTTP_NO_CONTENT
if action == 'obj_metadata':
if meta_headers:
try:
headers = head_object(storage_url, token, cont, obj)
except ClientException, err:
return err.http_status
headers = self.get_current_meta(headers)
def get_obj(self, container_name, obj_name): obj_data = SWIFT.get_object(self.swift_url, self.auth_token, container_name, obj_name) return obj_data
auth_url=json_data['identity_url'])
object_store_catalog = keystone.service_catalog.get_endpoints(
)['object-store']
region_endpoints = None
for endpoints in object_store_catalog:
if endpoints['region'] == region:
region_endpoints = endpoints
if not region_endpoints:
exit("Failing, region not found in endpoint list.")
try:
file = open(json_data['filename'], "w")
headers, body = swiftclient.get_object(region_endpoints['publicURL'],
json_data['token'],
container,
object_name,
resp_chunk_size=65000)
for chunk in body:
file.write(chunk)
file.close()
print "Success."
sys.exit(0)
except swiftclient.client.ClientException, err:
sys.exit("Failed to download agent file: " + err)
sys.exit("Failure.")
