def deprovision(instance_id):
"""
Deprovision an existing instance of this service
DELETE /v2/service_instances/<instance_id>:
<instance_id> is the Cloud Controller provided
value used to provision the instance
return:
As of API 2.3, an empty JSON document
is expected
"""
global subscription_id
global cert
global account_name
global account_key
if account_name and account_key:
blob_service = BlobService(account_name, account_key)
container_name = '{0}-{1}'.format(CONTAINER_NAME_PREFIX, instance_id)
blob_service.delete_container(container_name)
if account_name.startswith(STORAGE_ACCOUNT_NAME_PREFIX):
sms = ServiceManagementService(subscription_id, cert_file)
sms.delete_storage_account(account_name)
return jsonify({})
def delete(self):
properties = self.__SMS.get_deployment_by_name(self.name, self.name)
media_link = properties.role_list.roles[0].os_virtual_hard_disk.media_link
storage_name = media_link[media_link.find("//") + 2:media_link.find(".blob")]
from Azure.AzureVolumes.AzureVolumes import AzureVolumescls
volume_service = AzureVolumescls(credentials=self._credentials)
volumes = volume_service.list_volumes()
volume_to_be_deleted = None
for volume in volumes:
if volume.instance_id == self.name:
volume_to_be_deleted = volume
break
self.__SMS.delete_deployment(self.name, self.name)
self.__SMS.delete_hosted_service(self.name)
volume_to_be_deleted.delete()
# delete image from storge
from azure.storage import BlobService
keys = self.__SMS.get_storage_account_keys(storage_name)
blob_service = BlobService(account_name=storage_name, account_key=keys.storage_service_keys.primary)
blob_service.delete_container(self.name, fail_not_exist=True)
def test_azure_call(request):
import os
try:
from azure.storage import BlobService
bs = BlobService(os.environ["AZURE_STORAGE_ACCOUNT"], os.environ["AZURE_STORAGE_ACCESS_KEY"])
import random
container_name = hex(int(random.random() * 1000000000))
bs.create_container(container_name)
bs.put_blob(container_name, 'testblob', 'hello world\n', 'BlockBlob')
blob = bs.get_blob(container_name, 'testblob')
if blob != 'hello world\n':
return HttpResponse("Failed!", status = '404')
bs.delete_blob(container_name, 'testblob')
bs.delete_container(container_name)
return HttpResponse("Succeeded!")
except:
try:
import traceback
return HttpResponse(traceback.format_exc() + str(os.environ.keys()))
except:
import traceback
return HttpResponse(traceback.format_exc())
def do_step(context):
settings = context.meta['settings']
index_file = context.meta['index-file']
pivnetAPIToken = settings["pivnet-api-token"]
f = open("manifests/{0}".format(index_file))
manifests = yaml.safe_load(f)
f.close()
eula_urls = [
"https://network.pivotal.io/api/v2/products/{0}/releases/{1}/eula_acceptance".format(
m['release-name'],
m['release-number']) for m in manifests['manifests']]
release_urls = [
"https://network.pivotal.io/api/v2/products/{0}/releases/{1}/product_files/{2}/download".format(
m['release-name'],
m['release-number'],
m['file-number']) for m in manifests['manifests']]
stemcell_urls = [m['stemcell'] for m in manifests['manifests']]
# accept eula for each product
for url in eula_urls:
print url
if not "concourse" in url:
res = authorizedPost(url, pivnetAPIToken)
code = res.getcode()
# releases
is_release_file = re.compile("^releases\/.+")
if not os.path.exists("/tmp/releases"):
os.makedirs("/tmp/releases")
client = bosh_client.BoshClient("https://10.0.0.4:25555", "admin", "admin")
storage_account_name = settings["STORAGE-ACCOUNT-NAME"]
storage_access_key = settings["STORAGE-ACCESS-KEY"]
blob_service = BlobService(storage_account_name, storage_access_key)
blob_service.create_container(
container_name='tempreleases',
x_ms_blob_public_access='container')
print "Processing releases."
for url in release_urls:
print "Downloading {0}.".format(url)
if "concourse" in url:
release_url = "https://s3-us-west-2.amazonaws.com/bosh-azure-releases/concourse.zip"
res = urllib2.urlopen(release_url)
else:
res = authorizedPost(url, pivnetAPIToken)
code = res.getcode()
length = int(res.headers["Content-Length"])
# content-length
if code is 200:
total = 0
pcent = 0.0
CHUNK = 16 * 1024
with tempfile.TemporaryFile() as temp:
while True:
chunk = res.read(CHUNK)
total += CHUNK
pcent = (float(total) / float(length)) * 100
sys.stdout.write(
"Download progress: %.2f%% (%.2fM)\r" %
(pcent, total / 1000000.0))
sys.stdout.flush()
if not chunk:
break
temp.write(chunk)
print "Download complete."
z = zipfile.ZipFile(temp)
for name in z.namelist():
# is this a release?
if is_release_file.match(name):
release_filename = "/tmp/{0}".format(name)
print "Unpacking {0}.".format(name)
z.extract(name, "/tmp")
print "Uploading {0} to Azure blob store".format(name)
blob_service.put_block_blob_from_path(
'tempreleases',
name,
"/tmp/{0}".format(name),
x_ms_blob_content_type='application/x-compressed'
)
os.unlink(release_filename)
blob_url = "http://{0}.blob.core.windows.net/{1}/{2}".format(
storage_account_name, 'tempreleases', name)
print "Uploading release {0} to BOSH director.".format(name)
task_id = client.upload_release(blob_url)
client.wait_for_task(task_id)
z.close()
temp.close()
blob_service.delete_container("tempreleases")
# stemcells
print "Processing stemcells."
for url in stemcell_urls:
print "Processing stemcell {0}".format(url)
task_id = client.upload_stemcell(url)
client.wait_for_task(task_id)
return context
class AzureFS(LoggingMixIn, Operations):
"""Azure Blob Storage filesystem"""
blobs = None
containers = dict() # <cname, dict(stat:dict,
#files:None|dict<fname, stat>)
fds = dict() # <fd, (path, bytes, dirty)>
fd = 0
def __init__(self, account, key):
self.blobs = BlobService(account, key)
self.rebuild_container_list()
def convert_to_epoch(self, date):
"""Converts Tue, 31 Jul 2012 07:17:34 GMT format to epoch"""
return int(time.mktime(time.strptime(date, TIME_FORMAT)))
def rebuild_container_list(self):
cmap = dict()
cnames = set()
for c in self.blobs.list_containers():
date = c.properties.last_modified
cstat = dict(st_mode=(S_IFDIR | 0755), st_uid=getuid(), st_size=0,
st_mtime=self.convert_to_epoch(date))
cname = c.name
cmap['/' + cname] = dict(stat=cstat, files=None)
cnames.add(cname)
cmap['/'] = dict(files={},
stat=dict(st_mode=(S_IFDIR | 0755),
st_uid=getuid(), st_size=0,
st_mtime=int(time.time())))
self.containers = cmap # destroys fs tree cache resistant to misses
def _parse_path(self, path): # returns </dir, file(=None)>
if path.count('/') > 1: # file
return str(path[:path.rfind('/')]), str(path[path.rfind('/') + 1:])
else: # dir
pos = path.rfind('/', 1)
if pos == -1:
return path, None
else:
return str(path[:pos]), None
def parse_container(self, path):
base_container = path[1:] # /abc/def/g --> abc
if base_container.find('/') > -1:
base_container = base_container[:base_container.find('/')]
return str(base_container)
def _get_dir(self, path, contents_required=False):
if not self.containers:
self.rebuild_container_list()
if path in self.containers and not (contents_required and \
self.containers[path]['files'] is None):
return self.containers[path]
cname = self.parse_container(path)
if '/' + cname not in self.containers:
raise FuseOSError(ENOENT)
else:
if self.containers['/' + cname]['files'] is None:
# fetch contents of container
log.info("------> CONTENTS NOT FOUND: %s" % cname)
blobs = self.blobs.list_blobs(cname)
dirstat = dict(st_mode=(S_IFDIR | 0755), st_size=0,
st_uid=getuid(), st_mtime=time.time())
if self.containers['/' + cname]['files'] is None:
self.containers['/' + cname]['files'] = dict()
for f in blobs:
blob_name = f.name
blob_date = f.properties.last_modified
blob_size = long(f.properties.content_length)
node = dict(st_mode=(S_IFREG | 0644), st_size=blob_size,
st_mtime=self.convert_to_epoch(blob_date),
st_uid=getuid())
if blob_name.find('/') == -1: # file just under container
self.containers['/' + cname]['files'][blob_name] = node
return self.containers['/' + cname]
return None
def _get_file(self, path):
d, f = self._parse_path(path)
dir = self._get_dir(d, True)
if dir is not None and f in dir['files']:
return dir['files'][f]
def getattr(self, path, fh=None):
d, f = self._parse_path(path)
if f is None:
dir = self._get_dir(d)
return dir['stat']
else:
file = self._get_file(path)
if file:
return file
raise FuseOSError(ENOENT)
# FUSE
def mkdir(self, path, mode):
if path.count('/') <= 1: # create on root
name = path[1:]
if not 3 <= len(name) <= 63:
log.error("Container names can be 3 through 63 chars long.")
raise FuseOSError(ENAMETOOLONG)
if name is not name.lower():
log.error("Container names cannot contain uppercase \
characters.")
raise FuseOSError(EACCES)
if name.count('--') > 0:
log.error('Container names cannot contain consecutive \
dashes (-).')
raise FuseOSError(EAGAIN)
#TODO handle all "-"s must be preceded by letter or numbers
#TODO starts with only letter or number, can contain letter, nr,'-'
resp = self.blobs.create_container(name)
if resp:
self.rebuild_container_list()
log.info("CONTAINER %s CREATED" % name)
else:
raise FuseOSError(EACCES)
log.error("Invalid container name or container already \
exists.")
else:
raise FuseOSError(ENOSYS) # TODO support 2nd+ level mkdirs
def rmdir(self, path):
if path.count('/') == 1:
c_name = path[1:]
resp = self.blobs.delete_container(c_name)
if resp:
if path in self.containers:
del self.containers[path]
else:
raise FuseOSError(EACCES)
else:
raise FuseOSError(ENOSYS) # TODO support 2nd+ level mkdirs
def create(self, path, mode):
node = dict(st_mode=(S_IFREG | mode), st_size=0, st_nlink=1,
st_uid=getuid(), st_mtime=time.time())
d, f = self._parse_path(path)
if not f:
log.error("Cannot create files on root level: /")
raise FuseOSError(ENOSYS)
dir = self._get_dir(d, True)
if not dir:
raise FuseOSError(EIO)
dir['files'][f] = node
return self.open(path, data='') # reusing handler provider
def open(self, path, flags=0, data=None):
if data == None: # download contents
c_name = self.parse_container(path)
f_name = path[path.find('/', 1) + 1:]
try:
data = self.blobs.get_blob(c_name, f_name)
except WindowsAzureMissingResourceError:
dir = self._get_dir('/' + c_name, True)
if f_name in dir['files']:
del dir['files'][f_name]
raise FuseOSError(ENOENT)
except WindowsAzureError as e:
log.error("Read blob failed HTTP %d" % e.code)
raise FuseOSError(EAGAIN)
self.fd += 1
self.fds[self.fd] = (path, data, False)
return self.fd
def flush(self, path, fh=None):
if not fh:
raise FuseOSError(EIO)
else:
if fh not in self.fds:
raise FuseOSError(EIO)
path = self.fds[fh][0]
data = self.fds[fh][1]
dirty = self.fds[fh][2]
if not dirty:
return 0 # avoid redundant write
d, f = self._parse_path(path)
c_name = self.parse_container(path)
if data is None:
data = ''
try:
if len(data) < 64 * 1024 * 1024: # 64 mb
self.blobs.put_blob(c_name, f, data, 'BlockBlob')
else:
# divide file by blocks and upload
block_size = 8 * 1024 * 1024
num_blocks = int(math.ceil(len(data) * 1.0 / block_size))
rd = str(random.randint(1, 1e8))
block_ids = list()
for i in range(num_blocks):
part = data[i * block_size:min((i + 1) * block_size,
len(data))]
block_id = base64.encodestring('%s_%s' % (rd,
(8 - len(str(i))) * '0' + str(i)))
self.blobs.put_block(c_name, f, part, block_id)
block_ids.append(block_id)
self.blobs.put_block_list(c_name, f, block_ids)
except WindowsAzureError:
raise FuseOSError(EAGAIN)
dir = self._get_dir(d, True)
if not dir or f not in dir['files']:
raise FuseOSError(EIO)
# update local data
dir['files'][f]['st_size'] = len(data)
dir['files'][f]['st_mtime'] = time.time()
self.fds[fh] = (path, data, False) # mark as not dirty
return 0
def release(self, path, fh=None):
if fh is not None and fh in self.fds:
del self.fds[fh]
def truncate(self, path, length, fh=None):
return 0 # assume done, no need
def write(self, path, data, offset, fh=None):
if not fh or fh not in self.fds:
raise FuseOSError(ENOENT)
else:
d = self.fds[fh][1]
if d is None:
d = ""
self.fds[fh] = (self.fds[fh][0], d[:offset] + data, True)
return len(data)
def unlink(self, path):
c_name = self.parse_container(path)
d, f = self._parse_path(path)
try:
self.blobs.delete_blob(c_name, f)
_dir = self._get_dir(path, True)
if _dir and f in _dir['files']:
del _dir['files'][f]
return 0
except WindowsAzureMissingResourceError:
raise FuseOSError(ENOENT)
except Exception as e:
raise FuseOSError(EAGAIN)
def readdir(self, path, fh):
if path == '/':
return ['.', '..'] + [x[1:] for x in self.containers.keys() \
if x is not '/']
dir = self._get_dir(path, True)
if not dir:
raise FuseOSError(ENOENT)
return ['.', '..'] + dir['files'].keys()
def read(self, path, size, offset, fh):
if not fh or fh not in self.fds:
raise FuseOSError(ENOENT)
f_name = path[path.find('/', 1) + 1:]
c_name = path[1:path.find('/', 1)]
try:
data = self.blobs.get_blob(c_name, f_name)
self.fds[fh] = (self.fds[fh][0], data, False)
return data[offset:offset + size]
except URLError, e:
if e.code == 404:
raise FuseOSError(ENOENT)
elif e.code == 403:
raise FUSEOSError(EPERM)
else:
log.error("Read blob failed HTTP %d" % e.code)
raise FuseOSError(EAGAIN)
data = self.fds[fh][1]
if data is None:
data = ""
return data[offset:offset + size]
def apathetic_container_delete(container_name, *args, **kwargs):
conn = BlobService(*args, **kwargs)
conn.delete_container(container_name)
return conn
blob_service = BlobService(account_name, account_key)
for container in blob_service.list_containers():
c = container.name
if c == "heartbeat": continue
if options.date and not ( c == "processed-"+options.date ): continue
if debug: sys.stderr.write("Processing container: "+str(c)+"\n")
for b in blob_service.list_blobs(c):
if debug: sys.stderr.write("Processing blob: "+str(b.name)+"\n")
data = blob_service.get_blob(c, b.name)
cs = StringIO.StringIO(data)
gzipstream = gzip.GzipFile(fileobj=cs)
if output_format == "txt":
print gzipstream.read()
elif output_format == "json":
d = {}
i = 0
ds = gzipstream.read()
# some DCU entries contains more than 28 values (outside the
# definition of the headers)
for x in ds.strip().split("\t")[:27]:
d[headers[i]] = x
i=i+1
print (json.dumps(d, sort_keys=True))
if options.clear:
if debug: sys.stderr.write("Deleting blob: "+str(b.name)+"\n")
blob_service.delete_blob(c, b.name)
if options.clear:
if debug: sys.stderr.write("Deleting container: "+str(c)+"\n")
blob_service.delete_container(c)
def do_step(context):
settings = context.meta['settings']
index_file = context.meta['index-file']
pivnetAPIToken = settings["pivnet-api-token"]
f = open("manifests/{0}".format(index_file))
manifests = yaml.safe_load(f)
f.close()
eula_urls = [
"https://network.pivotal.io/api/v2/products/{0}/releases/{1}/eula_acceptance"
.format(m['release-name'], m['release-number'])
for m in manifests['manifests']
]
release_urls = [
"https://network.pivotal.io/api/v2/products/{0}/releases/{1}/product_files/{2}/download"
.format(m['release-name'], m['release-number'], m['file-number'])
for m in manifests['manifests']
]
stemcell_urls = [m['stemcell'] for m in manifests['manifests']]
# accept eula for each product
for url in eula_urls:
print url
if not "concourse" in url:
res = authorizedPost(url, pivnetAPIToken)
code = res.getcode()
# releases
is_release_file = re.compile("^releases\/.+")
if not os.path.exists("/tmp/releases"):
os.makedirs("/tmp/releases")
client = bosh_client.BoshClient("https://10.0.0.4:25555", "admin", "admin")
storage_account_name = settings["STORAGE-ACCOUNT-NAME"]
storage_access_key = settings["STORAGE-ACCESS-KEY"]
blob_service = BlobService(storage_account_name, storage_access_key)
blob_service.create_container(container_name='tempreleases',
x_ms_blob_public_access='container')
print "Processing releases."
for url in release_urls:
print "Downloading {0}.".format(url)
if "concourse" in url:
release_url = "https://s3-us-west-2.amazonaws.com/bosh-azure-releases/concourse.zip"
res = urllib2.urlopen(release_url)
else:
res = authorizedPost(url, pivnetAPIToken)
code = res.getcode()
length = int(res.headers["Content-Length"])
# content-length
if code is 200:
total = 0
pcent = 0.0
CHUNK = 16 * 1024
with tempfile.TemporaryFile() as temp:
while True:
chunk = res.read(CHUNK)
total += CHUNK
pcent = (float(total) / float(length)) * 100
sys.stdout.write("Download progress: %.2f%% (%.2fM)\r" %
(pcent, total / 1000000.0))
sys.stdout.flush()
if not chunk:
break
temp.write(chunk)
print "Download complete."
z = zipfile.ZipFile(temp)
for name in z.namelist():
# is this a release?
if is_release_file.match(name):
release_filename = "/tmp/{0}".format(name)
print "Unpacking {0}.".format(name)
z.extract(name, "/tmp")
print "Uploading {0} to Azure blob store".format(name)
blob_service.put_block_blob_from_path(
'tempreleases',
name,
"/tmp/{0}".format(name),
x_ms_blob_content_type='application/x-compressed')
os.unlink(release_filename)
blob_url = "http://{0}.blob.core.windows.net/{1}/{2}".format(
storage_account_name, 'tempreleases', name)
print "Uploading release {0} to BOSH director.".format(
name)
task_id = client.upload_release(blob_url)
client.wait_for_task(task_id)
z.close()
temp.close()
blob_service.delete_container("tempreleases")
# stemcells
print "Processing stemcells."
for url in stemcell_urls:
print "Processing stemcell {0}".format(url)
task_id = client.upload_stemcell(url)
client.wait_for_task(task_id)
return context
def delete_blobs():
print "deleting blobs: " + ", ".join(BLOB_CONTAINERS)
for i in range(len(ACCOUNT)):
bs = BlobService(ACCOUNT[i], KEY[i])
for container in BLOB_CONTAINERS:
bs.delete_container(container)