def _to_volume(self, volume):
extra = {
'organization': volume['organization'],
'volume_type': volume['volume_type'],
'creation_date': parse_date(volume['creation_date']),
'modification_date': parse_date(volume['modification_date']),
}
return StorageVolume(id=volume['id'],
name=volume['name'],
size=_to_lib_size(volume['size']),
driver=self,
extra=extra)
def _to_image(self, image):
extra = {
'arch': image['arch'],
'size': _to_lib_size(image.get('root_volume', {}).get('size', 0))
or 50,
'creation_date': parse_date(image['creation_date']),
'modification_date': parse_date(image['modification_date']),
'organization': image['organization'],
}
return NodeImage(id=image['id'],
name=image['name'],
driver=self,
extra=extra)
def _to_node(self, data):
extra_keys = ["cores", "power", "memory", "current_price", "relations"]
extra = self._extract_values_to_dict(data=data, keys=extra_keys)
ips = []
for diction in data["relations"]["public_ips"]:
ips.append(diction["ip"])
state = ""
if data["power"] is True:
state = NodeState.RUNNING
else:
state = NodeState.STOPPED
node = Node(
id=data["object_uuid"],
name=data["name"],
state=state,
public_ips=ips,
created_at=parse_date(data["create_time"]),
private_ips=None,
driver=self.connection.driver,
extra=extra,
)
return node
def _authenticate_2_0_with_body(self, reqbody):
resp = self.request('/v2.0/tokens',
data=reqbody,
headers={'Content-Type': 'application/json'},
method='POST')
if resp.status == httplib.UNAUTHORIZED:
raise InvalidCredsError()
elif resp.status not in [
httplib.OK, httplib.NON_AUTHORITATIVE_INFORMATION
]:
body = 'code: %s body: %s' % (resp.status, resp.body)
raise MalformedResponseError('Malformed response',
body=body,
driver=self.driver)
else:
body = resp.object
try:
access = body['access']
expires = access['token']['expires']
self.auth_token = access['token']['id']
self.auth_token_expires = parse_date(expires)
self.urls = access['serviceCatalog']
self.auth_user_info = access.get('user', {})
except KeyError:
e = sys.exc_info()[1]
raise MalformedResponseError(
'Auth JSON response is \
missing required elements', e)
return self
def _to_snapshot(self, api_node):
if "snapshot" in api_node:
api_node = api_node["snapshot"]
extra = {
"volume_id": api_node["volumeId"],
"name": api_node["displayName"],
"created": api_node["createdAt"],
"description": api_node["displayDescription"],
"status": api_node["status"],
}
state = self.SNAPSHOT_STATE_MAP.get(
api_node["status"], VolumeSnapshotState.UNKNOWN
)
try:
created_td = parse_date(api_node["createdAt"])
except ValueError:
created_td = None
snapshot = VolumeSnapshot(
id=api_node["id"],
driver=self,
size=api_node["size"],
extra=extra,
created=created_td,
state=state,
name=api_node["displayName"],
)
return snapshot
def authenticate_2_0_with_body(self, reqbody):
resp = self.request('/v2.0/tokens', data=reqbody,
headers={'Content-Type': 'application/json'},
method='POST')
if resp.status == httplib.UNAUTHORIZED:
raise InvalidCredsError()
elif resp.status not in [httplib.OK,
httplib.NON_AUTHORITATIVE_INFORMATION]:
body = 'code: %s body: %s' % (resp.status, resp.body)
raise MalformedResponseError('Malformed response', body=body,
driver=self.driver)
else:
try:
body = json.loads(resp.body)
except Exception:
e = sys.exc_info()[1]
raise MalformedResponseError('Failed to parse JSON', e)
try:
access = body['access']
expires = access['token']['expires']
self.auth_token = access['token']['id']
self.auth_token_expires = parse_date(expires)
self.urls = access['serviceCatalog']
self.auth_user_info = access.get('user', {})
except KeyError:
e = sys.exc_info()[1]
raise MalformedResponseError('Auth JSON response is \
missing required elements', e)
return self
def _to_node(self, data):
extra_keys = ['memory', 'vcpus', 'disk', 'region', 'image',
'size_slug', 'locked', 'created_at', 'networks',
'kernel', 'backup_ids', 'snapshot_ids', 'features']
if 'status' in data:
state = self.NODE_STATE_MAP.get(data['status'], NodeState.UNKNOWN)
else:
state = NodeState.UNKNOWN
created = parse_date(data['created_at'])
networks = data['networks']
private_ips = []
public_ips = []
if networks:
for net in networks['v4']:
if net['type'] == 'private':
private_ips = [net['ip_address']]
if net['type'] == 'public':
public_ips = [net['ip_address']]
extra = {}
for key in extra_keys:
if key in data:
extra[key] = data[key]
node = Node(id=data['id'], name=data['name'], state=state,
public_ips=public_ips, private_ips=private_ips,
created_at=created, driver=self, extra=extra)
return node
def authenticate_1_1(self):
reqbody = json.dumps({'credentials': {'username': self.user_id,
'key': self.key}})
resp = self.request("/v1.1/auth",
data=reqbody,
headers={},
method='POST')
if resp.status == httplib.UNAUTHORIZED:
# HTTP UNAUTHORIZED (401): auth failed
raise InvalidCredsError()
elif resp.status != httplib.OK:
raise MalformedResponseError('Malformed response',
body='code: %s body:%s' % (resp.status, resp.body),
driver=self.driver)
else:
try:
body = json.loads(resp.body)
except Exception:
e = sys.exc_info()[1]
raise MalformedResponseError('Failed to parse JSON', e)
try:
expires = body['auth']['token']['expires']
self.auth_token = body['auth']['token']['id']
self.auth_token_expires = parse_date(expires)
self.urls = body['auth']['serviceCatalog']
self.auth_user_info = None
except KeyError:
e = sys.exc_info()[1]
raise MalformedResponseError('Auth JSON response is \
missing required elements', e)
return self
def _to_snapshot(self, api_node):
if 'snapshot' in api_node:
api_node = api_node['snapshot']
extra = {'volume_id': api_node['volumeId'],
'name': api_node['displayName'],
'created': api_node['createdAt'],
'description': api_node['displayDescription'],
'status': api_node['status']}
state = self.SNAPSHOT_STATE_MAP.get(
api_node['status'],
VolumeSnapshotState.UNKNOWN
)
try:
created_td = parse_date(api_node['createdAt'])
except ValueError:
created_td = None
snapshot = VolumeSnapshot(id=api_node['id'], driver=self,
size=api_node['size'],
extra=extra,
created=created_td,
state=state)
return snapshot
def _to_node(self, data):
extra_keys = ['cores', 'power', 'memory', 'current_price', 'relations']
extra = self._extract_values_to_dict(data=data, keys=extra_keys)
ips = []
for diction in data['relations']['public_ips']:
ips.append(diction['ip'])
state = ''
if data['power'] is True:
state = NodeState.RUNNING
else:
state = NodeState.STOPPED
node = Node(id=data['object_uuid'],
name=data['name'],
state=state,
public_ips=ips,
created_at=parse_date(data['create_time']),
private_ips=None,
driver=self.connection.driver,
extra=extra)
return node
def authenticate_1_1(self):
reqbody = json.dumps({'credentials': {'username': self.user_id,
'key': self.key}})
resp = self.request('/v1.1/auth', data=reqbody, headers={},
method='POST')
if resp.status == httplib.UNAUTHORIZED:
# HTTP UNAUTHORIZED (401): auth failed
raise InvalidCredsError()
elif resp.status != httplib.OK:
body = 'code: %s body:%s' % (resp.status, resp.body)
raise MalformedResponseError('Malformed response', body=body,
driver=self.driver)
else:
try:
body = json.loads(resp.body)
except Exception:
e = sys.exc_info()[1]
raise MalformedResponseError('Failed to parse JSON', e)
try:
expires = body['auth']['token']['expires']
self.auth_token = body['auth']['token']['id']
self.auth_token_expires = parse_date(expires)
self.urls = body['auth']['serviceCatalog']
self.auth_user_info = None
except KeyError:
e = sys.exc_info()[1]
raise MalformedResponseError('Auth JSON response is \
missing required elements', e)
return self
def _to_snapshot(self, api_node):
if 'snapshot' in api_node:
api_node = api_node['snapshot']
extra = {
'volume_id': api_node['volumeId'],
'name': api_node['displayName'],
'created': api_node['createdAt'],
'description': api_node['displayDescription'],
'status': api_node['status']
}
state = self.SNAPSHOT_STATE_MAP.get(api_node['status'],
VolumeSnapshotState.UNKNOWN)
try:
created_td = parse_date(api_node['createdAt'])
except ValueError:
created_td = None
snapshot = VolumeSnapshot(id=api_node['id'],
driver=self,
size=api_node['size'],
extra=extra,
created=created_td,
state=state)
return snapshot
def _to_recovery_point(self, item, target):
id = item.id
date = parse_date(item.creationTimestamp)
point = BackupTargetRecoveryPoint(
id=id,
date=date,
target=target,
driver=self.connection.driver,
extra={"snapshot-id": id},
)
return point
def _to_node(self, data):
extra_keys = ['memory', 'vcpus', 'disk', 'image', 'size',
'size_slug', 'locked', 'created_at', 'networks',
'kernel', 'backup_ids', 'snapshot_ids', 'features',
'tags']
if 'status' in data:
state = self.NODE_STATE_MAP.get(data['status'], NodeState.UNKNOWN)
else:
state = NodeState.UNKNOWN
created = parse_date(data['created_at'])
networks = data['networks']
private_ips = []
public_ips = []
if networks:
for net in networks['v4']:
if net['type'] == 'private':
private_ips = [net['ip_address']]
if net['type'] == 'public':
public_ips = [net['ip_address']]
extra = {}
for key in extra_keys:
if key in data:
extra[key] = data[key]
extra['region'] = data.get('region', {}).get('name')
# Untouched extra values, backwards compatibility
resolve_data = data.get('image')
if resolve_data:
image = self._to_image(resolve_data)
else:
image = None
resolve_data = extra.get('size')
if resolve_data:
size = self._to_size(resolve_data)
else:
size = None
node = Node(
id=data['id'],
name=data['name'],
state=state,
image=image,
size=size,
public_ips=public_ips,
private_ips=private_ips,
created_at=created,
driver=self,
extra=extra)
return node
def _to_recovery_point(self, item, target):
id = item.id
date = parse_date(item.creationTimestamp)
point = BackupTargetRecoveryPoint(
id=id,
date=date,
target=target,
driver=self.connection.driver,
extra={
'snapshot-id': id,
},
)
return point
def _to_snapshot(self, snapshot):
state = self.SNAPSHOT_STATE_MAP.get(snapshot['state'],
VolumeSnapshotState.UNKNOWN)
extra = {
'organization': snapshot['organization'],
'volume_type': snapshot['volume_type'],
}
return VolumeSnapshot(id=snapshot['id'],
driver=self,
size=_to_lib_size(snapshot['size']),
created=parse_date(snapshot['creation_date']),
state=state,
extra=extra)
def _to_volume_snapshot(self, data):
extra_keys = ['labels', 'status', 'usage_in_minutes',
'location_country', 'current_price', 'parent_uuid']
extra = self._extract_values_to_dict(data=data, keys=extra_keys)
volume_snapshot = VolumeSnapshot(id=data['object_uuid'],
driver=self.connection.driver,
size=data['capacity'], extra=extra,
created=parse_date(
data['create_time']),
state=StorageVolumeState.AVAILABLE,
name=data['name'])
return volume_snapshot
def _to_version(self, data):
try:
updated = parse_date(data['updated'])
except Exception:
updated = None
try:
url = data['links'][0]['href']
except IndexError:
url = None
version = OpenStackIdentityVersion(version=data['id'],
status=data['status'],
updated=updated,
url=url)
return version
def _to_node(self, server):
public_ip = server['public_ip']
private_ip = server['private_ip']
location = server['location'] or {}
return Node(id=server['id'],
name=server['name'],
state=NodeState.fromstring(server['state']),
public_ips=[public_ip['address']] if public_ip else [],
private_ips=[private_ip] if private_ip else [],
driver=self,
extra={'volumes': server['volumes'],
'tags': server['tags'],
'arch': server['arch'],
'organization': server['organization'],
'region': location.get('zone_id', 'par1')},
created_at=parse_date(server['creation_date']))
def _to_version(self, data):
try:
updated = parse_date(data['updated'])
except Exception:
updated = None
try:
url = data['links'][0]['href']
except IndexError:
url = None
version = OpenStackIdentityVersion(version=data['id'],
status=data['status'],
updated=updated,
url=url)
return version
def _to_recovery_point(self, el, target):
id = findtext(element=el, xpath='snapshotId', namespace=NS)
date = parse_date(
findtext(element=el, xpath='startTime', namespace=NS))
tags = self._get_resource_tags(el)
point = BackupTargetRecoveryPoint(
id=id,
date=date,
target=target,
driver=self.connection.driver,
extra={
'snapshot-id': id,
'tags': tags
},
)
return point
def _to_recovery_point(self, el, target):
id = findtext(element=el, xpath='snapshotId', namespace=NS)
date = parse_date(findtext(element=el, xpath='startTime',
namespace=NS))
tags = self._get_resource_tags(el)
point = BackupTargetRecoveryPoint(
id=id,
date=date,
target=target,
driver=self.connection.driver,
extra={
'snapshot-id': id,
'tags': tags
},
)
return point
def authenticate(self, force=False):
reqbody = json.dumps({
'auth': {
'passwordCredentials': {
'username': self.user_id,
'password': self.key
},
'tenantId': self._ex_tenant_id
}
})
resp = self.request('/tokens', data=reqbody, headers={}, method='POST')
if resp.status == httplib.UNAUTHORIZED:
# HTTP UNAUTHORIZED (401): auth failed
raise InvalidCredsError()
elif resp.status != httplib.OK:
body = 'code: %s body:%s' % (resp.status, resp.body)
raise MalformedResponseError('Malformed response',
body=body,
driver=self.driver)
else:
try:
body = json.loads(resp.body)
except Exception as e:
raise MalformedResponseError('Failed to parse JSON', e)
try:
expires = body['access']['token']['expires']
self.auth_token = body['access']['token']['id']
self.auth_token_expires = parse_date(expires)
self.urls = body['access']['serviceCatalog']
self.auth_user_info = None
except KeyError as e:
raise MalformedResponseError(
'Auth JSON response is \
missing required elements', e)
return self
def _to_volume_snapshot(self, data):
extra_keys = [
"labels",
"status",
"usage_in_minutes",
"location_country",
"current_price",
"parent_uuid",
]
extra = self._extract_values_to_dict(data=data, keys=extra_keys)
volume_snapshot = VolumeSnapshot(
id=data["object_uuid"],
driver=self.connection.driver,
size=data["capacity"],
extra=extra,
created=parse_date(data["create_time"]),
state=StorageVolumeState.AVAILABLE,
name=data["name"],
)
return volume_snapshot
def main():
driver = get_libcloud_driver()
nodes = driver.list_nodes()
print("Looking for and deleting old running automated test nodes...")
nodes_to_delete = []
for node in nodes:
if INSTANCE_NAME_STRING not in node.name:
continue
launch_time = node.extra.get("launch_time", None)
if not launch_time:
continue
launch_time_dt = parse_date(launch_time).replace(tzinfo=None)
if launch_time_dt >= DELETE_OLD_NODES_THRESHOLD_DT:
continue
print(('Found node "%s" for deletion.' % (node.name)))
nodes_to_delete.append(node)
nodes = []
# TODO: For now we only print the node names to ensure script doesn't incorrectly delete
# wrong nodes. We should uncomment out deletion once we are sure the script is correct.
for node in nodes_to_delete:
assert INSTANCE_NAME_STRING in node.name
print("")
destroy_node_and_cleanup(driver=driver, node=node)
print("")
print("Destroyed %s old nodes" % (len(nodes_to_delete)))
def authenticate(self, force=False):
"""
Perform authentication.
"""
if not self._is_authentication_needed(force=force):
return self
data = {
'auth': {
'identity': {
'methods': ['password'],
'password': {
'user': {
'domain': {
'name': self.domain_name
},
'name': self.user_id,
'password': self.key
}
}
}
}
}
if self.token_scope == OpenStackIdentityTokenScope.PROJECT:
# Scope token to project (tenant)
data['auth']['scope'] = {
'project': {
'domain': {
'name': self.domain_name
},
'name': self.tenant_name
}
}
elif self.token_scope == OpenStackIdentityTokenScope.DOMAIN:
# Scope token to domain
data['auth']['scope'] = {'domain': {'name': self.domain_name}}
elif self.token_scope == OpenStackIdentityTokenScope.UNSCOPED:
pass
else:
raise ValueError('Token needs to be scoped either to project or '
'a domain')
data = json.dumps(data)
response = self.request('/v3/auth/tokens',
data=data,
headers={'Content-Type': 'application/json'},
method='POST')
if response.status == httplib.UNAUTHORIZED:
# Invalid credentials
raise InvalidCredsError()
elif response.status in [httplib.OK, httplib.CREATED]:
headers = response.headers
try:
body = json.loads(response.body)
except Exception:
e = sys.exc_info()[1]
raise MalformedResponseError('Failed to parse JSON', e)
try:
roles = self._to_roles(body['token']['roles'])
except Exception:
e = sys.exc_info()[1]
roles = []
try:
expires = body['token']['expires_at']
self.auth_token = headers['x-subject-token']
self.auth_token_expires = parse_date(expires)
# Note: catalog is not returned for unscoped tokens
self.urls = body['token'].get('catalog', None)
self.auth_user_info = None
self.auth_user_roles = roles
except KeyError:
e = sys.exc_info()[1]
raise MalformedResponseError(
'Auth JSON response is \
missing required elements', e)
body = 'code: %s body:%s' % (response.status, response.body)
else:
raise MalformedResponseError('Malformed response',
body=body,
driver=self.driver)
return self
def authenticate(self, force=False):
"""
Perform authentication.
"""
if not self._is_authentication_needed(force=force):
return self
data = {
'auth': {
'identity': {
'methods': ['password'],
'password': {
'user': {
'domain': {
'name': self.domain_name
},
'name': self.user_id,
'password': self.key
}
}
}
}
}
if self.token_scope == OpenStackIdentityTokenScope.PROJECT:
# Scope token to project (tenant)
data['auth']['scope'] = {
'project': {
'domain': {
'name': self.domain_name
},
'name': self.tenant_name
}
}
elif self.token_scope == OpenStackIdentityTokenScope.DOMAIN:
# Scope token to domain
data['auth']['scope'] = {
'domain': {
'name': self.domain_name
}
}
elif self.token_scope == OpenStackIdentityTokenScope.UNSCOPED:
pass
else:
raise ValueError('Token needs to be scoped either to project or '
'a domain')
data = json.dumps(data)
response = self.request('/v3/auth/tokens', data=data,
headers={'Content-Type': 'application/json'},
method='POST')
if response.status == httplib.UNAUTHORIZED:
# Invalid credentials
raise InvalidCredsError()
elif response.status in [httplib.OK, httplib.CREATED]:
headers = response.headers
try:
body = json.loads(response.body)
except Exception:
e = sys.exc_info()[1]
raise MalformedResponseError('Failed to parse JSON', e)
try:
roles = self._to_roles(body['token']['roles'])
except Exception:
e = sys.exc_info()[1]
roles = []
try:
expires = body['token']['expires_at']
self.auth_token = headers['x-subject-token']
self.auth_token_expires = parse_date(expires)
# Note: catalog is not returned for unscoped tokens
self.urls = body['token'].get('catalog', None)
self.auth_user_info = None
self.auth_user_roles = roles
except KeyError:
e = sys.exc_info()[1]
raise MalformedResponseError('Auth JSON response is \
missing required elements', e)
body = 'code: %s body:%s' % (response.status, response.body)
else:
raise MalformedResponseError('Malformed response', body=body,
driver=self.driver)
return self
def authenticate_3_x_with_password(self):
# TODO: Support for custom domain
# TODO: Refactor and add a class per API version
domain = 'Default'
data = {
'auth': {
'identity': {
'methods': ['password'],
'password': {
'user': {
'domain': {
'name': domain
},
'name': self.user_id,
'password': self.key
}
}
},
'scope': {
'project': {
'domain': {
'name': domain
},
'name': self.tenant_name
}
}
}
}
if self.tenant_name:
data['auth']['scope'] = {
'project': {
'domain': {
'name': domain
},
'name': self.tenant_name
}
}
data = json.dumps(data)
response = self.request('/v3/auth/tokens',
data=data,
headers={'Content-Type': 'application/json'},
method='POST')
if response.status == httplib.UNAUTHORIZED:
# Invalid credentials
raise InvalidCredsError()
elif response.status in [httplib.OK, httplib.CREATED]:
headers = response.headers
try:
body = json.loads(response.body)
except Exception:
e = sys.exc_info()[1]
raise MalformedResponseError('Failed to parse JSON', e)
try:
expires = body['token']['expires_at']
self.auth_token = headers['x-subject-token']
self.auth_token_expires = parse_date(expires)
self.urls = body['token']['catalog']
self.auth_user_info = None
except KeyError:
e = sys.exc_info()[1]
raise MalformedResponseError(
'Auth JSON response is \
missing required elements', e)
body = 'code: %s body:%s' % (response.status, response.body)
else:
raise MalformedResponseError('Malformed response',
body=body,
driver=self.driver)
return self
def _to_node(self, data):
if 'status' in data:
state = self.NODE_STATE_MAP.get(data['status'], NodeState.UNKNOWN)
if state == NodeState.RUNNING and \
data['power_status'] != 'running':
state = NodeState.STOPPED
else:
state = NodeState.UNKNOWN
if 'main_ip' in data and data['main_ip'] is not None:
public_ips = [data['main_ip']]
else:
public_ips = []
# simple check that we have ip address in value
if len(data['internal_ip']) > 0:
private_ips = [data['internal_ip']]
else:
private_ips = []
created_at = parse_date(data['date_created'])
# response ordering
extra_keys = [
"location", # Location name
"default_password", "pending_charges", "cost_per_month",
"current_bandwidth_gb", "allowed_bandwidth_gb", "netmask_v4",
"gateway_v4", "power_status", "server_state",
"v6_networks",
# TODO: Does we really need kvm_url?
"kvm_url",
"auto_backups", "tag",
# "OSID", # Operating system to use. See v1/os/list.
"APPID", "FIREWALLGROUPID"
]
extra = self._helper.handle_extra(extra_keys, data)
resolve_data = VULTR_COMPUTE_INSTANCE_IMAGES.get(data["OSID"])
if resolve_data:
image = self._to_image(resolve_data)
else:
image = None
resolve_data = VULTR_COMPUTE_INSTANCE_SIZES.get(data["VPSPLANID"])
if resolve_data:
size = self._to_size(resolve_data)
else:
size = None
# resolve_data = VULTR_COMPUTE_INSTANCE_LOCATIONS.get(data['DCID'])
# if resolve_data:
# location = self._to_location(resolve_data)
# extra['location'] = location
node = Node(
id=data['SUBID'],
name=data['label'],
state=state,
public_ips=public_ips,
private_ips=private_ips,
image=image,
size=size,
extra=extra,
created_at=created_at,
driver=self)
return node
def _to_node(self, data):
extra_keys = [
"memory",
"vcpus",
"disk",
"image",
"size",
"size_slug",
"locked",
"created_at",
"networks",
"kernel",
"backup_ids",
"snapshot_ids",
"features",
"tags",
]
if "status" in data:
state = self.NODE_STATE_MAP.get(data["status"], NodeState.UNKNOWN)
else:
state = NodeState.UNKNOWN
created = parse_date(data["created_at"])
networks = data["networks"]
private_ips = []
public_ips = []
if networks:
for net in networks["v4"]:
if net["type"] == "private":
private_ips = [net["ip_address"]]
if net["type"] == "public":
public_ips = [net["ip_address"]]
extra = {}
for key in extra_keys:
if key in data:
extra[key] = data[key]
extra["region"] = data.get("region", {}).get("name")
# Untouched extra values, backwards compatibility
resolve_data = data.get("image")
if resolve_data:
image = self._to_image(resolve_data)
else:
image = None
resolve_data = extra.get("size")
if resolve_data:
size = self._to_size(resolve_data)
else:
size = None
node = Node(
id=data["id"],
name=data["name"],
state=state,
image=image,
size=size,
public_ips=public_ips,
private_ips=private_ips,
created_at=created,
driver=self,
extra=extra,
)
return node
