def _to_volume(self, volume): extra = { 'organization': volume['organization'], 'volume_type': volume['volume_type'], 'creation_date': parse_date(volume['creation_date']), 'modification_date': parse_date(volume['modification_date']), } return StorageVolume(id=volume['id'], name=volume['name'], size=_to_lib_size(volume['size']), driver=self, extra=extra)
def _to_image(self, image): extra = { 'arch': image['arch'], 'size': _to_lib_size(image.get('root_volume', {}).get('size', 0)) or 50, 'creation_date': parse_date(image['creation_date']), 'modification_date': parse_date(image['modification_date']), 'organization': image['organization'], } return NodeImage(id=image['id'], name=image['name'], driver=self, extra=extra)
def _to_node(self, data): extra_keys = ["cores", "power", "memory", "current_price", "relations"] extra = self._extract_values_to_dict(data=data, keys=extra_keys) ips = [] for diction in data["relations"]["public_ips"]: ips.append(diction["ip"]) state = "" if data["power"] is True: state = NodeState.RUNNING else: state = NodeState.STOPPED node = Node( id=data["object_uuid"], name=data["name"], state=state, public_ips=ips, created_at=parse_date(data["create_time"]), private_ips=None, driver=self.connection.driver, extra=extra, ) return node
def _authenticate_2_0_with_body(self, reqbody): resp = self.request('/v2.0/tokens', data=reqbody, headers={'Content-Type': 'application/json'}, method='POST') if resp.status == httplib.UNAUTHORIZED: raise InvalidCredsError() elif resp.status not in [ httplib.OK, httplib.NON_AUTHORITATIVE_INFORMATION ]: body = 'code: %s body: %s' % (resp.status, resp.body) raise MalformedResponseError('Malformed response', body=body, driver=self.driver) else: body = resp.object try: access = body['access'] expires = access['token']['expires'] self.auth_token = access['token']['id'] self.auth_token_expires = parse_date(expires) self.urls = access['serviceCatalog'] self.auth_user_info = access.get('user', {}) except KeyError: e = sys.exc_info()[1] raise MalformedResponseError( 'Auth JSON response is \ missing required elements', e) return self
def _to_snapshot(self, api_node): if "snapshot" in api_node: api_node = api_node["snapshot"] extra = { "volume_id": api_node["volumeId"], "name": api_node["displayName"], "created": api_node["createdAt"], "description": api_node["displayDescription"], "status": api_node["status"], } state = self.SNAPSHOT_STATE_MAP.get( api_node["status"], VolumeSnapshotState.UNKNOWN ) try: created_td = parse_date(api_node["createdAt"]) except ValueError: created_td = None snapshot = VolumeSnapshot( id=api_node["id"], driver=self, size=api_node["size"], extra=extra, created=created_td, state=state, name=api_node["displayName"], ) return snapshot
def authenticate_2_0_with_body(self, reqbody): resp = self.request('/v2.0/tokens', data=reqbody, headers={'Content-Type': 'application/json'}, method='POST') if resp.status == httplib.UNAUTHORIZED: raise InvalidCredsError() elif resp.status not in [httplib.OK, httplib.NON_AUTHORITATIVE_INFORMATION]: body = 'code: %s body: %s' % (resp.status, resp.body) raise MalformedResponseError('Malformed response', body=body, driver=self.driver) else: try: body = json.loads(resp.body) except Exception: e = sys.exc_info()[1] raise MalformedResponseError('Failed to parse JSON', e) try: access = body['access'] expires = access['token']['expires'] self.auth_token = access['token']['id'] self.auth_token_expires = parse_date(expires) self.urls = access['serviceCatalog'] self.auth_user_info = access.get('user', {}) except KeyError: e = sys.exc_info()[1] raise MalformedResponseError('Auth JSON response is \ missing required elements', e) return self
def _to_node(self, data): extra_keys = ['memory', 'vcpus', 'disk', 'region', 'image', 'size_slug', 'locked', 'created_at', 'networks', 'kernel', 'backup_ids', 'snapshot_ids', 'features'] if 'status' in data: state = self.NODE_STATE_MAP.get(data['status'], NodeState.UNKNOWN) else: state = NodeState.UNKNOWN created = parse_date(data['created_at']) networks = data['networks'] private_ips = [] public_ips = [] if networks: for net in networks['v4']: if net['type'] == 'private': private_ips = [net['ip_address']] if net['type'] == 'public': public_ips = [net['ip_address']] extra = {} for key in extra_keys: if key in data: extra[key] = data[key] node = Node(id=data['id'], name=data['name'], state=state, public_ips=public_ips, private_ips=private_ips, created_at=created, driver=self, extra=extra) return node
def authenticate_1_1(self): reqbody = json.dumps({'credentials': {'username': self.user_id, 'key': self.key}}) resp = self.request("/v1.1/auth", data=reqbody, headers={}, method='POST') if resp.status == httplib.UNAUTHORIZED: # HTTP UNAUTHORIZED (401): auth failed raise InvalidCredsError() elif resp.status != httplib.OK: raise MalformedResponseError('Malformed response', body='code: %s body:%s' % (resp.status, resp.body), driver=self.driver) else: try: body = json.loads(resp.body) except Exception: e = sys.exc_info()[1] raise MalformedResponseError('Failed to parse JSON', e) try: expires = body['auth']['token']['expires'] self.auth_token = body['auth']['token']['id'] self.auth_token_expires = parse_date(expires) self.urls = body['auth']['serviceCatalog'] self.auth_user_info = None except KeyError: e = sys.exc_info()[1] raise MalformedResponseError('Auth JSON response is \ missing required elements', e) return self
def _to_snapshot(self, api_node): if 'snapshot' in api_node: api_node = api_node['snapshot'] extra = {'volume_id': api_node['volumeId'], 'name': api_node['displayName'], 'created': api_node['createdAt'], 'description': api_node['displayDescription'], 'status': api_node['status']} state = self.SNAPSHOT_STATE_MAP.get( api_node['status'], VolumeSnapshotState.UNKNOWN ) try: created_td = parse_date(api_node['createdAt']) except ValueError: created_td = None snapshot = VolumeSnapshot(id=api_node['id'], driver=self, size=api_node['size'], extra=extra, created=created_td, state=state) return snapshot
def _to_node(self, data): extra_keys = ['cores', 'power', 'memory', 'current_price', 'relations'] extra = self._extract_values_to_dict(data=data, keys=extra_keys) ips = [] for diction in data['relations']['public_ips']: ips.append(diction['ip']) state = '' if data['power'] is True: state = NodeState.RUNNING else: state = NodeState.STOPPED node = Node(id=data['object_uuid'], name=data['name'], state=state, public_ips=ips, created_at=parse_date(data['create_time']), private_ips=None, driver=self.connection.driver, extra=extra) return node
def authenticate_1_1(self): reqbody = json.dumps({'credentials': {'username': self.user_id, 'key': self.key}}) resp = self.request('/v1.1/auth', data=reqbody, headers={}, method='POST') if resp.status == httplib.UNAUTHORIZED: # HTTP UNAUTHORIZED (401): auth failed raise InvalidCredsError() elif resp.status != httplib.OK: body = 'code: %s body:%s' % (resp.status, resp.body) raise MalformedResponseError('Malformed response', body=body, driver=self.driver) else: try: body = json.loads(resp.body) except Exception: e = sys.exc_info()[1] raise MalformedResponseError('Failed to parse JSON', e) try: expires = body['auth']['token']['expires'] self.auth_token = body['auth']['token']['id'] self.auth_token_expires = parse_date(expires) self.urls = body['auth']['serviceCatalog'] self.auth_user_info = None except KeyError: e = sys.exc_info()[1] raise MalformedResponseError('Auth JSON response is \ missing required elements', e) return self
def _to_snapshot(self, api_node): if 'snapshot' in api_node: api_node = api_node['snapshot'] extra = { 'volume_id': api_node['volumeId'], 'name': api_node['displayName'], 'created': api_node['createdAt'], 'description': api_node['displayDescription'], 'status': api_node['status'] } state = self.SNAPSHOT_STATE_MAP.get(api_node['status'], VolumeSnapshotState.UNKNOWN) try: created_td = parse_date(api_node['createdAt']) except ValueError: created_td = None snapshot = VolumeSnapshot(id=api_node['id'], driver=self, size=api_node['size'], extra=extra, created=created_td, state=state) return snapshot
def _to_recovery_point(self, item, target): id = item.id date = parse_date(item.creationTimestamp) point = BackupTargetRecoveryPoint( id=id, date=date, target=target, driver=self.connection.driver, extra={"snapshot-id": id}, ) return point
def _to_node(self, data): extra_keys = ['memory', 'vcpus', 'disk', 'image', 'size', 'size_slug', 'locked', 'created_at', 'networks', 'kernel', 'backup_ids', 'snapshot_ids', 'features', 'tags'] if 'status' in data: state = self.NODE_STATE_MAP.get(data['status'], NodeState.UNKNOWN) else: state = NodeState.UNKNOWN created = parse_date(data['created_at']) networks = data['networks'] private_ips = [] public_ips = [] if networks: for net in networks['v4']: if net['type'] == 'private': private_ips = [net['ip_address']] if net['type'] == 'public': public_ips = [net['ip_address']] extra = {} for key in extra_keys: if key in data: extra[key] = data[key] extra['region'] = data.get('region', {}).get('name') # Untouched extra values, backwards compatibility resolve_data = data.get('image') if resolve_data: image = self._to_image(resolve_data) else: image = None resolve_data = extra.get('size') if resolve_data: size = self._to_size(resolve_data) else: size = None node = Node( id=data['id'], name=data['name'], state=state, image=image, size=size, public_ips=public_ips, private_ips=private_ips, created_at=created, driver=self, extra=extra) return node
def _to_recovery_point(self, item, target): id = item.id date = parse_date(item.creationTimestamp) point = BackupTargetRecoveryPoint( id=id, date=date, target=target, driver=self.connection.driver, extra={ 'snapshot-id': id, }, ) return point
def _to_snapshot(self, snapshot): state = self.SNAPSHOT_STATE_MAP.get(snapshot['state'], VolumeSnapshotState.UNKNOWN) extra = { 'organization': snapshot['organization'], 'volume_type': snapshot['volume_type'], } return VolumeSnapshot(id=snapshot['id'], driver=self, size=_to_lib_size(snapshot['size']), created=parse_date(snapshot['creation_date']), state=state, extra=extra)
def _to_volume_snapshot(self, data): extra_keys = ['labels', 'status', 'usage_in_minutes', 'location_country', 'current_price', 'parent_uuid'] extra = self._extract_values_to_dict(data=data, keys=extra_keys) volume_snapshot = VolumeSnapshot(id=data['object_uuid'], driver=self.connection.driver, size=data['capacity'], extra=extra, created=parse_date( data['create_time']), state=StorageVolumeState.AVAILABLE, name=data['name']) return volume_snapshot
def _to_version(self, data): try: updated = parse_date(data['updated']) except Exception: updated = None try: url = data['links'][0]['href'] except IndexError: url = None version = OpenStackIdentityVersion(version=data['id'], status=data['status'], updated=updated, url=url) return version
def _to_node(self, server): public_ip = server['public_ip'] private_ip = server['private_ip'] location = server['location'] or {} return Node(id=server['id'], name=server['name'], state=NodeState.fromstring(server['state']), public_ips=[public_ip['address']] if public_ip else [], private_ips=[private_ip] if private_ip else [], driver=self, extra={'volumes': server['volumes'], 'tags': server['tags'], 'arch': server['arch'], 'organization': server['organization'], 'region': location.get('zone_id', 'par1')}, created_at=parse_date(server['creation_date']))
def _to_recovery_point(self, el, target): id = findtext(element=el, xpath='snapshotId', namespace=NS) date = parse_date( findtext(element=el, xpath='startTime', namespace=NS)) tags = self._get_resource_tags(el) point = BackupTargetRecoveryPoint( id=id, date=date, target=target, driver=self.connection.driver, extra={ 'snapshot-id': id, 'tags': tags }, ) return point
def _to_recovery_point(self, el, target): id = findtext(element=el, xpath='snapshotId', namespace=NS) date = parse_date(findtext(element=el, xpath='startTime', namespace=NS)) tags = self._get_resource_tags(el) point = BackupTargetRecoveryPoint( id=id, date=date, target=target, driver=self.connection.driver, extra={ 'snapshot-id': id, 'tags': tags }, ) return point
def authenticate(self, force=False): reqbody = json.dumps({ 'auth': { 'passwordCredentials': { 'username': self.user_id, 'password': self.key }, 'tenantId': self._ex_tenant_id } }) resp = self.request('/tokens', data=reqbody, headers={}, method='POST') if resp.status == httplib.UNAUTHORIZED: # HTTP UNAUTHORIZED (401): auth failed raise InvalidCredsError() elif resp.status != httplib.OK: body = 'code: %s body:%s' % (resp.status, resp.body) raise MalformedResponseError('Malformed response', body=body, driver=self.driver) else: try: body = json.loads(resp.body) except Exception as e: raise MalformedResponseError('Failed to parse JSON', e) try: expires = body['access']['token']['expires'] self.auth_token = body['access']['token']['id'] self.auth_token_expires = parse_date(expires) self.urls = body['access']['serviceCatalog'] self.auth_user_info = None except KeyError as e: raise MalformedResponseError( 'Auth JSON response is \ missing required elements', e) return self
def _to_volume_snapshot(self, data): extra_keys = [ "labels", "status", "usage_in_minutes", "location_country", "current_price", "parent_uuid", ] extra = self._extract_values_to_dict(data=data, keys=extra_keys) volume_snapshot = VolumeSnapshot( id=data["object_uuid"], driver=self.connection.driver, size=data["capacity"], extra=extra, created=parse_date(data["create_time"]), state=StorageVolumeState.AVAILABLE, name=data["name"], ) return volume_snapshot
def main(): driver = get_libcloud_driver() nodes = driver.list_nodes() print("Looking for and deleting old running automated test nodes...") nodes_to_delete = [] for node in nodes: if INSTANCE_NAME_STRING not in node.name: continue launch_time = node.extra.get("launch_time", None) if not launch_time: continue launch_time_dt = parse_date(launch_time).replace(tzinfo=None) if launch_time_dt >= DELETE_OLD_NODES_THRESHOLD_DT: continue print(('Found node "%s" for deletion.' % (node.name))) nodes_to_delete.append(node) nodes = [] # TODO: For now we only print the node names to ensure script doesn't incorrectly delete # wrong nodes. We should uncomment out deletion once we are sure the script is correct. for node in nodes_to_delete: assert INSTANCE_NAME_STRING in node.name print("") destroy_node_and_cleanup(driver=driver, node=node) print("") print("Destroyed %s old nodes" % (len(nodes_to_delete)))
def authenticate(self, force=False): """ Perform authentication. """ if not self._is_authentication_needed(force=force): return self data = { 'auth': { 'identity': { 'methods': ['password'], 'password': { 'user': { 'domain': { 'name': self.domain_name }, 'name': self.user_id, 'password': self.key } } } } } if self.token_scope == OpenStackIdentityTokenScope.PROJECT: # Scope token to project (tenant) data['auth']['scope'] = { 'project': { 'domain': { 'name': self.domain_name }, 'name': self.tenant_name } } elif self.token_scope == OpenStackIdentityTokenScope.DOMAIN: # Scope token to domain data['auth']['scope'] = {'domain': {'name': self.domain_name}} elif self.token_scope == OpenStackIdentityTokenScope.UNSCOPED: pass else: raise ValueError('Token needs to be scoped either to project or ' 'a domain') data = json.dumps(data) response = self.request('/v3/auth/tokens', data=data, headers={'Content-Type': 'application/json'}, method='POST') if response.status == httplib.UNAUTHORIZED: # Invalid credentials raise InvalidCredsError() elif response.status in [httplib.OK, httplib.CREATED]: headers = response.headers try: body = json.loads(response.body) except Exception: e = sys.exc_info()[1] raise MalformedResponseError('Failed to parse JSON', e) try: roles = self._to_roles(body['token']['roles']) except Exception: e = sys.exc_info()[1] roles = [] try: expires = body['token']['expires_at'] self.auth_token = headers['x-subject-token'] self.auth_token_expires = parse_date(expires) # Note: catalog is not returned for unscoped tokens self.urls = body['token'].get('catalog', None) self.auth_user_info = None self.auth_user_roles = roles except KeyError: e = sys.exc_info()[1] raise MalformedResponseError( 'Auth JSON response is \ missing required elements', e) body = 'code: %s body:%s' % (response.status, response.body) else: raise MalformedResponseError('Malformed response', body=body, driver=self.driver) return self
def authenticate(self, force=False): """ Perform authentication. """ if not self._is_authentication_needed(force=force): return self data = { 'auth': { 'identity': { 'methods': ['password'], 'password': { 'user': { 'domain': { 'name': self.domain_name }, 'name': self.user_id, 'password': self.key } } } } } if self.token_scope == OpenStackIdentityTokenScope.PROJECT: # Scope token to project (tenant) data['auth']['scope'] = { 'project': { 'domain': { 'name': self.domain_name }, 'name': self.tenant_name } } elif self.token_scope == OpenStackIdentityTokenScope.DOMAIN: # Scope token to domain data['auth']['scope'] = { 'domain': { 'name': self.domain_name } } elif self.token_scope == OpenStackIdentityTokenScope.UNSCOPED: pass else: raise ValueError('Token needs to be scoped either to project or ' 'a domain') data = json.dumps(data) response = self.request('/v3/auth/tokens', data=data, headers={'Content-Type': 'application/json'}, method='POST') if response.status == httplib.UNAUTHORIZED: # Invalid credentials raise InvalidCredsError() elif response.status in [httplib.OK, httplib.CREATED]: headers = response.headers try: body = json.loads(response.body) except Exception: e = sys.exc_info()[1] raise MalformedResponseError('Failed to parse JSON', e) try: roles = self._to_roles(body['token']['roles']) except Exception: e = sys.exc_info()[1] roles = [] try: expires = body['token']['expires_at'] self.auth_token = headers['x-subject-token'] self.auth_token_expires = parse_date(expires) # Note: catalog is not returned for unscoped tokens self.urls = body['token'].get('catalog', None) self.auth_user_info = None self.auth_user_roles = roles except KeyError: e = sys.exc_info()[1] raise MalformedResponseError('Auth JSON response is \ missing required elements', e) body = 'code: %s body:%s' % (response.status, response.body) else: raise MalformedResponseError('Malformed response', body=body, driver=self.driver) return self
def authenticate_3_x_with_password(self): # TODO: Support for custom domain # TODO: Refactor and add a class per API version domain = 'Default' data = { 'auth': { 'identity': { 'methods': ['password'], 'password': { 'user': { 'domain': { 'name': domain }, 'name': self.user_id, 'password': self.key } } }, 'scope': { 'project': { 'domain': { 'name': domain }, 'name': self.tenant_name } } } } if self.tenant_name: data['auth']['scope'] = { 'project': { 'domain': { 'name': domain }, 'name': self.tenant_name } } data = json.dumps(data) response = self.request('/v3/auth/tokens', data=data, headers={'Content-Type': 'application/json'}, method='POST') if response.status == httplib.UNAUTHORIZED: # Invalid credentials raise InvalidCredsError() elif response.status in [httplib.OK, httplib.CREATED]: headers = response.headers try: body = json.loads(response.body) except Exception: e = sys.exc_info()[1] raise MalformedResponseError('Failed to parse JSON', e) try: expires = body['token']['expires_at'] self.auth_token = headers['x-subject-token'] self.auth_token_expires = parse_date(expires) self.urls = body['token']['catalog'] self.auth_user_info = None except KeyError: e = sys.exc_info()[1] raise MalformedResponseError( 'Auth JSON response is \ missing required elements', e) body = 'code: %s body:%s' % (response.status, response.body) else: raise MalformedResponseError('Malformed response', body=body, driver=self.driver) return self
def _to_node(self, data): if 'status' in data: state = self.NODE_STATE_MAP.get(data['status'], NodeState.UNKNOWN) if state == NodeState.RUNNING and \ data['power_status'] != 'running': state = NodeState.STOPPED else: state = NodeState.UNKNOWN if 'main_ip' in data and data['main_ip'] is not None: public_ips = [data['main_ip']] else: public_ips = [] # simple check that we have ip address in value if len(data['internal_ip']) > 0: private_ips = [data['internal_ip']] else: private_ips = [] created_at = parse_date(data['date_created']) # response ordering extra_keys = [ "location", # Location name "default_password", "pending_charges", "cost_per_month", "current_bandwidth_gb", "allowed_bandwidth_gb", "netmask_v4", "gateway_v4", "power_status", "server_state", "v6_networks", # TODO: Does we really need kvm_url? "kvm_url", "auto_backups", "tag", # "OSID", # Operating system to use. See v1/os/list. "APPID", "FIREWALLGROUPID" ] extra = self._helper.handle_extra(extra_keys, data) resolve_data = VULTR_COMPUTE_INSTANCE_IMAGES.get(data["OSID"]) if resolve_data: image = self._to_image(resolve_data) else: image = None resolve_data = VULTR_COMPUTE_INSTANCE_SIZES.get(data["VPSPLANID"]) if resolve_data: size = self._to_size(resolve_data) else: size = None # resolve_data = VULTR_COMPUTE_INSTANCE_LOCATIONS.get(data['DCID']) # if resolve_data: # location = self._to_location(resolve_data) # extra['location'] = location node = Node( id=data['SUBID'], name=data['label'], state=state, public_ips=public_ips, private_ips=private_ips, image=image, size=size, extra=extra, created_at=created_at, driver=self) return node
def _to_node(self, data): extra_keys = [ "memory", "vcpus", "disk", "image", "size", "size_slug", "locked", "created_at", "networks", "kernel", "backup_ids", "snapshot_ids", "features", "tags", ] if "status" in data: state = self.NODE_STATE_MAP.get(data["status"], NodeState.UNKNOWN) else: state = NodeState.UNKNOWN created = parse_date(data["created_at"]) networks = data["networks"] private_ips = [] public_ips = [] if networks: for net in networks["v4"]: if net["type"] == "private": private_ips = [net["ip_address"]] if net["type"] == "public": public_ips = [net["ip_address"]] extra = {} for key in extra_keys: if key in data: extra[key] = data[key] extra["region"] = data.get("region", {}).get("name") # Untouched extra values, backwards compatibility resolve_data = data.get("image") if resolve_data: image = self._to_image(resolve_data) else: image = None resolve_data = extra.get("size") if resolve_data: size = self._to_size(resolve_data) else: size = None node = Node( id=data["id"], name=data["name"], state=state, image=image, size=size, public_ips=public_ips, private_ips=private_ips, created_at=created, driver=self, extra=extra, ) return node