def test_list(self):
update = Update(Mock(), 'driver_test')
update.make_request = Mock()
update.get('key1')
update.make_request.assert_called_once_with(method='GET',
uri='updates/job/key1')
def __init__(self, env, create_updates):
self.env = env
self.create_updates = create_updates
auth_inst = auth.Auth(api_url=GLOBOMAP_LOADER_API_URL,
username=GLOBOMAP_LOADER_API_USERNAME,
password=GLOBOMAP_LOADER_API_PASSWORD)
self.update = Update(auth=auth_inst, driver_name='cloudstack')
def __init__(self):
self.search = {'asorting_cols': ['-id']}
self.client = NetworkAPI().client
auth_inst = auth.Auth(
api_url=settings.GLOBOMAP_LOADER_API_URL,
username=settings.GLOBOMAP_LOADER_API_USERNAME,
password=settings.GLOBOMAP_LOADER_API_PASSWORD
)
self.update = Update(auth=auth_inst, driver_name='napi')
def test_post_list(self):
update = Update(Mock(), 'driver_test')
update.make_request = Mock()
update.post([{'doc': 1}])
update.make_request.assert_called_once_with(method='POST',
uri='updates/',
data=[{
'doc': 1
}])
def run():
while True:
inst = Napi()
auth_inst = auth.Auth(api_url=settings.GLOBOMAP_LOADER_API_URL,
username=settings.GLOBOMAP_LOADER_API_USERNAME,
password=settings.GLOBOMAP_LOADER_API_PASSWORD)
update = Update(auth=auth_inst, driver_name='napi')
try:
inst.process_updates(update.post)
except Exception:
LOGGER.exception('Error syncing updates from driver')
finally:
LOGGER.debug('No more updates found')
LOGGER.debug('Sleeping for %ss' % 30)
sleep(30)
class CloudstackDataLoader(object):
def __init__(self, env, create_updates):
self.env = env
self.create_updates = create_updates
auth_inst = auth.Auth(api_url=GLOBOMAP_LOADER_API_URL,
username=GLOBOMAP_LOADER_API_USERNAME,
password=GLOBOMAP_LOADER_API_PASSWORD)
self.update = Update(auth=auth_inst, driver_name='cloudstack')
def run(self):
start_time = int(time())
acs_service = self._get_cloudstack_service()
self._process_accounts(acs_service)
self._process_projects(acs_service)
self._clear_not_updated_elements(start_time)
logger.info('Processing finished')
def _process_projects(self, acs_service):
projects = acs_service.list_projects()
logger.info('%s projects found. Processing:' % len(projects))
for project in projects:
prj_name = project.get('name', project.get('displaytext'))
logger.info('Processing project %s' % prj_name)
pages = math.ceil(project.get('vmtotal', 0) / 500)
for page in range(1, pages + 1):
vms = acs_service.list_virtual_machines_by_project(
project['id'], page, 500)
logger.info('Creating %s VM events' % len(vms))
for vm in vms:
event = self._create_event(vm['id'])
self._publish_updates(self.create_updates(event))
def _process_accounts(self, acs_service):
accounts = acs_service.list_accounts()
logger.info('%s accounts found. Processing:' % len(accounts))
for account in accounts:
prj_name = account.get('name', account.get('displaytext'))
logger.info('Processing account %s' % prj_name)
pages = math.ceil(account.get('vmtotal', 0) / 500)
for page in range(1, pages + 1):
vms = acs_service.list_virtual_machines_by_account(
account['id'], page, 500)
logger.info('Creating %s VM events' % len(vms))
for vm in vms:
event = self._create_event(vm['id'])
self._publish_updates(self.create_updates(event))
def _create_event(self, vm_id):
event_date = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
return {
'event': 'VM.CREATE',
'resource': 'com.cloud.vm.VirtualMachine',
'eventDateTime': event_date,
'id': vm_id
}
def _publish_updates(self, updates):
try:
self._send(updates)
except Exception:
logger.exception('Unable to publish event. Aborting execution')
raise
def _clear_not_updated_elements(self, start_time):
logger.info('[Clear] Deleting old elements')
clears = list()
clears.append(
self._clear(Collection.COMP_UNIT, Collection.type_name(),
start_time))
clears.append(
self._clear(Collection.ZONE, Collection.type_name(), start_time))
clears.append(
self._clear(Collection.REGION, Collection.type_name(), start_time))
clears.append(self._clear(Edge.ZONE_HOST, Edge.type_name(),
start_time))
clears.append(
self._clear(Edge.ZONE_REGION, Edge.type_name(), start_time))
clears.append(
self._clear(Edge.PROCESS_COMP_UNIT, Edge.type_name(), start_time))
clears.append(
self._clear(Edge.BUSINESS_SERVICE_COMP_UNIT, Edge.type_name(),
start_time))
clears.append(
self._clear(Edge.CLIENT_COMP_UNIT, Edge.type_name(), start_time))
clears.append(
self._clear(Edge.HOST_COMP_UNIT, Edge.type_name(), start_time))
self._send(clears)
def _clear(self, collection, type, timestamp):
logger.info("Cleaning '%s' before %s" % (collection, timestamp))
return {
'action':
GloboMapActions.CLEAR,
'collection':
collection,
'type':
type,
'element': [[
self._filter('timestamp', timestamp, '<'),
self._filter('properties.environment', self.env, '=='),
self._filter('properties.iaas_provider', 'cloudstack', '==')
]]
}
def _filter(self, field, value, operator):
return {'field': field, 'value': value, 'operator': operator}
def _send(self, data):
try:
res = self.update.post(data)
except Exception:
logger.exception('Message dont sent %s', json.dumps(data))
else:
logger.debug('Message was sent %s', res)
def _get_cloudstack_service(self):
acs_url = self._get_setting('API_URL')
logger.info('Connecting to ACS: %s' % acs_url)
acs_client = CloudStackClient(acs_url, self._get_setting('API_KEY'),
self._get_setting('API_SECRET_KEY'),
True)
return CloudstackService(acs_client)
def _get_setting(self, key, default=None):
return get_setting(self.env, key, default)
import time
import os
from globomap_loader_api_client.auth import Auth
from globomap_loader_api_client.update import Update
GMAP_LOADER_API_PORT = os.getenv('GMAP_LOADER_API_PORT', '7010')
auth_inst = Auth(api_url='http://localhost:{}'.format(GMAP_LOADER_API_PORT),
username='******',
password='******')
update = Update(auth=auth_inst, driver_name='talk_globomap')
data = {
"action": "CLEAR",
"collection": "vip",
"element": [[{
"field": "timestamp",
"operator": ">",
"value": 1
}]],
"type": "collections"
}
res = update.post(data)
data = {
"action": "CLEAR",
"collection": "vip_vm",
"element": [[{
"field": "timestamp",
"operator": ">",
class Loader(object):
def __init__(self):
self.search = {'asorting_cols': ['-id']}
self.client = NetworkAPI().client
auth_inst = auth.Auth(
api_url=settings.GLOBOMAP_LOADER_API_URL,
username=settings.GLOBOMAP_LOADER_API_USERNAME,
password=settings.GLOBOMAP_LOADER_API_PASSWORD
)
self.update = Update(auth=auth_inst, driver_name='napi')
def _construct(self, action, collection, type_coll, content):
new_time = int(time())
content['timestamp'] = new_time
data = {
'action': action,
'collection': collection,
'type': type_coll,
'element': content
}
if action != 'CREATE':
data['key'] = '{}_{}'.format(content['provider'], content['id'])
return data
def send(self, data):
try:
res = self.update.post(data)
except Exception:
LOGGER.exception('Message dont sent %s', json.dumps(data))
else:
return res
def vips(self):
"""Load vips"""
data_list = []
obj = self.client.create_api_vip_request()
pages = self._paging(obj=obj, key='vips', next_search=self.search,
kind='details')
while True:
vips = next(pages)
for vip in vips:
data_list.append(self._vip(vip))
data_list += self._ports(vip)
res = data_list
data_list = []
yield res
def pools(self):
"""Load pools"""
data_list = []
obj = self.client.create_api_pool()
pages = self._paging(obj=obj, key='server_pools',
next_search=self.search, kind='details')
while True:
pools = next(pages)
for pool in pools:
data_list.append(self._pool(pool))
data_list += self._pool_members(pool)
res = data_list
data_list = []
yield res
def equipments(self):
"""Load equipments"""
data_list = []
obj = self.client.create_api_v4_equipment()
pages = self._paging(
obj=obj, key='equipments', next_search=self.search,
include=['ipsv4__basic__ip__ip_formated',
'ipsv6__basic__ip__ip_formated',
'equipment_type__details'])
while True:
equipments = next(pages)
for equipment in equipments:
data_list.append(self._equipament(equipment))
data_list += self._equipment_ipv4(equipment)
data_list += self._equipment_ipv6(equipment)
res = data_list
data_list = []
yield res
def networksv4(self):
"""Load networksv4"""
data_list = []
obj = self.client.create_api_network_ipv4()
pages = self._paging(obj=obj, key='networks',
next_search=self.search, kind='details')
while True:
networks = next(pages)
for network in networks:
data_list.append(self._networkv4(network))
data_list.append(self._network_vlan(network))
res = data_list
data_list = []
yield res
def networksv6(self):
"""Load networksv6"""
data_list = []
obj = self.client.create_api_network_ipv6()
pages = self._paging(obj=obj, key='networks',
next_search=self.search, kind='details')
while True:
networks = next(pages)
for network in networks:
data_list.append(self._networkv6(network))
data_list.append(self._network_vlan(network))
res = data_list
data_list = []
yield res
def vlans(self):
"""Load vlans"""
data_list = []
obj = self.client.create_api_vlan()
pages = self._paging(obj=obj, key='vlans',
next_search=self.search,
include=['environment__basic'])
while True:
vlans = next(pages)
for vlan in vlans:
data_list.append(self._vlan(vlan))
data_list.append(self._environment_vlan(vlan))
res = data_list
data_list = []
yield res
def environments(self):
"""Load environments"""
data_list = []
obj = self.client.create_api_environment()
pages = self._paging(
obj=obj, key='environments', next_search=self.search,
include=['default_vrf__details', 'father_environment__basic'])
while True:
environments = next(pages)
for environment in environments:
data_list.append(self._environment(environment))
if environment['father_environment']:
data_list.append(self._father_environment(environment))
res = data_list
data_list = []
yield res
def _vip(self, vip):
content = DataSpec().vip(vip)
data = self._construct('UPDATE', 'vip', 'collections', content)
return data
def _ports(self, vip):
data_list = []
for port in vip['ports']:
for pool in port['pools']:
pool['port'] = port['port']
pool['options'] = port['options']
content = DataSpec().port(pool, vip['id'])
data = self._construct('UPDATE', 'port', 'edges', content)
data_list.append(data)
return data_list
def _pool(self, pool):
content = DataSpec().pool(pool)
data = self._construct('UPDATE', 'pool', 'collections', content)
return data
def _pool_members(self, pool):
data_list = []
for member in pool.get('server_pool_members', []):
content = DataSpec().pool_comp_unit(member, pool['id'])
data = self._construct(
'UPDATE', 'pool_comp_unit', 'edges', content)
data_list.append(data)
return data_list
def _equipament(self, equipment):
equipment['ipv4'] = [ip['ip'] for ip in equipment['ipsv4']]
equipment['ipv6'] = [ip['ip'] for ip in equipment['ipsv6']]
content = DataSpec().comp_unit(equipment)
data = self._construct('PATCH', 'comp_unit', 'collections', content)
return data
def _equipment_ipv4(self, equipment):
data_list = []
for ipv4 in equipment.get('ipsv4'):
ipv4['ip']['networkipv4'] = ipv4['ip']['networkipv4']['id']
content = DataSpec().network_comp_unit(
ipv4['ip'], equipment['name'], ipv4['id'])
data = self._construct(
'UPDATE', 'network_comp_unit', 'edges', content)
data_list.append(data)
return data_list
def _equipment_ipv6(self, equipment):
data_list = []
for ipv6 in equipment.get('ipsv6'):
ipv6['ip']['networkipv6'] = ipv6['ip']['networkipv6']['id']
content = DataSpec().network_comp_unit(
ipv6['ip'], equipment['name'], ipv6['id'])
data = self._construct(
'UPDATE', 'network_comp_unit', 'edges', content)
data_list.append(data)
return data_list
def _networkv6(self, network):
content = DataSpec().network(network)
data = self._construct('UPDATE', 'network', 'collections', content)
return data
def _network_vlan(self, network):
content = DataSpec().vlan_network(network)
data = self._construct('UPDATE', 'vlan_network', 'edges', content)
return data
def _networkv4(self, network):
content = DataSpec().network(network)
data = self._construct('UPDATE', 'network', 'collections', content)
return data
def _vlan(self, vlan):
content = DataSpec().vlan(vlan)
data = self._construct('UPDATE', 'vlan', 'collections', content)
return data
def _environment_vlan(self, vlan):
content = DataSpec().environment_vlan(vlan)
data = self._construct('UPDATE', 'environment_vlan',
'edges', content)
return data
def _environment(self, environment):
content = DataSpec().environment(environment)
data = self._construct('UPDATE', 'environment',
'collections', content)
return data
def _father_environment(self, environment):
content = DataSpec().father_environment(environment)
data = self._construct('UPDATE', 'father_environment',
'edges', content)
return data
def _paging(self, **kwargs):
obj = kwargs.get('obj')
key = kwargs.get('key')
next_search = kwargs.get('next_search')
kind = kwargs.get('kind', '')
fields = kwargs.get('fields', [])
include = kwargs.get('include', [])
while True:
LOGGER.debug(
'[DriverNapi][loader][request] %s - next_search %s' %
(key, next_search)
)
try:
objs = obj.search(search=next_search, kind=kind,
fields=fields, include=include)
except NetworkAPIClientError as err:
LOGGER.error(
'[DriverNapi][loader][response] %s %s' % (key, err))
break
else:
LOGGER.debug(
'[DriverNapi][loader][response] %s %s' %
(key, objs[key])
)
if objs[key]:
next_search = objs['next_search']
yield objs[key]
else:
break
def run(self):
current_time = int(time())
for messages in self.vips():
self.send(messages)
for messages in self.pools():
self.send(messages)
for messages in self.environments():
self.send(messages)
for messages in self.vlans():
self.send(messages)
for messages in self.networksv4():
self.send(messages)
for messages in self.networksv6():
self.send(messages)
for messages in self.equipments():
self.send(messages)
self.send([clear('vip', 'collections', current_time)])
self.send([clear('pool', 'collections', current_time)])
self.send([clear('comp_unit', 'collections', current_time)])
self.send([clear('network', 'collections', current_time)])
self.send([clear('vlan', 'collections', current_time)])
self.send([clear('environment', 'collections', current_time)])
self.send([clear('port', 'edges', current_time)])
self.send([clear('pool_comp_unit', 'edges', current_time)])
self.send([clear('network_comp_unit', 'edges', current_time)])
self.send([clear('vlan_network', 'edges', current_time)])
self.send([clear('environment_vlan', 'edges', current_time)])
self.send([clear('father_environment', 'edges', current_time)])
def __init__(self):
self.driver = Driver()
auth_inst = auth.Auth(api_url=settings.GLOBOMAP_LOADER_API_URL,
username=settings.GLOBOMAP_LOADER_API_USERNAME,
password=settings.GLOBOMAP_LOADER_API_PASSWORD)
Loader.update = Update(auth=auth_inst, driver_name='sample')