class NuvlaApi(object):
def __init__(self):
self.endpoint = os.getenv('NUVLA_ENDPOINT', "https://nuvla.io")
self.api = Api(endpoint=self.endpoint, reauthenticate=True)
self.apikey = os.getenv('NUVLA_DEV_APIKEY')
self.apisecret = os.getenv('NUVLA_DEV_APISECRET')
def login(self):
self.api.login_apikey(self.apikey, self.apisecret)
def get_deployment_api(self, deployment_id):
creds = Deployment._get_attr(
Deployment(self.api).get(deployment_id), 'api-credentials')
insecure = not self.api.session.verify
api = Api(endpoint=self.api.endpoint,
insecure=insecure,
persist_cookie=False,
reauthenticate=True)
api.login_apikey(creds['api-key'], creds['api-secret'])
return Deployment(api)
def execute(self):
self.name = self.args.name if self.args.name is not None else names[
int(random.uniform(1, len(names) - 1))]
# true unless header authentication is used
reauthenticate = self.args.api_authn_header is None
self.api = Api(endpoint=self.args.api_url, insecure=self.args.api_insecure,
persist_cookie=False, reauthenticate=reauthenticate,
authn_header=self.args.api_authn_header)
try:
if self.args.api_authn_header is None:
if self.args.api_key and self.args.api_secret:
response = self.api.login_apikey(self.args.api_key, self.args.api_secret)
else:
response = self.api.login_password(self.args.api_user, self.args.api_pass)
if response.status_code == 403:
raise ConnectionError(
'Login with following user/apikey {} failed!'.format(self.args.api_user))
# Uncomment following lines for manual remote test
# session_id = self.api.current_session()
# self.api.operation(self.api.get(session_id), 'switch-group',
# {'claim': "group/nuvla-admin"})
except ConnectionError as e:
logging.error('Unable to connect to Nuvla endpoint {}! {}'.format(self.api.endpoint, e))
exit(1)
if self.args.zk_hosts:
from kazoo.client import KazooClient, KazooRetry
self._kz = KazooClient(','.join(self.args.zk_hosts),
connection_retry=KazooRetry(max_tries=-1),
command_retry=KazooRetry(max_tries=-1), timeout=30.0)
self._kz.start()
if self.args.statsd:
statsd_hp = self.args.statsd.split(':')
statsd_port = STATSD_PORT
statsd_host = statsd_hp[0]
if len(statsd_hp) > 1:
statsd_port = statsd_hp[1]
try:
self.statsd = StatsClient(host=statsd_host,
port=statsd_port,
prefix=None,
ipv6=False)
except Exception as ex:
logging.error(f'Failed to initialise StatsD client for {self.args.statsd}: {ex}')
self.do_work()
while True:
signal.pause()
def nuvla_init(config):
log.info("Authenticating with Nuvla...")
nuvla = Nuvla(endpoint=config.get('nuvla', {}).get('endpoint'), insecure=True)
n = 0
ts = time.time()
while True:
try:
nuvla.login_password(config['nuvla']['username'], config['nuvla']['password'])
break
except RETRY_EXCEPTIONS as ex:
log.error("Authenticating with Nuvla... failed: {}".format(ex))
st = 2 ** n
log.info("Authenticating with Nuvla... re-attempting in {} sec.".format(st))
time.sleep(st)
n = (n < 7) and (n + 1) or 0
log.info("Authenticating with Nuvla... done. (time took: {} sec)".format(int(time.time() - ts)))
return nuvla
def _get_user_api(self):
authn_info = self.payload['authn-info']
insecure = not self.api.session.verify
return Api(endpoint=self.api.endpoint, insecure=insecure,
persist_cookie=False, reauthenticate=True,
authn_header=f'{authn_info["user-id"]} '
f'{authn_info["active-claim"]} '
f'{" ".join(authn_info["claims"])}')
def nuvla_api():
""" Initialize API instance """
if os.path.exists(utils.nuvla_configuration):
nuvla_endpoint_raw = nuvla_endpoint_insecure_raw = None
with open(utils.nuvla_configuration) as nuvla_conf:
for line in nuvla_conf.read().split():
try:
if line and 'NUVLA_ENDPOINT=' in line:
nuvla_endpoint_raw = line.split('=')[-1]
if line and 'NUVLA_ENDPOINT_INSECURE=' in line:
nuvla_endpoint_insecure_raw = bool(line.split('=')[-1])
except IndexError:
pass
if nuvla_endpoint_raw and nuvla_endpoint_insecure_raw:
api = Api(endpoint='https://{}'.format(nuvla_endpoint_raw),
insecure=nuvla_endpoint_insecure_raw,
reauthenticate=True)
else:
raise Exception(
f'Misconfigured Nuvla parameters in {utils.nuvla_configuration}. Cannot perform operation'
)
else:
raise Exception(
"NuvlaBox is not yet ready to be operated. Missing Nuvla configuration parameters"
)
try:
with open(utils.activation_flag) as a:
user_info = json.loads(a.read())
except FileNotFoundError:
raise Exception("Cannot authenticate back with Nuvla")
api.login_apikey(user_info['api-key'], user_info['secret-key'])
return api
def dr_create(nuvla: Nuvla, num_create: int):
while num_create > 0:
try:
tm = datetime.utcnow().replace(tzinfo=timezone.utc) \
.replace(microsecond=0).isoformat().replace('+00:00', 'Z')
dr_object.update({'timestamp': tm})
res = nuvla.add('data-record', dr_object)
if res.data['status'] != 201:
print('Failed to create object.')
else:
print('Created object.')
except Exception as ex:
print('Exception while adding data-record: {}'.format(ex))
time.sleep(0.1)
num_create -= 1
def set_api(nuvla_endpoint):
tmp_api = Api(endpoint=nuvla_endpoint, insecure=False, reauthenticate=True)
# check SSL connection
try:
tmp_api.get("session")
except requests.exceptions.SSLError:
logging.warning("Setting {} with an insecure connection".format(nuvla_endpoint))
tmp_api = Api(endpoint=nuvla_endpoint, insecure=True, reauthenticate=True)
return tmp_api
def data_records_to_delete(nuvla: Nuvla, obj_num, filter, size_bytes_after, page_size, thresholdLow_station):
data_records = []
# Pagination.
first = 1
if obj_num < page_size:
last = obj_num
else:
last = page_size
log.info('filter: {}'.format(filter))
orderby = 'gnss:timestamp:asc'
log.info('orderby: {}'.format(orderby))
aggrs = 'sum:bytes'
log.info('aggregations: {}'.format(aggrs))
select = 'id,bucket,object,bytes'
log.info('select: {}'.format(select))
while (size_bytes_after > thresholdLow_station) and (last <= obj_num):
log.info("Collecting data-records with paging: {0} {1} {2} {3}"
.format(size_bytes_after, thresholdLow_station, first, last))
res = nuvla.search('data-record',
filter=filter,
orderby=orderby,
aggregation=aggrs,
first=first,
last=last,
select=select)
drs = []
for dr in res.data['resources']:
drs.append(dr)
size_bytes_after -= dr['bytes']
if size_bytes_after < thresholdLow_station:
log.info("size_bytes_after {0} below or equal watermark {1}. Ready to delete."
.format(size_bytes_after, thresholdLow_station))
break
data_records.append(drs)
first = last + 1
last += page_size
return data_records
def __init__(self):
self.endpoint = os.getenv('NUVLA_ENDPOINT', "https://nuvla.io")
self.api = Api(endpoint=self.endpoint, reauthenticate=True)
self.apikey = os.getenv('NUVLA_DEV_APIKEY')
self.apisecret = os.getenv('NUVLA_DEV_APISECRET')
#
# Ensure complete environment or bail out.
#
if (endpoint is None or api_key is None or api_secret is None
or deployment_id is None):
print(
'Missing required configuration information; skipping data mounts...')
sys.exit()
#
# setup the Nuvla API
#
api = Api(endpoint=endpoint, insecure=True)
api.login_apikey(api_key, api_secret)
# Recover deployment information.
deployment = api.get(deployment_id)
print('Getting data records...')
records = {}
if 'data' in deployment.data and 'records' in deployment.data['data']:
data_record_filters = deployment.data['data']['records']['filters']
print('Filters to process', data_record_filters)
for drf in data_record_filters:
dr_filter = drf['filter']
time_start = drf['time-start']
time_end = drf['time-end']
import time from nuvla.api import Api as Nuvla from nuvla.api.resources.module import Module from nuvla.api.resources.deployment import Deployment from nuvla.api.resources.user import User from utils import nuvla_conf_user_pass # Set to True to print Nuvla request / response messages to stdout. debug = False # Get username and password from configuration file. username, password = nuvla_conf_user_pass() # Create Nuvla client. No authentication required. nuvla = Nuvla(debug=debug) # nuvla = Nuvla(endpoint='https://localhost', insecure=True, debug=debug) # # Login to Nuvla. # user_api = User(nuvla) user_api.login_password(username, password) # # Create Deployment API handler. # dpl_api = Deployment(nuvla) # Define application module to start. module_api = Module(nuvla)
#
# Ensure complete environment or bail out.
#
if (endpoint is None or api_key is None or api_secret is None
or deployment_id is None):
print(
"missing required configuration information; skipping nuvla integration..."
)
sys.exit()
#
# setup the Nuvla API
#
api = Api(endpoint=endpoint, insecure=True)
api.login_apikey(api_key, api_secret)
# Recover deployment information.
deployment = api.get(deployment_id)
#
# recover the access token
#
token = None
with open('/home/jovyan/token.txt', 'r') as f:
token = f.read()
#
def api(self):
""" Returns an Api object """
return Api(endpoint='https://{}'.format(self.nuvla_endpoint),
insecure=self.nuvla_endpoint_insecure,
reauthenticate=True)
try:
import http.client as http_client
except ImportError:
# Python 2
import httplib as http_client
http_client.HTTPConnection.debuglevel = 1
# You must initialize logging, otherwise you'll not see debug output.
logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)
requests_log = logging.getLogger("requests.packages.urllib3")
requests_log.setLevel(logging.DEBUG)
requests_log.propagate = True
nuvla = Nuvla(endpoint='https://localhost', insecure=True)
res = nuvla.login_password('super', 'supeR8-supeR8')
if not res.ok:
raise Exception('Login failed: {}'.format(res.json().get('message', '')))
dpl_api = Deployment(nuvla)
module_id = 'module/20b6ea7a-ee55-44e7-93e7-7f26b4caef18'
data_records = ['data-record/36fe11cc-7f4f-4ee5-a123-0fafacf4a24b',
'data-record/6803e2e4-1db6-42b4-8b8c-b3273c938366']
data_objects = ['data-object/36fe11cc-7f4f-4ee5-a123-0fafacf4a24b',
'data-object/6803e2e4-1db6-42b4-8b8c-b3273c938366']
data_sets = [{'id': 'data-set/31d8ce24-7567-455e-9d39-4763c1d4010e',
'time-start': '2020-02-14T23:00:00Z',
'time-end': '2020-03-16T22:45:00Z'},
#
# Ensure complete environment or bail out.
#
if (endpoint is None or
api_key is None or
api_secret is None or
deployment_id is None):
print("missing required configuration information; skipping nuvla integration...")
sys.exit()
#
# setup the Nuvla API
#
api = Api(endpoint=endpoint, insecure=True)
api.login_apikey(api_key, api_secret)
# Recover deployment information.
deployment = api.get(deployment_id)
#
# recover the access token
#
token = None
with open('/home/jovyan/token.txt', 'r') as f:
token = f.read()
#
class Base(object):
stop_event = threading.Event()
def __init__(self):
self.args = None
self._init_args_parser()
self._kz: KazooClient = None
self.api: Api = None
self.name = None
self.statsd: StatsClient = None
self._init_logger()
signal.signal(signal.SIGTERM, Base.on_exit)
signal.signal(signal.SIGINT, Base.on_exit)
def _init_args_parser(self):
parser = argparse.ArgumentParser(description='Process Nuvla jobs')
required_args = parser.add_argument_group('required named arguments')
parser.add_argument(
'--zk-hosts', dest='zk_hosts', default=None, nargs='+', metavar='HOST',
help='ZooKeeper list of hosts [localhost:port]. (default: 127.0.0.1:2181)')
parser.add_argument('--api-url', dest='api_url', default='https://nuvla.io', metavar='URL',
help='Nuvla endpoint to connect to (default: https://nuvla.io)')
required_args.add_argument('--api-user', dest='api_user', help='Nuvla Username',
metavar='USERNAME')
required_args.add_argument('--api-pass', dest='api_pass', help='Nuvla Password',
metavar='PASSWORD')
required_args.add_argument('--api-key', dest='api_key', help='Nuvla API Key Id',
metavar='API_KEY')
required_args.add_argument('--api-secret', dest='api_secret', help='Nuvla API Key Secret',
metavar='API_SECRET')
parser.add_argument('--api-insecure', dest='api_insecure', default=False,
action='store_true',
help='Do not check Nuvla certificate')
parser.add_argument('--api-authn-header', dest='api_authn_header', default=None,
help='Set header for internal authentication')
parser.add_argument('--name', dest='name', metavar='NAME', default=None,
help='Base name for this process')
parser.add_argument('--statsd', dest='statsd', metavar='STATSD',
default=None, help=f'StatsD server as host[:{STATSD_PORT}].')
self._set_command_specific_options(parser)
self.args = parser.parse_args()
def _set_command_specific_options(self, parser):
pass
@staticmethod
def _init_logger():
log_format_str = '%(asctime)s - %(levelname)s - %(filename)s:%(lineno)s - %(message)s'
format_log = logging.Formatter(log_format_str)
logger = logging.getLogger()
logger.handlers[0].setFormatter(format_log)
logger.setLevel(logging.INFO)
logging.getLogger('kazoo').setLevel(logging.WARN)
logging.getLogger('elasticsearch').setLevel(logging.WARN)
logging.getLogger('nuvla').setLevel(logging.INFO)
logging.getLogger('urllib3').setLevel(logging.WARN)
def _publish_metric(self, name, value):
if self.statsd:
self.statsd.gauge(name, value)
logging.debug(f'published: {name} {value}')
@staticmethod
def on_exit(signum, frame):
print('\n\nExecution interrupted by the user!')
Base.stop_event.set()
def do_work(self):
raise NotImplementedError()
def execute(self):
self.name = self.args.name if self.args.name is not None else names[
int(random.uniform(1, len(names) - 1))]
# true unless header authentication is used
reauthenticate = self.args.api_authn_header is None
self.api = Api(endpoint=self.args.api_url, insecure=self.args.api_insecure,
persist_cookie=False, reauthenticate=reauthenticate,
authn_header=self.args.api_authn_header)
try:
if self.args.api_authn_header is None:
if self.args.api_key and self.args.api_secret:
response = self.api.login_apikey(self.args.api_key, self.args.api_secret)
else:
response = self.api.login_password(self.args.api_user, self.args.api_pass)
if response.status_code == 403:
raise ConnectionError(
'Login with following user/apikey {} failed!'.format(self.args.api_user))
# Uncomment following lines for manual remote test
# session_id = self.api.current_session()
# self.api.operation(self.api.get(session_id), 'switch-group',
# {'claim': "group/nuvla-admin"})
except ConnectionError as e:
logging.error('Unable to connect to Nuvla endpoint {}! {}'.format(self.api.endpoint, e))
exit(1)
if self.args.zk_hosts:
from kazoo.client import KazooClient, KazooRetry
self._kz = KazooClient(','.join(self.args.zk_hosts),
connection_retry=KazooRetry(max_tries=-1),
command_retry=KazooRetry(max_tries=-1), timeout=30.0)
self._kz.start()
if self.args.statsd:
statsd_hp = self.args.statsd.split(':')
statsd_port = STATSD_PORT
statsd_host = statsd_hp[0]
if len(statsd_hp) > 1:
statsd_port = statsd_hp[1]
try:
self.statsd = StatsClient(host=statsd_host,
port=statsd_port,
prefix=None,
ipv6=False)
except Exception as ex:
logging.error(f'Failed to initialise StatsD client for {self.args.statsd}: {ex}')
self.do_work()
while True:
signal.pause()
