def add_zeppelin_notebooks():
'''Add all Zeppelin notes to the current Zeppelin instalation
Looks inside ``configuration/zeppelin/notes`` for any ``*.json`` files.
Each JSON file will be attempted to be added to the current Zeppelin instance
Returns:
bool: True is every Note was added successfully. True is returned on an empty directory. False otherwise.
'''
logger.info('Adding zeppelin notebooks to installation')
all_success = True
note_dir = config.get_conf_dir() + 'zeppelin/notes'
# First need to authenticate to Zeppelin and retrieve JSESSIONID
conf = config.read_config('global.conf')['ZEPPELIN']
session = get_zeppelin_session(conf['username'], conf['password'])
for item in os.listdir(note_dir):
logger.debug('Found item in zeppelin/notes: ' + item)
item_path = note_dir + '/' + item
if os.path.isfile(item_path) and str(item).endswith('.json'):
logger.info('POSTing ' + item + ' to Zeppelin')
result = post_notebook(item_path, session)
if not result:
logger.error(
'Not all notebooks were added to Zeppelin successfully')
all_success = False
return all_success
def add_zeppelin_notebooks():
'''Add all Zeppelin notes to the current Zeppelin instalation
Looks inside ``configuration/zeppelin/notes`` for any ``*.json`` files.
Each JSON file will be attempted to be added to the current Zeppelin instance
Returns:
bool: True is every Note was added successfully. True is returned on an empty directory. False otherwise.
'''
logger.info('Adding zeppelin notebooks to installation')
all_success = True
note_dir = config.get_conf_dir() + 'zeppelin/notes'
# First need to authenticate to Zeppelin and retrieve JSESSIONID
conf = config.read_config('global.conf')['ZEPPELIN']
session = get_zeppelin_session(conf['username'], conf['password'])
for item in os.listdir(note_dir):
logger.debug('Found item in zeppelin/notes: ' + item)
item_path = note_dir + '/' + item
if os.path.isfile(item_path) and str(item).endswith('.json'):
logger.info('POSTing ' + item + ' to Zeppelin')
result = post_notebook(item_path, session)
if not result:
logger.error('Not all notebooks were added to Zeppelin successfully')
all_success = False
return all_success
def add_nifi_templates():
logger.info('Attempting to add NiFi templates')
all_success = True
template_dir = config.get_conf_dir() + 'nifi/templates'
for item in os.listdir(template_dir):
logger.debug('Found item in nifi/templates: ' + item)
item_path = template_dir + '/' + item
if os.path.isfile(item_path) and str(item).endswith('.xml'):
result = post_template(item_path)
if not result:
logger.error('Not all templates were added to NiFi successfully')
all_success = False
return all_success
def add_zeppelin_notebooks():
logger.info('Adding zeppelin notebooks to installation')
all_success = True
note_dir = config.get_conf_dir() + 'zeppelin/notes'
for item in os.listdir(note_dir):
logger.debug('Found item in zeppelin/notes: ' + item)
item_path = note_dir + '/' + item
if os.path.isfile(item_path) and str(item).endswith('.json'):
logger.info('POSTing ' + item + ' to Zeppelin')
result = post_notebook(item_path)
if not result:
logger.error('Not all notebooks were added to Zeppelin successfully')
all_success = False
return all_success
def check_schema(self, schema):
path = config.get_conf_dir() + schema
with open(path) as data_file:
conf = json.load(data_file)
if not type(conf) == list:
logger.error("JSON Schema not formatted properly")
raise TypeError("Root of JSON Schema is not a list")
for field in conf:
if not "fieldName" in field:
logger.error("fieldName not found in schema at " + path)
raise KeyError("Could not find 'fieldName' in field of schema: " + schema)
if not "type" in field:
logger.error("type not found in schema at " + path)
raise KeyError("Could not find 'type' in field of schema: " + schema)
field_type = field["type"]
logger.debug("Attempting to register datum with type: " + str(field_type))
datum = AbstractDatum(field)
if not datum.field_name in self.field_names:
self.field_names.append(datum.field_name)
logger.debug("Added datum to field set with type: " + str(field_type))
else:
raise ValueError("Cannot have duplicate field names")
if "string" == field_type:
datum = StringDatum(field)
elif "int" == field_type:
datum = IntDatum(field)
elif "decimal" == field_type:
datum = DecimalDatum(field)
elif "map" == field_type:
datum = MapDatum(field)
elif "boolean" == field_type:
datum = BooleanDatum(field)
else:
raise RuntimeError(
"Field type was not found. Please change the field type or implement a new datum"
)
datum.check() # Check to make sure the field has necessary attributes
logger.info("Datum passed check successfully")
self.data_fields.append(datum)
def add_nifi_templates():
'''Add any number of NiFi templates to a running NiFi installation via the REST API
Looks inside ``configuration/nifi/templates`` for XML templates to add to the installation
Returns:
bool: True if we were successful in uploading every template. False otherwise. An empty directory results in True
'''
logger.info('Attempting to add NiFi templates')
all_success = True
template_dir = config.get_conf_dir() + 'nifi/templates'
for item in os.listdir(template_dir):
logger.debug('Found item in nifi/templates: ' + item)
item_path = template_dir + '/' + item
if os.path.isfile(item_path) and str(item).endswith('.xml'):
result = post_template(item_path)
if not result:
logger.error('Not all templates were added to NiFi successfully')
all_success = False
return all_success
def add_nifi_templates():
'''Add any number of NiFi templates to a running NiFi installation via the REST API
Looks inside ``configuration/nifi/templates`` for XML templates to add to the installation
Returns:
bool: True if we were successful in uploading every template. False otherwise. An empty directory results in True
'''
logger.info('Attempting to add NiFi templates')
all_success = True
template_dir = config.get_conf_dir() + 'nifi/templates'
for item in os.listdir(template_dir):
logger.debug('Found item in nifi/templates: ' + item)
item_path = template_dir + '/' + item
if os.path.isfile(item_path) and str(item).endswith('.xml'):
result = post_template(item_path)
if not result:
logger.error(
'Not all templates were added to NiFi successfully')
all_success = False
return all_success
