Exemplo n.º 1
0
def add_zeppelin_notebooks():
    '''Add all Zeppelin notes to the current Zeppelin instalation
  
  Looks inside ``configuration/zeppelin/notes`` for any ``*.json`` files.
  
  Each JSON file will be attempted to be added to the current Zeppelin instance
  
  Returns:
    bool: True is every Note was added successfully. True is returned on an empty directory. False otherwise.
  
  
  '''
    logger.info('Adding zeppelin notebooks to installation')
    all_success = True
    note_dir = config.get_conf_dir() + 'zeppelin/notes'

    # First need to authenticate to Zeppelin and retrieve JSESSIONID
    conf = config.read_config('global.conf')['ZEPPELIN']
    session = get_zeppelin_session(conf['username'], conf['password'])

    for item in os.listdir(note_dir):
        logger.debug('Found item in zeppelin/notes: ' + item)
        item_path = note_dir + '/' + item
        if os.path.isfile(item_path) and str(item).endswith('.json'):
            logger.info('POSTing ' + item + ' to Zeppelin')
            result = post_notebook(item_path, session)
            if not result:
                logger.error(
                    'Not all notebooks were added to Zeppelin successfully')
                all_success = False
    return all_success
def add_zeppelin_notebooks():
  '''Add all Zeppelin notes to the current Zeppelin instalation
  
  Looks inside ``configuration/zeppelin/notes`` for any ``*.json`` files.
  
  Each JSON file will be attempted to be added to the current Zeppelin instance
  
  Returns:
    bool: True is every Note was added successfully. True is returned on an empty directory. False otherwise.
  
  
  '''
  logger.info('Adding zeppelin notebooks to installation')
  all_success = True
  note_dir = config.get_conf_dir() + 'zeppelin/notes'
  
  # First need to authenticate to Zeppelin and retrieve JSESSIONID
  conf = config.read_config('global.conf')['ZEPPELIN']
  session = get_zeppelin_session(conf['username'], conf['password'])
  
  for item in os.listdir(note_dir):
    logger.debug('Found item in zeppelin/notes: ' + item)
    item_path = note_dir + '/' + item
    if os.path.isfile(item_path) and str(item).endswith('.json'):
      logger.info('POSTing ' + item + ' to Zeppelin')
      result = post_notebook(item_path, session)
      if not result:
        logger.error('Not all notebooks were added to Zeppelin successfully')
        all_success = False
  return all_success
def add_nifi_templates():
	logger.info('Attempting to add NiFi templates')
	all_success = True
	template_dir = config.get_conf_dir() + 'nifi/templates'
	for item in os.listdir(template_dir):
		logger.debug('Found item in nifi/templates: ' + item)
		item_path = template_dir + '/' + item
		if os.path.isfile(item_path) and str(item).endswith('.xml'):
			result = post_template(item_path)
			if not result:
				logger.error('Not all templates were added to NiFi successfully')
				all_success = False
	return all_success
def add_zeppelin_notebooks():
	logger.info('Adding zeppelin notebooks to installation')
	all_success = True
	note_dir = config.get_conf_dir() + 'zeppelin/notes'
	for item in os.listdir(note_dir):
		logger.debug('Found item in zeppelin/notes: ' + item)
		item_path = note_dir + '/' + item
		if os.path.isfile(item_path) and str(item).endswith('.json'):
			logger.info('POSTing ' + item + ' to Zeppelin')
			result = post_notebook(item_path)
			if not result:
				logger.error('Not all notebooks were added to Zeppelin successfully')
				all_success = False
	return all_success
Exemplo n.º 5
0
    def check_schema(self, schema):
        path = config.get_conf_dir() + schema

        with open(path) as data_file:
            conf = json.load(data_file)

            if not type(conf) == list:
                logger.error("JSON Schema not formatted properly")
                raise TypeError("Root of JSON Schema is not a list")

            for field in conf:
                if not "fieldName" in field:
                    logger.error("fieldName not found in schema at " + path)
                    raise KeyError("Could not find 'fieldName' in field of schema: " + schema)

                if not "type" in field:
                    logger.error("type not found in schema at " + path)
                    raise KeyError("Could not find 'type' in field of schema: " + schema)

                field_type = field["type"]
                logger.debug("Attempting to register datum with type: " + str(field_type))
                datum = AbstractDatum(field)
                if not datum.field_name in self.field_names:
                    self.field_names.append(datum.field_name)
                    logger.debug("Added datum to field set with type: " + str(field_type))
                else:
                    raise ValueError("Cannot have duplicate field names")
                if "string" == field_type:
                    datum = StringDatum(field)
                elif "int" == field_type:
                    datum = IntDatum(field)
                elif "decimal" == field_type:
                    datum = DecimalDatum(field)
                elif "map" == field_type:
                    datum = MapDatum(field)
                elif "boolean" == field_type:
                    datum = BooleanDatum(field)
                else:
                    raise RuntimeError(
                        "Field type was not found. Please change the field type or implement a new datum"
                    )

                datum.check()  # Check to make sure the field has necessary attributes
                logger.info("Datum passed check successfully")
                self.data_fields.append(datum)
def add_nifi_templates():
  '''Add any number of NiFi templates to a running NiFi installation via the REST API
  
  Looks inside ``configuration/nifi/templates`` for XML templates to add to the installation
  
  Returns:
    bool: True if we were successful in uploading every template. False otherwise. An empty directory results in True
  '''
  logger.info('Attempting to add NiFi templates')
  all_success = True
  template_dir = config.get_conf_dir() + 'nifi/templates'
  for item in os.listdir(template_dir):
    logger.debug('Found item in nifi/templates: ' + item)
    item_path = template_dir + '/' + item
    if os.path.isfile(item_path) and str(item).endswith('.xml'):
      result = post_template(item_path)
      if not result:
        logger.error('Not all templates were added to NiFi successfully')
        all_success = False
  return all_success
Exemplo n.º 7
0
def add_nifi_templates():
    '''Add any number of NiFi templates to a running NiFi installation via the REST API
  
  Looks inside ``configuration/nifi/templates`` for XML templates to add to the installation
  
  Returns:
    bool: True if we were successful in uploading every template. False otherwise. An empty directory results in True
  '''
    logger.info('Attempting to add NiFi templates')
    all_success = True
    template_dir = config.get_conf_dir() + 'nifi/templates'
    for item in os.listdir(template_dir):
        logger.debug('Found item in nifi/templates: ' + item)
        item_path = template_dir + '/' + item
        if os.path.isfile(item_path) and str(item).endswith('.xml'):
            result = post_template(item_path)
            if not result:
                logger.error(
                    'Not all templates were added to NiFi successfully')
                all_success = False
    return all_success