def _register_pack_db(self, pack_name, pack_dir): content = get_pack_metadata(pack_dir=pack_dir) # The rules for the pack ref are as follows: # 1. If ref attribute is available, we used that # 2. If pack_name is available we use that (this only applies to packs # 2hich are in sub-directories) # 2. If attribute is not available, but pack name is and pack name meets the valid name # criteria, we use that content['ref'] = get_pack_ref_from_metadata( metadata=content, pack_directory_name=pack_name) # Include a list of pack files pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS) content['files'] = pack_file_list pack_api = PackAPI(**content) pack_api.validate() pack_db = PackAPI.to_model(pack_api) try: pack_db.id = Pack.get_by_ref(content['ref']).id except StackStormDBObjectNotFoundError: LOG.debug('Pack %s not found. Creating new one.', pack_name) pack_db = Pack.add_or_update(pack_db) LOG.debug('Pack %s registered.' % (pack_name)) return pack_db
def _get_action_alias_db_by_name(self, name): """ Retrieve ActionAlias DB object for the provided alias name. """ base_pack_path = self._get_base_pack_path() pack_yaml_path = os.path.join(base_pack_path, MANIFEST_FILE_NAME) if os.path.isfile(pack_yaml_path): # 1. 1st try to infer pack name from pack metadata file meta_loader = MetaLoader() pack_metadata = meta_loader.load(pack_yaml_path) pack = get_pack_ref_from_metadata(metadata=pack_metadata) else: # 2. If pack.yaml is not available, fail back to directory name # Note: For this to work, directory name needs to match pack name _, pack = os.path.split(base_pack_path) pack_loader = ContentPackLoader() registrar = AliasesRegistrar(use_pack_cache=False) aliases_path = pack_loader.get_content_from_pack(pack_dir=base_pack_path, content_type='aliases') aliases = registrar._get_aliases_from_pack(aliases_dir=aliases_path) for alias_path in aliases: action_alias_db = registrar._get_action_alias_db(pack=pack, action_alias=alias_path) if action_alias_db.name == name: return action_alias_db raise ValueError('Alias with name "%s" not found' % (name))
def _get_action_alias_db_by_name(self, name): """ Retrieve ActionAlias DB object for the provided alias name. """ base_pack_path = self._get_base_pack_path() pack_yaml_path = os.path.join(base_pack_path, MANIFEST_FILE_NAME) if os.path.isfile(pack_yaml_path): # 1. 1st try to infer pack name from pack metadata file meta_loader = MetaLoader() pack_metadata = meta_loader.load(pack_yaml_path) pack = get_pack_ref_from_metadata(metadata=pack_metadata) else: # 2. If pack.yaml is not available, fail back to directory name # Note: For this to work, directory name needs to match pack name _, pack = os.path.split(base_pack_path) pack_loader = ContentPackLoader() registrar = AliasesRegistrar(use_pack_cache=False) aliases_path = pack_loader.get_content_from_pack(pack_dir=base_pack_path, content_type='aliases') aliases = registrar._get_aliases_from_pack(aliases_dir=aliases_path) for alias_path in aliases: action_alias_db = registrar._get_action_alias_db(pack=pack, action_alias=alias_path, ignore_metadata_file_error=True) if action_alias_db.name == name: return action_alias_db raise ValueError('Alias with name "%s" not found' % (name))
def _register_pack_db(self, pack_name, pack_dir): content = get_pack_metadata(pack_dir=pack_dir) # The rules for the pack ref are as follows: # 1. If ref attribute is available, we used that # 2. If pack_name is available we use that (this only applies to packs # 2hich are in sub-directories) # 2. If attribute is not available, but pack name is and pack name meets the valid name # criteria, we use that content['ref'] = get_pack_ref_from_metadata(metadata=content, pack_directory_name=pack_name) # Include a list of pack files pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS) content['files'] = pack_file_list pack_api = PackAPI(**content) pack_api.validate() pack_db = PackAPI.to_model(pack_api) try: pack_db.id = Pack.get_by_ref(content['ref']).id except StackStormDBObjectNotFoundError: LOG.debug('Pack %s not found. Creating new one.', pack_name) pack_db = Pack.add_or_update(pack_db) LOG.debug('Pack %s registered.' % (pack_name)) return pack_db
def get_pack_ref(pack_dir): """ Read pack reference from the metadata file and sanitize it. """ metadata = get_pack_metadata(pack_dir=pack_dir) pack_ref = get_pack_ref_from_metadata(metadata=metadata, pack_directory_name=None) return pack_ref
def _get_pack_ref(pack_dir): """ Read pack name from the metadata file and sanitize it. """ metadata = DownloadGitRepoAction._get_pack_metadata(pack_dir=pack_dir) pack_ref = get_pack_ref_from_metadata(metadata=metadata, pack_directory_name=None) return pack_ref
def build_index(path_glob, output_path): result = OrderedDict({ 'packs': OrderedDict(), 'metadata': OrderedDict({ 'generated_ts': None, # Timestamp of when the file has been generated 'hash': None # MD5 hash of all the content, useful when mirror the index }) }) data_hash = hashlib.md5() path_glob = os.path.expanduser(path_glob) generator = sorted(glob(path_glob)) counter = 0 for filename in generator: with open(filename, 'r') as pack: pack_meta = yaml.load(pack) pack_name = pack_meta['name'] pack_ref = get_pack_ref_from_metadata(metadata=pack_meta) sanitized_pack_name = pack_ref print('Processing pack: %s (%s)' % (pack_name, filename)) pack_meta['repo_url'] = 'https://github.com/%s/%s-%s' % ( EXCHANGE_NAME, EXCHANGE_PREFIX, sanitized_pack_name) # Note: Key in the index dictionary is ref and not a name result['packs'][pack_ref] = pack_meta # Remove any old entry for pack name when we incorrectly used name instead of ref for the # key if pack_name != pack_ref: result['packs'].pop(pack_name, None) data_hash.update(str(pack_meta)) counter += 1 result['metadata']['generated_ts'] = int(time.time()) result['metadata']['hash'] = data_hash.hexdigest() output_path = os.path.expanduser(os.path.join(output_path, 'index.json')) with open(output_path, 'w') as outfile: json.dump(result, outfile, indent=4, sort_keys=True, separators=(',', ': ')) print('') print('Processed %s packs.' % (counter)) print('Index data written to "%s".' % (output_path))
def _register_pack_db(self, pack_name, pack_dir): pack_name = pack_name or '' manifest_path = os.path.join(pack_dir, MANIFEST_FILE_NAME) if not os.path.isfile(manifest_path): raise ValueError('Pack "%s" is missing %s file' % (pack_name, MANIFEST_FILE_NAME)) content = self._meta_loader.load(manifest_path) if not content: raise ValueError('Pack "%s" metadata file is empty' % (pack_name)) # The rules for the pack ref are as follows: # 1. If ref attribute is available, we used that # 2. If pack_name is available we use that (this only applies to packs # 2hich are in sub-directories) # 2. If attribute is not available, but pack name is and pack name meets the valid name # criteria, we use that content['ref'] = get_pack_ref_from_metadata( metadata=content, pack_directory_name=pack_name) # Include a list of pack files pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS) content['files'] = pack_file_list pack_api = PackAPI(**content) pack_api.validate() pack_db = PackAPI.to_model(pack_api) try: pack_db.id = Pack.get_by_ref(content['ref']).id except StackStormDBObjectNotFoundError: LOG.debug('Pack %s not found. Creating new one.', pack_name) pack_db = Pack.add_or_update(pack_db) LOG.debug('Pack %s registered.' % (pack_name)) return pack_db
def _register_pack_db(self, pack_name, pack_dir): pack_name = pack_name or '' manifest_path = os.path.join(pack_dir, MANIFEST_FILE_NAME) if not os.path.isfile(manifest_path): raise ValueError('Pack "%s" is missing %s file' % (pack_name, MANIFEST_FILE_NAME)) content = self._meta_loader.load(manifest_path) if not content: raise ValueError('Pack "%s" metadata file is empty' % (pack_name)) # The rules for the pack ref are as follows: # 1. If ref attribute is available, we used that # 2. If pack_name is available we use that (this only applies to packs # 2hich are in sub-directories) # 2. If attribute is not available, but pack name is and pack name meets the valid name # criteria, we use that content['ref'] = get_pack_ref_from_metadata(metadata=content, pack_directory_name=pack_name) # Include a list of pack files pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS) content['files'] = pack_file_list pack_api = PackAPI(**content) pack_api.validate() pack_db = PackAPI.to_model(pack_api) try: pack_db.id = Pack.get_by_ref(content['ref']).id except StackStormDBObjectNotFoundError: LOG.debug('Pack %s not found. Creating new one.', pack_name) pack_db = Pack.add_or_update(pack_db) LOG.debug('Pack %s registered.' % (pack_name)) return pack_db
def validate_pack_contains_valid_ref_or_name(pack_meta): ref = get_pack_ref_from_metadata(metadata=pack_meta) return ref
def test_get_pack_ref_from_meta_ref_valid(self): pack_metadata = {"name": "Pack1", "ref": "pack1"} pack_ref = get_pack_ref_from_metadata(pack_metadata) self.assertEqual("pack1", pack_ref)
def build_index(path_glob, output_path): result = OrderedDict({ 'packs': OrderedDict(), 'metadata': OrderedDict([ ('generated_ts', None), # Timestamp of when the file has been generated ('hash', None) # MD5 hash of all the content, useful when mirror the index ]) }) data_hash = hashlib.md5() path_glob = os.path.expanduser(path_glob) generator = sorted(glob(path_glob)) counter = 0 failed_count = 0 for filename in generator: with open(filename, 'r') as pack: pack_meta = yaml.safe_load(pack) pack_name = pack_meta['name'] pack_ref = get_pack_ref_from_metadata(metadata=pack_meta) sanitized_pack_name = pack_ref print('Processing pack: %s (%s)' % (pack_name, filename)) pack_meta['repo_url'] = 'https://github.com/%s/%s-%s' % ( EXCHANGE_NAME, EXCHANGE_PREFIX, sanitized_pack_name ) versions = get_available_versions_for_pack(pack_ref) if versions is None: failed_count += 1 if versions is not None: pack_meta['versions'] = versions # Note: Key in the index dictionary is ref and not a name result['packs'][pack_ref] = pack_meta # Remove any old entry for pack name when we incorrectly used name instead of ref for the # key if pack_name != pack_ref: result['packs'].pop(pack_name, None) if six.PY2: data_hash.update(str(pack_meta)) else: data_hash.update(str(pack_meta).encode('utf-8')) counter += 1 result['metadata']['generated_ts'] = int(time.time()) result['metadata']['hash'] = data_hash.hexdigest() output_path = os.path.expanduser(os.path.join(output_path, 'index.json')) with open(output_path, 'w') as outfile: json.dump(result, outfile, indent=4, sort_keys=True, separators=(',', ': ')) failed_message = '' if failed_count > 0: failed_message = ( ', {failed_count} packs failed to update.\n' 'The GitHub Personal Access Tokens for CircleCI for the pack may ' 'need to be refreshed.\n' '\n' 'See the tools/reset_github_user_token_and_update_circleci.sh script in\n' ' https://github.com/{exchange_name}/ci\n' '\n' 'If you do not have the necessary GitHub and CircleCI credentials, you\n' 'will need to ask a member of the StackStorm TSC to update the Personal\n' 'Access Token on your behalf.' ).format(failed_count=failed_count, exchange_name=EXCHANGE_NAME) print('') print('Processed %s packs%s.' % (counter, failed_message)) print('Index data written to "%s".' % (output_path))