def _register_pack_db(self, pack_name, pack_dir):
content = get_pack_metadata(pack_dir=pack_dir)
# The rules for the pack ref are as follows:
# 1. If ref attribute is available, we used that
# 2. If pack_name is available we use that (this only applies to packs
# 2hich are in sub-directories)
# 2. If attribute is not available, but pack name is and pack name meets the valid name
# criteria, we use that
content['ref'] = get_pack_ref_from_metadata(
metadata=content, pack_directory_name=pack_name)
# Include a list of pack files
pack_file_list = get_file_list(directory=pack_dir,
exclude_patterns=EXCLUDE_FILE_PATTERNS)
content['files'] = pack_file_list
pack_api = PackAPI(**content)
pack_api.validate()
pack_db = PackAPI.to_model(pack_api)
try:
pack_db.id = Pack.get_by_ref(content['ref']).id
except StackStormDBObjectNotFoundError:
LOG.debug('Pack %s not found. Creating new one.', pack_name)
pack_db = Pack.add_or_update(pack_db)
LOG.debug('Pack %s registered.' % (pack_name))
return pack_db
def _get_action_alias_db_by_name(self, name):
"""
Retrieve ActionAlias DB object for the provided alias name.
"""
base_pack_path = self._get_base_pack_path()
pack_yaml_path = os.path.join(base_pack_path, MANIFEST_FILE_NAME)
if os.path.isfile(pack_yaml_path):
# 1. 1st try to infer pack name from pack metadata file
meta_loader = MetaLoader()
pack_metadata = meta_loader.load(pack_yaml_path)
pack = get_pack_ref_from_metadata(metadata=pack_metadata)
else:
# 2. If pack.yaml is not available, fail back to directory name
# Note: For this to work, directory name needs to match pack name
_, pack = os.path.split(base_pack_path)
pack_loader = ContentPackLoader()
registrar = AliasesRegistrar(use_pack_cache=False)
aliases_path = pack_loader.get_content_from_pack(pack_dir=base_pack_path,
content_type='aliases')
aliases = registrar._get_aliases_from_pack(aliases_dir=aliases_path)
for alias_path in aliases:
action_alias_db = registrar._get_action_alias_db(pack=pack,
action_alias=alias_path)
if action_alias_db.name == name:
return action_alias_db
raise ValueError('Alias with name "%s" not found' % (name))
def _get_action_alias_db_by_name(self, name):
"""
Retrieve ActionAlias DB object for the provided alias name.
"""
base_pack_path = self._get_base_pack_path()
pack_yaml_path = os.path.join(base_pack_path, MANIFEST_FILE_NAME)
if os.path.isfile(pack_yaml_path):
# 1. 1st try to infer pack name from pack metadata file
meta_loader = MetaLoader()
pack_metadata = meta_loader.load(pack_yaml_path)
pack = get_pack_ref_from_metadata(metadata=pack_metadata)
else:
# 2. If pack.yaml is not available, fail back to directory name
# Note: For this to work, directory name needs to match pack name
_, pack = os.path.split(base_pack_path)
pack_loader = ContentPackLoader()
registrar = AliasesRegistrar(use_pack_cache=False)
aliases_path = pack_loader.get_content_from_pack(pack_dir=base_pack_path,
content_type='aliases')
aliases = registrar._get_aliases_from_pack(aliases_dir=aliases_path)
for alias_path in aliases:
action_alias_db = registrar._get_action_alias_db(pack=pack,
action_alias=alias_path,
ignore_metadata_file_error=True)
if action_alias_db.name == name:
return action_alias_db
raise ValueError('Alias with name "%s" not found' % (name))
def _register_pack_db(self, pack_name, pack_dir):
content = get_pack_metadata(pack_dir=pack_dir)
# The rules for the pack ref are as follows:
# 1. If ref attribute is available, we used that
# 2. If pack_name is available we use that (this only applies to packs
# 2hich are in sub-directories)
# 2. If attribute is not available, but pack name is and pack name meets the valid name
# criteria, we use that
content['ref'] = get_pack_ref_from_metadata(metadata=content,
pack_directory_name=pack_name)
# Include a list of pack files
pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS)
content['files'] = pack_file_list
pack_api = PackAPI(**content)
pack_api.validate()
pack_db = PackAPI.to_model(pack_api)
try:
pack_db.id = Pack.get_by_ref(content['ref']).id
except StackStormDBObjectNotFoundError:
LOG.debug('Pack %s not found. Creating new one.', pack_name)
pack_db = Pack.add_or_update(pack_db)
LOG.debug('Pack %s registered.' % (pack_name))
return pack_db
def get_pack_ref(pack_dir):
"""
Read pack reference from the metadata file and sanitize it.
"""
metadata = get_pack_metadata(pack_dir=pack_dir)
pack_ref = get_pack_ref_from_metadata(metadata=metadata, pack_directory_name=None)
return pack_ref
def _get_pack_ref(pack_dir):
"""
Read pack name from the metadata file and sanitize it.
"""
metadata = DownloadGitRepoAction._get_pack_metadata(pack_dir=pack_dir)
pack_ref = get_pack_ref_from_metadata(metadata=metadata,
pack_directory_name=None)
return pack_ref
def _get_pack_ref(pack_dir):
"""
Read pack name from the metadata file and sanitize it.
"""
metadata = DownloadGitRepoAction._get_pack_metadata(pack_dir=pack_dir)
pack_ref = get_pack_ref_from_metadata(metadata=metadata,
pack_directory_name=None)
return pack_ref
def get_pack_ref(pack_dir):
"""
Read pack reference from the metadata file and sanitize it.
"""
metadata = get_pack_metadata(pack_dir=pack_dir)
pack_ref = get_pack_ref_from_metadata(metadata=metadata,
pack_directory_name=None)
return pack_ref
def build_index(path_glob, output_path):
result = OrderedDict({
'packs':
OrderedDict(),
'metadata':
OrderedDict({
'generated_ts':
None, # Timestamp of when the file has been generated
'hash':
None # MD5 hash of all the content, useful when mirror the index
})
})
data_hash = hashlib.md5()
path_glob = os.path.expanduser(path_glob)
generator = sorted(glob(path_glob))
counter = 0
for filename in generator:
with open(filename, 'r') as pack:
pack_meta = yaml.load(pack)
pack_name = pack_meta['name']
pack_ref = get_pack_ref_from_metadata(metadata=pack_meta)
sanitized_pack_name = pack_ref
print('Processing pack: %s (%s)' % (pack_name, filename))
pack_meta['repo_url'] = 'https://github.com/%s/%s-%s' % (
EXCHANGE_NAME, EXCHANGE_PREFIX, sanitized_pack_name)
# Note: Key in the index dictionary is ref and not a name
result['packs'][pack_ref] = pack_meta
# Remove any old entry for pack name when we incorrectly used name instead of ref for the
# key
if pack_name != pack_ref:
result['packs'].pop(pack_name, None)
data_hash.update(str(pack_meta))
counter += 1
result['metadata']['generated_ts'] = int(time.time())
result['metadata']['hash'] = data_hash.hexdigest()
output_path = os.path.expanduser(os.path.join(output_path, 'index.json'))
with open(output_path, 'w') as outfile:
json.dump(result,
outfile,
indent=4,
sort_keys=True,
separators=(',', ': '))
print('')
print('Processed %s packs.' % (counter))
print('Index data written to "%s".' % (output_path))
def _register_pack_db(self, pack_name, pack_dir):
pack_name = pack_name or ''
manifest_path = os.path.join(pack_dir, MANIFEST_FILE_NAME)
if not os.path.isfile(manifest_path):
raise ValueError('Pack "%s" is missing %s file' %
(pack_name, MANIFEST_FILE_NAME))
content = self._meta_loader.load(manifest_path)
if not content:
raise ValueError('Pack "%s" metadata file is empty' % (pack_name))
# The rules for the pack ref are as follows:
# 1. If ref attribute is available, we used that
# 2. If pack_name is available we use that (this only applies to packs
# 2hich are in sub-directories)
# 2. If attribute is not available, but pack name is and pack name meets the valid name
# criteria, we use that
content['ref'] = get_pack_ref_from_metadata(
metadata=content, pack_directory_name=pack_name)
# Include a list of pack files
pack_file_list = get_file_list(directory=pack_dir,
exclude_patterns=EXCLUDE_FILE_PATTERNS)
content['files'] = pack_file_list
pack_api = PackAPI(**content)
pack_api.validate()
pack_db = PackAPI.to_model(pack_api)
try:
pack_db.id = Pack.get_by_ref(content['ref']).id
except StackStormDBObjectNotFoundError:
LOG.debug('Pack %s not found. Creating new one.', pack_name)
pack_db = Pack.add_or_update(pack_db)
LOG.debug('Pack %s registered.' % (pack_name))
return pack_db
def _register_pack_db(self, pack_name, pack_dir):
pack_name = pack_name or ''
manifest_path = os.path.join(pack_dir, MANIFEST_FILE_NAME)
if not os.path.isfile(manifest_path):
raise ValueError('Pack "%s" is missing %s file' % (pack_name, MANIFEST_FILE_NAME))
content = self._meta_loader.load(manifest_path)
if not content:
raise ValueError('Pack "%s" metadata file is empty' % (pack_name))
# The rules for the pack ref are as follows:
# 1. If ref attribute is available, we used that
# 2. If pack_name is available we use that (this only applies to packs
# 2hich are in sub-directories)
# 2. If attribute is not available, but pack name is and pack name meets the valid name
# criteria, we use that
content['ref'] = get_pack_ref_from_metadata(metadata=content,
pack_directory_name=pack_name)
# Include a list of pack files
pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS)
content['files'] = pack_file_list
pack_api = PackAPI(**content)
pack_api.validate()
pack_db = PackAPI.to_model(pack_api)
try:
pack_db.id = Pack.get_by_ref(content['ref']).id
except StackStormDBObjectNotFoundError:
LOG.debug('Pack %s not found. Creating new one.', pack_name)
pack_db = Pack.add_or_update(pack_db)
LOG.debug('Pack %s registered.' % (pack_name))
return pack_db
def validate_pack_contains_valid_ref_or_name(pack_meta):
ref = get_pack_ref_from_metadata(metadata=pack_meta)
return ref
def test_get_pack_ref_from_meta_ref_valid(self):
pack_metadata = {"name": "Pack1", "ref": "pack1"}
pack_ref = get_pack_ref_from_metadata(pack_metadata)
self.assertEqual("pack1", pack_ref)
def build_index(path_glob, output_path):
result = OrderedDict({
'packs': OrderedDict(),
'metadata': OrderedDict([
('generated_ts', None), # Timestamp of when the file has been generated
('hash', None) # MD5 hash of all the content, useful when mirror the index
])
})
data_hash = hashlib.md5()
path_glob = os.path.expanduser(path_glob)
generator = sorted(glob(path_glob))
counter = 0
failed_count = 0
for filename in generator:
with open(filename, 'r') as pack:
pack_meta = yaml.safe_load(pack)
pack_name = pack_meta['name']
pack_ref = get_pack_ref_from_metadata(metadata=pack_meta)
sanitized_pack_name = pack_ref
print('Processing pack: %s (%s)' % (pack_name, filename))
pack_meta['repo_url'] = 'https://github.com/%s/%s-%s' % (
EXCHANGE_NAME, EXCHANGE_PREFIX, sanitized_pack_name
)
versions = get_available_versions_for_pack(pack_ref)
if versions is None:
failed_count += 1
if versions is not None:
pack_meta['versions'] = versions
# Note: Key in the index dictionary is ref and not a name
result['packs'][pack_ref] = pack_meta
# Remove any old entry for pack name when we incorrectly used name instead of ref for the
# key
if pack_name != pack_ref:
result['packs'].pop(pack_name, None)
if six.PY2:
data_hash.update(str(pack_meta))
else:
data_hash.update(str(pack_meta).encode('utf-8'))
counter += 1
result['metadata']['generated_ts'] = int(time.time())
result['metadata']['hash'] = data_hash.hexdigest()
output_path = os.path.expanduser(os.path.join(output_path, 'index.json'))
with open(output_path, 'w') as outfile:
json.dump(result, outfile, indent=4, sort_keys=True,
separators=(',', ': '))
failed_message = ''
if failed_count > 0:
failed_message = (
', {failed_count} packs failed to update.\n'
'The GitHub Personal Access Tokens for CircleCI for the pack may '
'need to be refreshed.\n'
'\n'
'See the tools/reset_github_user_token_and_update_circleci.sh script in\n'
' https://github.com/{exchange_name}/ci\n'
'\n'
'If you do not have the necessary GitHub and CircleCI credentials, you\n'
'will need to ask a member of the StackStorm TSC to update the Personal\n'
'Access Token on your behalf.'
).format(failed_count=failed_count, exchange_name=EXCHANGE_NAME)
print('')
print('Processed %s packs%s.' % (counter, failed_message))
print('Index data written to "%s".' % (output_path))
