def test_save(tmpdir): temp_dir = tmpdir.mkdir('mazer_collection_archive_manifest_unit_test') file_name = "example_artifact_manifest1.yml" temp_file = temp_dir.join(file_name) test_data_path = os.path.join(os.path.dirname(__file__), '%s' % file_name) with open(test_data_path, 'r') as data_fd: res = collection_artifact_manifest.load(data_fd) log.debug('temp_file.strpath: %s', temp_file.strpath) yaml_persist.save(res, temp_file.strpath) # open the save()'ed file and load a new collection_artifact_manifest from it with open(temp_file.strpath, 'r') as read_fd: reloaded = collection_artifact_manifest.load(read_fd) log.debug('reloaded: %s', reloaded) # verify the object loaded from known example matches the example after # a save()/load() cycle assert reloaded == res # read the file again and log the file contents read_fd.seek(0) buf = read_fd.read() log.debug('buf: %s', buf)
def test_load(): file_name = "example_artifact_manifest1.yml" test_data_path = os.path.join(os.path.dirname(__file__), '%s' % file_name) with open(test_data_path, 'r') as data_fd: res = collection_artifact_manifest.load(data_fd) log.debug('res: %s', res) assert isinstance(res, CollectionArtifactManifest)
def load_from_archive(repository_archive, namespace=None, installed=True): repo_tarfile = repository_archive.tar_file archive_path = repository_archive.info.archive_path manifest_filename = os.path.join( collection_artifact_manifest.COLLECTION_MANIFEST_FILENAME) manifest_data = None log.debug('Trying to extract %s from %s', manifest_filename, archive_path) try: mfd = repo_tarfile.extractfile(manifest_filename) if mfd: manifest_data = collection_artifact_manifest.load(mfd) log.debug('md: %s', manifest_data) log.debug('md.collection_info: %s', manifest_data.collection_info) log.debug('manifest_data.collection_info.name: %s', manifest_data.collection_info.name) except KeyError as e: log.warning('No %s found in archive: %s (Error: %s)', manifest_filename, archive_path, e) if not manifest_data: raise exceptions.GalaxyArchiveError( 'No collection manifest (%s) found in %s' % (collection_artifact_manifest.COLLECTION_MANIFEST_FILENAME, archive_path), archive_path=archive_path) col_info = manifest_data.collection_info log.debug('col_info: %s', col_info) # if we specify a namespace, use it otherwise use the info from the manifest col_info repo_spec = RepositorySpec( namespace=namespace or col_info.namespace, name=col_info.name, version=col_info.version, spec_string=archive_path, # fetch_method=None, src=archive_path) log.debug('repo spec from %s: %r', archive_path, repo_spec) requirements_list = requirements.from_dependencies_dict( col_info.dependencies, repository_spec=repo_spec) repository = Repository( repository_spec=repo_spec, path=None, installed=installed, requirements=requirements_list, ) log.debug('repository: %s', repository) return repository
def _publish(galaxy_context, archive_path, publish_api_key=None, display_callback=None): results = {'errors': [], 'success': True} archive = tarfile.open(archive_path, 'r') top_dir = os.path.commonprefix(archive.getnames()) manifest_path = os.path.join( top_dir, collection_artifact_manifest.COLLECTION_MANIFEST_FILENAME) try: manifest_file = archive.extractfile(manifest_path) except tarfile.TarError as exc: raise exceptions.GalaxyPublishError(str(exc), archive_path=archive_path) try: manifest = collection_artifact_manifest.load(manifest_file) except Exception as exc: raise exceptions.GalaxyPublishError(str(exc), archive_path=archive_path) # display_callback('Creating publish task for %s from artifact archive %s' % # (manifest.collection_info.label, archive_path)) # display_callback(json.dumps(attr.asdict(manifest.collection_info))) api = GalaxyAPI(galaxy_context) collection_name = manifest.collection_info.name data = { 'sha256': _get_file_checksum(archive_path), 'name': collection_name, 'version': manifest.collection_info.version } log.debug("Publishing file %s with data: %s" % (archive_path, json.dumps(data))) b_response_body = api.publish_file(data, archive_path, publish_api_key) response_body = to_text(b_response_body, errors='surrogate_or_strict') # TODO: try/except on json error but let bubble up for now response_data = json.loads(response_body) results['response_data'] = response_data return results
def test_load(): file_name = "example_artifact_manifest1.yml" test_data_path = os.path.join(os.path.dirname(__file__), '%s' % file_name) with open(test_data_path, 'r') as data_fd: res = collection_artifact_manifest.load(data_fd) log.debug('res: %s', res) assert isinstance(res, CollectionArtifactManifest) assert isinstance(res.files, list) assert isinstance(res.files[0], CollectionArtifactFile) assert res.files[0].name == 'roles/some_role/defaults/main.yml' assert res.files[0].ftype == 'file' assert res.files[0].chksum_type == 'sha256' assert res.files[0].chksum_sha256 == 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'
def _publish(galaxy_context, archive_path, display_callback=None): results = {'errors': [], 'success': True} archive = tarfile.open(archive_path, 'r') top_dir = os.path.commonprefix(archive.getnames()) manifest_path = os.path.join( top_dir, collection_artifact_manifest.COLLECTION_MANIFEST_FILENAME) try: manifest_file = archive.extractfile(manifest_path) except tarfile.TarError as exc: raise exceptions.GalaxyPublishError(str(exc), archive_path=archive_path) try: manifest = collection_artifact_manifest.load(manifest_file) except Exception as exc: raise exceptions.GalaxyPublishError(str(exc), archive_path=archive_path) display_callback(json.dumps(attr.asdict(manifest.collection_info))) api = GalaxyAPI(galaxy_context) collection_name = manifest.collection_info.name namespace_name = manifest.collection_info.namespace log.debug("Attempting to fetch Galaxy namespace %s" % namespace_name) namespace = api.fetch_namespace(namespace_name) namespace_id = namespace.get('id') data = { 'sha256': _get_file_checksum(archive_path), 'name': collection_name, 'version': manifest.collection_info.version } log.debug("Publishing file %s with data: %s" % (archive_path, json.dumps(data))) api.publish_file(namespace_id, data, archive_path) return results
def load_from_archive(repository_archive, namespace=None, installed=True): repo_tarfile = repository_archive.tar_file archive_path = repository_archive.info.archive_path # path_name = os.path.join(content_dir, namespace, name) path_name = repository_archive.info.top_dir manifest_filename = os.path.join( path_name, collection_artifact_manifest.COLLECTION_MANIFEST_FILENAME) manifest_data = None log.debug('Trying to extract %s from %s', manifest_filename, archive_path) try: mfd = repo_tarfile.extractfile(manifest_filename) if mfd: manifest_data = collection_artifact_manifest.load(mfd) log.debug('md: %s', manifest_data) log.debug('md.collection_info: %s', manifest_data.collection_info) log.debug('manifest_data.collection_info.name: %s', manifest_data.collection_info.name) except KeyError as e: log.warning('No %s found in archive: %s (Error: %s)', manifest_filename, archive_path, e) # load galaxy.yml galaxy_filename = os.path.join(path_name, collection_info.COLLECTION_INFO_FILENAME) collection_info_data = None try: gfd = repo_tarfile.extractfile(galaxy_filename) if gfd: collection_info_data = collection_info.load(gfd) except KeyError as e: log.warning('No %s found in archive: %s - %s', galaxy_filename, archive_path, e) # log.debug('No galaxy.yml collection info found for collection %s.%s: %s', namespace, name, e) # TODO/FIXME: what takes precedence? # - the dir name in the archive that a collection lives in ~/.ansible/content/my_ns/my_name # - Or the namespace/name from galaxy.yml? # log.debug('collection_info_data: %s', collection_info_data) col_info = None if manifest_data: col_info = manifest_data.collection_info log.debug('md.col_info: %s', col_info) elif collection_info_data: col_info = collection_info_data else: raise exceptions.GalaxyArchiveError( 'No galaxy collection info or manifest found in %s', archive_path) log.debug('col_info: %s', col_info) # FIXME: change collectionInfo to have separate name/namespace so we dont have to 'parse' the name # repo_spec = repository_spec.repository_spec_from_string(col_info.name, namespace_override=namespace) # spec_data = repository_spec_parse.parse_string(col_info.name) # log.debug('spec_data: %s', spec_data) # log.debug('repo_spec: %s', repo_spec) # Build a repository_spec of the repo now so we can pass it things like requirements.load() # that need to know what requires something # if we specify a namespace, use it otherwise use the info from galaxy.yml repo_spec = RepositorySpec( namespace=namespace or col_info.namespace, name=col_info.name, version=col_info.version, spec_string=archive_path, # fetch_method=None, src=archive_path) log.debug('repo spec from %s: %r', archive_path, repo_spec) requirements_list = [] requirements_list = requirements.from_requirement_spec_strings( col_info.dependencies, repository_spec=repo_spec) repository = Repository( repository_spec=repo_spec, path=None, installed=installed, requirements=requirements_list, # Assuming this is a collection artifact, FIXME if we support role artifacts dependencies=[]) log.debug('repository: %s', repository) return repository
def load_from_dir(content_dir, namespace, name, installed=True): # TODO: or artifact path_name = os.path.join(content_dir, namespace, name) # TODO: add trad role or collection detection rules here # Or possibly earlier so we could call 'collection' loading # code/class or trad-role-as-collection loading code/class # and avoid intermingly the impls. # Maybe: # if more than one role in roles/ -> collection if not os.path.isdir(path_name): log.debug( 'The directory %s does not exist, unable to load a Repository from it', path_name) return None requirements_list = [] # Now look for any install_info for the repository install_info_data = None install_info_filename = os.path.join(path_name, 'meta/.galaxy_install_info') try: with open(install_info_filename, 'r') as ifd: install_info_data = install_info.load(ifd) except EnvironmentError as e: log.warning( 'Unable to find or load meta/.galaxy_install_info for repository %s.%s: %s', namespace, name, e) # TODO: figure out what to do if the version from install_info conflicts with version # from galaxy.yml etc. install_info_version = getattr(install_info_data, 'version', None) # Try to load a MANIFEST.json if we have one manifest_filename = os.path.join( path_name, collection_artifact_manifest.COLLECTION_MANIFEST_FILENAME) manifest_data = None try: with open(manifest_filename, 'r') as mfd: manifest_data = collection_artifact_manifest.load(mfd) except EnvironmentError: # log.debug('No galaxy.yml collection info found for collection %s.%s: %s', namespace, name, e) pass # load galaxy.yml galaxy_filename = os.path.join(path_name, collection_info.COLLECTION_INFO_FILENAME) collection_info_data = None try: with open(galaxy_filename, 'r') as gfd: collection_info_data = collection_info.load(gfd) except EnvironmentError: # log.debug('No galaxy.yml collection info found for collection %s.%s: %s', namespace, name, e) pass # Now try the repository as a role-as-collection # FIXME: For a repository with one role that matches the collection name and doesn't # have a galaxy.yml, that's indistinguishable from a role-as-collection # FIXME: But in theory, if there is more than one role in roles/, we should skip this role_meta_main_filename = os.path.join(path_name, 'roles', name, 'meta', 'main.yml') role_meta_main = None role_name = '%s.%s' % (namespace, name) try: with open(role_meta_main_filename, 'r') as rmfd: # FIXME: kluge to avoid circular import on py2 # repository->role_metadata->dependencies->repository_spec->repository (loop) # repository->requirements->repository_spec->repository (loop) from ansible_galaxy import role_metadata role_meta_main = role_metadata.load(rmfd, role_name=role_name) except EnvironmentError: # log.debug('No meta/main.yml was loaded for repository %s.%s: %s', namespace, name, e) pass # Prefer version from install_info, but for a editable installed, there may be only galaxy version installed_version = install_info_version if manifest_data: installed_version = manifest_data.collection_info.version elif collection_info_data: installed_version = collection_info_data.version # if role_meta_main: # installed_version = installed_version or role_meta_main.version # TODO/FIXME: what takes precedence? # - the dir names a collection lives in ~/.ansible/content/my_ns/my_name # - Or the namespace/name from galaxy.yml? # log.debug('collection_info_data: %s', collection_info_data) # Build a repository_spec of the repo now so we can pass it things like requirements.load() # that need to know what requires something repository_spec = RepositorySpec(namespace=namespace, name=name, version=installed_version) # The current galaxy.yml 'dependencies' are actually 'requirements' in ansible/ansible terminology # (ie, install-time) if collection_info_data: collection_requires = requirements.from_dependencies_dict( collection_info_data.dependencies, repository_spec=repository_spec) requirements_list.extend(collection_requires) # TODO: add requirements loaded from galaxy.yml # TODO: should the requirements in galaxy.yml be plain strings or dicts? # TODO: should there be requirements in galaxy.yml at all? in liue of requirements.yml # collection_info_requirements = [] requirements_filename = os.path.join(path_name, 'requirements.yml') try: with open(requirements_filename, 'r') as rfd: requirements_list.extend( requirements.load(rfd, repository_spec=repository_spec)) except EnvironmentError: # log.debug('No requirements.yml was loaded for repository %s.%s: %s', namespace, name, e) pass # TODO: if there are other places to load dependencies (ie, runtime deps) we will need # to load them and combine them with role_depenency_specs role_dependency_specs = [] if role_meta_main: role_dependency_specs = role_meta_main.dependencies repository = Repository(repository_spec=repository_spec, path=path_name, installed=installed, requirements=requirements_list, dependencies=role_dependency_specs) log.debug('Repository %s loaded from %s', repository.repository_spec.label, path_name) return repository
def load_from_dir(content_dir, namespace_path, namespace, name, installed=True): path_name = os.path.join(namespace_path, name) log.debug('Loading repository %s.%s from path: %s', namespace, name, path_name) if not os.path.isdir(path_name): log.debug( 'The directory %s does not exist, unable to load a Repository from it', path_name) return None # Now look for any install_info for the repository install_info_data = None install_info_filename = os.path.join(path_name, 'meta/.galaxy_install_info') try: with open(install_info_filename, 'r') as ifd: install_info_data = install_info.load(ifd) except EnvironmentError as e: log.warning( 'Unable to find or load meta/.galaxy_install_info for repository %s.%s: %s', namespace, name, e) # TODO: figure out what to do if the version from install_info conflicts with version # from galaxy.yml etc. install_info_version = getattr(install_info_data, 'version', None) # Try to load a MANIFEST.json if we have one manifest_filename = os.path.join( path_name, collection_artifact_manifest.COLLECTION_MANIFEST_FILENAME) manifest_data = None try: with open(manifest_filename, 'r') as mfd: manifest_data = collection_artifact_manifest.load(mfd) except EnvironmentError: # log.debug('No galaxy.yml collection info found for collection %s.%s: %s', namespace, name, e) pass # # TODO/FIXME: do we even need to load file_manifest here? # file_manifest_filename = os.path.join(path_name, collection_artifact_file_manifest.COLLECTION_FILE_MANIFEST_FILENAME) # file_manifest_data = None # try: # with open(file_manifest_filename, 'r') as mfd: # file_manifest_data = collection_artifact_file_manifest.load(mfd) # except EnvironmentError: # # log.debug('No galaxy.yml collection info found for collection %s.%s: %s', namespace, name, e) # pass # load galaxy.yml galaxy_filename = os.path.join(path_name, collection_info.COLLECTION_INFO_FILENAME) galaxy_yml_data = None try: with open(galaxy_filename, 'r') as gfd: if gfd: galaxy_yml_data = collection_info.load(gfd) except EnvironmentError: # for the case of collections that are not from or intended for galaxy, they do not # need to provide a galaxy.yml or MANIFEST.json, so an error here is exceptable. # log.debug('No galaxy.yml collection info found for collection %s.%s: %s', namespace, name, e) pass # TODO: make existence of a galaxy.yml and a MANIFEST.json mutual exclude and raise an exception for that case col_info = None # MANIFEST.json is higher prec than galaxy.yml if galaxy_yml_data: col_info = galaxy_yml_data if manifest_data: col_info = manifest_data.collection_info # Prefer version from install_info, but for a editable installed, there may be only galaxy version installed_version = install_info_version if col_info: installed_version = col_info.version # TODO/FIXME: what takes precedence? # - the dir names a collection lives in ~/.ansible/content/my_ns/my_name # - Or the namespace/name from galaxy.yml? # - Or the namespace/name from MANIFEST.json # Ditto for requirements # log.debug('collection_info_data: %s', collection_info_data) # Build a repository_spec of the repo now so we can pass it things like # requirements.from_dependencies_dict that need to know what requires something. repository_spec = RepositorySpec(namespace=namespace, name=name, version=installed_version) # The current galaxy.yml 'dependencies' are actually 'requirements' in ansible/ansible terminology # (ie, install-time) requirements_list = [] if col_info: requirements_list = requirements.from_dependencies_dict( col_info.dependencies, repository_spec=repository_spec) repository = Repository( repository_spec=repository_spec, path=path_name, installed=installed, requirements=requirements_list, ) log.debug('Loaded repository %s from %s', repository.repository_spec.label, path_name) return repository