def has_file(repo: Repository.Repository, file_path: str) -> bool:
"""Return true if file_path is found to exist """
RateLimiter.check()
try:
repo.get_contents(file_path)
Out.log(f"[{repo.full_name}] Found [{file_path}] file")
return True
except UnknownObjectException:
Out.log(f"[{repo.full_name}] No [{file_path}] file")
return False
def process_commit(self, repo: Repository, commit_id) -> bool:
files = {}
files_content = {}
scan_result = {}
commit_data = repo.get_commit(sha=commit_id)
files = commit_data.raw_data.get('files')
for file in files:
code = (file['patch'])
lines = code.split("\n")
file_scan = False
for line in lines: #if one line was added or changed scan the full file.
if line[0] == '+' or line[0] == 'M':
file_scan = True
break
#wfp calculation
if file_scan:
contents = repo.get_contents(file['filename'], ref=commit_id)
if contents:
files_content[file['filename']] = contents.decoded_content
try:
asset_json = repo.get_contents(self.sbom_file).decoded_content
except Exception:
self.logger.info("No assets")
asset_json = {}
self.logger.debug(asset_json)
scan_result = self.scanner.scan_files(files_content, asset_json)
result = {'comment': 'No results', 'validation': True, 'cyclondx': {}}
if scan_result:
result = self.scanner.format_scan_results(scan_result)
# Add a comment to the commit
if (not result['validation']
or self.comment_always) and result['comment']:
full_comment = result['comment']
if result['cyclondx']:
full_comment += "\n Please find the CycloneDX component details to add to your %s to declare the missing components here:\n" % self.sbom_file
if asset_json:
full_comment += "```\n" + json.dumps(
result['cyclondx']['components'], indent=2) + "\n```"
else:
full_comment += "```\n" + json.dumps(result['cyclondx'],
indent=2) + "\n```"
self.logger.debug(full_comment)
commit_data.create_comment(full_comment)
return result['validation'], full_comment
def update_changelog(version: str, github_repository: Repository) -> None:
"""
Add a version title to the changelog.
"""
changelog_path = Path('CHANGELOG.rst')
branch = 'master'
changelog_content_file = github_repository.get_contents(
path=str(changelog_path),
ref=branch,
)
# ``get_contents`` can return a ``ContentFile`` or a list of
# ``ContentFile``s.
assert isinstance(changelog_content_file, ContentFile)
changelog_bytes = changelog_content_file.decoded_content
changelog_contents = changelog_bytes.decode('utf-8')
new_changelog_contents = changelog_contents.replace(
'Next\n----',
f'Next\n----\n\n{version}\n------------',
)
github_repository.update_file(
path=str(changelog_path),
message=f'Update for release {version}',
content=new_changelog_contents,
sha=changelog_content_file.sha,
)
def get_all_files_in_the_repository(self,
repository: Repository,
branch: Branch,
directory="") -> List[ContentFile]:
"""
Get list of paths of all files in the repository on given branch.
:param repository: Forked repository
:param branch: Name of the branch.
:param directory: Starting directory.
:return: List of all files' paths.
"""
LOGGER.info(
"Looking for files in <%s> repository in <%s> branch in <%s> directory.",
repository.name, branch.name, directory)
all_contents = []
main_dir_contents = repository.get_contents(directory, ref=branch.name)
while main_dir_contents:
file_content = main_dir_contents.pop(0)
if file_content.type == "dir":
directory_contents = self.get_all_files_in_the_repository(
repository,
branch,
file_content.path,
)
all_contents.extend(directory_contents)
else:
all_contents.append(file_content)
LOGGER.info("Files found in the <%s> directory <%s>.", directory,
", ".join([str(file) for file in all_contents]))
return all_contents
def metadata(repo: Repository.Repository) -> dict:
"""Returns either the contents of the metadata file, or empty dict"""
metadata_file_path: str = "./metadata.json"
data: dict = {}
try:
data = json.loads(
repo.get_contents(metadata_file_path).decoded_content)
except UnknownObjectException:
Out.log(f"[{repo.full_name}] Failed to fetch metadata file")
return data
def analyse_content_files(
self,
repository: Repository,
prev_knowledge: Dict[str, Any],
is_local: bool = False) -> Optional[Dict[str, Any]]:
"""Analyse content files in repository.
Arguments:
repository {Repository} -- currently the PyGithub lib is used because of its functionality
ogr unfortunatelly did not provide enough to properly analyze content files
prev_knowledge {Dict[str, Any]} -- previous knowledge stored.
is_local -- flag to state if the knowledge should be collected locally or on Ceph.
"""
_LOGGER.info("-------------Content Files Analysis-------------")
# TODO: Extend to all types of files. Currently only README are considered.
# TODO: Add all types of README extensions available
content_file_text = ""
for file_name in ["README.md", "README.rst"]:
try:
content_file = repository.get_contents(file_name)
file_path = content_file.path
encoded = content_file.decoded_content
# TODO: Adjust because most of the files are not text
content_file_text = encoded.decode("utf-8")
except Exception as e:
_LOGGER.info("%r not found for: %r" %
(file_name, repository.full_name))
_LOGGER.warning(e)
if content_file_text:
with KnowledgeAnalysis(
entity_type=EntityTypeEnum.CONTENT_FILE.value,
new_entities=[[
file_name, file_path, content_file_text
]],
accumulator=prev_knowledge,
store_method=self.store_content_file,
) as analysis:
accumulated = analysis.store()
break
if not content_file_text:
return None
return accumulated
def update_homebrew(
homebrew_filename: str,
version_str: str,
github_repository: Repository,
homebrew_tap_github_repository: Repository,
) -> None:
"""
Update a Homebrew file in a given Homebrew tap with an archive from a given
repository.
"""
archive_url = github_repository.get_archive_link(
archive_format='tarball',
ref=version_str,
)
new_recipe_contents = get_homebrew_formula(
archive_url=archive_url,
head_url=github_repository.clone_url,
homebrew_recipe_filename=homebrew_filename,
)
message = f'Homebrew recipe for version {version_str}'
try:
content_file = homebrew_tap_github_repository.get_contents(
path=homebrew_filename,
ref='master',
)
except UnknownObjectException:
homebrew_tap_github_repository.create_file(
path=homebrew_filename,
message=message,
content=new_recipe_contents,
)
else:
# ``get_contents`` can return a ``ContentFile`` or a list of
# ``ContentFile``s.
assert isinstance(content_file, ContentFile)
homebrew_tap_github_repository.update_file(
path=homebrew_filename,
message=message,
content=new_recipe_contents,
sha=content_file.sha,
)
def update_version(repo: Repository, version: str) -> None:
"""Update manifest.json with the new version"""
print("Updating manifest.json...")
manifest = repo.get_contents("custom_components/google_home/manifest.json")
assert isinstance(manifest, ContentFile)
manifest_json = json.loads(manifest.decoded_content)
manifest_json["version"] = version
updated_manifest = json.dumps(manifest_json, indent=2) + "\n"
branch = repo.get_branch("master")
# Disable branch protection before commit
branch.remove_admin_enforcement()
repo.update_file(
path=manifest.path,
message=f"Release v{version}",
content=updated_manifest,
sha=manifest.sha,
)
# Re-enable branch protection
branch.set_admin_enforcement()
def __download_directory(repository: Repository, sha: str, server_path: str) -> None:
"""
Recursively downloads directory from GitHub repo.
"""
safe_create_directory(server_path)
contents = repository.get_dir_contents(server_path, ref=sha)
for content in contents:
logging.info("Downloading: %s", content.path)
if content.type == "dir" and not content.path.endswith("THIRD-PARTY"):
os.makedirs(content.path)
__download_directory(repository, sha, content.path)
else:
try:
path = content.path
if path.endswith(constants.FILE_EXT):
file_content = repository.get_contents(path, ref=sha)
file_data = base64.b64decode(file_content.content) # type: ignore
file_out = open(content.path, "wb+")
file_out.write(file_data)
file_out.close()
except (GithubException, IOError) as exc:
logging.info("Error processing %s: %s", content.path, exc)
def get_repo_files(cls, repo: Repository):
'''
Lists all the files in a specific repository.
'''
return [el.path for el in repo.get_contents('')]
