def main():
if PROJECT_PATH is None:
raise Exception("No PROJECT_ROOT_PATH")
if GITHUB_ACCESS_TOKEN is None:
raise Exception("No GITHUB_ACCESS_TOKEN")
issues = utils.read_csv_ignore_headers(
ISSUES_TO_COLLECT_COMMENTS_FOR_CSV_FILE_PATH,
ISSUES_CSV_FILE_FIELD_NAMES)
print("Getting all comments for issues...")
total = len(issues)
count = 0
print('Skipping thru read ones...')
for issue in issues:
try:
count += 1
issue_id = issue['id']
repo_name = issue['repo_name']
comments_url = issue['comments_url']
output_file_name = "{}/comments@issue@{}@{}.json".format(
OUTPUT_FOLDER_PATH, issue_id, repo_name.replace('/', '@'))
if utils.file_or_read_file_already_exists(output_file_name):
continue
print("\t{}/{} repo={}".format(count, total, repo_name))
issue_comments = get_comments(comments_url)
utils.write_to_json_file(output_file_name, issue_comments)
except Exception as e:
print("ERROR: Failed getting comments for issue={}".format(
issue['id']))
print("Done")
def main():
if PROJECT_PATH is None:
raise Exception("No PROJECT_ROOT_PATH")
if GITHUB_ACCESS_TOKEN is None:
raise Exception("No GITHUB_ACCESS_TOKEN")
commits_to_collect = utils.read_csv_ignore_headers(
INPUT_CSV_FILE_PATH, INPUT_CSV_FILE_FIELD_NAMES)
print("Getting all comments for issues...")
total = len(commits_to_collect)
count = 0
for c in commits_to_collect:
try:
issue_id = c['issue_id']
repo_name = c['repo_name']
sha = c['commit_id']
count += 1
print("\t{}/{} sha={}".format(count, total, sha))
output_file_name = "{}/commit@{}@{}.json".format(
OUTPUT_FOLDER_PATH, sha, repo_name.replace('/', '@'))
if utils.file_or_read_file_already_exists(output_file_name):
print("File Exists - Continuing")
continue
commit_url = f'https://api.github.com/repos/{repo_name}/commits/{sha}'
commit_json = get_commit_json(commit_url)
utils.write_to_json_file(output_file_name, commit_json)
except Exception as e:
print("ERROR: Failed getting comments for issue={}".format(
c['issue_id']))
print("Done")
def main():
if PROJECT_PATH is None:
raise Exception("No PROJECT_ROOT_PATH")
if GITHUB_ACCESS_TOKEN is None:
raise Exception("No GITHUB_ACCESS_TOKEN")
if LIBRARIES_IO_ACCESS_TOKEN is None:
raise Exception("No LIBRARIES_IO_ACCESS_TOKEN")
entity_to_collect_for = utils.read_csv_ignore_headers(
INPUT_CSV_FILE_PATH, INPUT_CSV_FILE_FIELD_NAMES)
total = len(entity_to_collect_for)
count = 0
for x in entity_to_collect_for:
try:
repo_half_name = x['repo_name']
count += 1
print("\t{}/{} repo={}".format(count, total, repo_half_name))
repo_full_name = get_repo_full_name_from_libraries_io(
repo_half_name)
output_file_name = "{}/{}.json".format(
OUTPUT_FOLDER_PATH, repo_full_name.replace('/', '@'))
if utils.file_or_read_file_already_exists(output_file_name):
print("File Exists - Continuing")
continue
dependents = get_dependents(repo_full_name)
utils.write_to_json_file(output_file_name, dependents)
except Exception as e:
print("ERROR: Failed for repo={}".format(x['repo_name']))
print("Done")
def main():
if PROJECT_PATH is None:
raise Exception("No PROJECT_ROOT_PATH")
if GITHUB_ACCESS_TOKEN is None:
raise Exception("No GITHUB_ACCESS_TOKEN")
repos = utils.read_csv_ignore_headers(INPUT_CSV_FILE_PATH,
INPUT_CSV_FILE_FIELD_NAMES)
total = len(repos)
count = 0
list_to_write = list()
for repo in repos:
repo_name = repo['repo_name']
try:
count += 1
print("\t{}/{} repo={}".format(count, total, repo_name))
project_git_folder = f"{REPOS_PATH}/repos/{repo_name.replace('/', '#')}"
if os.path.isdir(project_git_folder):
package_json_path = f"{project_git_folder}/package.json"
if utils.file_or_read_file_already_exists(package_json_path):
package_json_contents = utils.load_json_file(
package_json_path)
# Will throw if 'name' is not there
list_to_write.append({
'repo_name':
repo_name,
'package_name':
package_json_contents['name']
})
else:
# package.json doesn't exist
list_to_write.append({
'repo_name': repo_name,
'package_name': None
})
else:
# Could not clone project
list_to_write.append({
'repo_name': repo_name,
'package_name': None
})
except Exception as e:
list_to_write.append({
'repo_name': repo_name,
'package_name': None
})
utils.write_lines_to_new_csv(OUTPUT_FILE_PATH, OUTPUT_FIELD_NAMES,
list_to_write)
print("Done")
def main():
if PROJECT_PATH is None:
raise Exception("No PROJECT_ROOT_PATH")
if GITHUB_ACCESS_TOKEN is None:
raise Exception("No GITHUB_ACCESS_TOKEN")
repos = utils.read_csv_ignore_headers(REPOS_FILE_PATH, REPOs_FILED_NAMES)
print("Getting all Issues for repos...")
total = len(repos)
count = 0
for repo in repos:
try:
repo_name = repo['dependent']
count += 1
print("\t{}/{} repo={}".format(count, total, repo_name))
output_file_name = "{}/issues@{}.json".format(OUTPUT_FOLDER_PATH, repo_name.replace('/', '@'))
if utils.file_or_read_file_already_exists(output_file_name):
print("File Exists - Continuing -- repo={}".format(repo_name))
continue
repo_issues = get_issues(repo_name)
utils.write_to_json_file(output_file_name, repo_issues)
except Exception as e:
print("ERROR: Failed getting issues for repo={}".format(repo['repo']))
print("Done")