def test_prepare_list(self, _, repo_list, repositories, expected_len):
writeLinesFile(repo_list, repositories, appendWithNewLine=True)
prepare_list(repo_list, _rep_count, _email_notify)
new_repo_list = readLinesFile(repo_list)
self.assertEqual(expected_len, len(new_repo_list))
def dump_errors(email_notify, skipped_filename):
with Connector() as connector:
git_repos = connector.get_repositories_with_errors()
writeLinesFile(skipped_filename,
git_repos,
mode='a',
appendWithNewLine=True)
MailNotifier(subject, email_notify).send_notification(
f"Skipped {len(git_repos)} repositories: {git_repos}")
def prepare_list(repo_list: str, rep_count: int, email_notify: str):
content = readLinesFile(repo_list)
writeLinesFile(temp_repo_list, content[:rep_count], appendWithNewLine=True)
MailNotifier(subject, email_notify).send_notification(
f"Was repos: {content.__len__()}. "
f"Taking only {rep_count} reps. "
f"Become repos: {content[rep_count:].__len__()}.")
writeLinesFile(repo_list, content[rep_count:], appendWithNewLine=True)
def test_correctInitExtensions(self):
languages = ["Go", "Java", "CSS"]
writeLinesFile(_filename, languages, appendWithNewLine=True)
_extensions = extensions(_filename)
if languages is not None:
languagesFromPath = [
_extensions[extension].split('/')[0]
for extension in _extensions
]
for language in languages:
self.assertIn(language, languagesFromPath)
def stat_extensions(self, extensions):
logExt = ["Folder, number or files"]
for ext, path in tqdm(extensions.items()):
directory = os.path.join(self.target_directory, 'languages', path)
count = count_all_files(directory, ext)
logExt.append(f"{path}, {count}\n")
if len(logExt) > 0:
writeLinesFile(
f"{statistic_folder}/statistic-extensions-{str(time.time_ns())}.csv",
logExt)
self.notifier.send_notification(logExt)
def stat_languages(self, languages_file):
logLang = ["Folder, number or files"]
languages = readLinesFile(languages_file)
for language in tqdm(languages):
directory = os.path.join(self.target_directory, 'languages',
language)
count = count_all_files(directory, None)
logLang.append(f"{language}, {count}\n")
if len(logLang) > 0:
writeLinesFile(
f"{statistic_folder}/stat-languages-{str(time.time_ns())}.csv",
logLang)
self.notifier.send_notification(logLang)
def stat_repositories(self):
logRepos = []
total = 0
for user in tqdm(os.listdir(self.target_directory)):
for rep in os.listdir(os.path.join(self.target_directory, user)):
url = f"https://github.com/{user}/{rep}/\n"
total += len(
os.listdir(os.path.join(self.target_directory, user, rep)))
logRepos.append(url)
if len(logRepos) > 0:
writeLinesFile(
f"{statistic_folder}/statistic-repositories-{str(time.time_ns())}.txt",
logRepos)
self.notifier.send_notification(
f"Total amount of repositories with different branches: {total}"
)
def calibrate(email, target_directory, only_master):
notifier = MailNotifier(subject, email)
root_reps = os.path.join(target_directory, 'repositories')
root_langs = os.path.join(target_directory, 'languages')
repos = _get_all_repos(root_reps)
files = _log_files(repos, root_reps, root_langs)
paths = _log_paths(repos, root_reps, root_langs)
branches = _check_for_branches(root_reps, only_master)
if len(paths) > 0:
remove_answer = input(
f"Remove {len(paths)} files not listed in paths.json? (Y/n) "
).lower() in ["yes", 'y']
if remove_answer:
for file in tqdm(paths, desc="Remove corrupted files"):
os.remove(file)
timestamp = str(time.time_ns())
if len(files) > 0:
writeLinesFile(f"{calibrate_folder}/calibration-files-{timestamp}.txt",
files)
notifier.send_notification(
f"Total amount of errors with files: {len(files)}")
if len(paths) > 0:
writeLinesFile(f"{calibrate_folder}/calibration-paths-{timestamp}.txt",
paths)
notifier.send_notification(
f"Total amount of errors with paths: {len(paths)}")
if len(branches) > 0:
writeLinesFile(
f"{calibrate_folder}/calibration-branches-{timestamp}.txt",
branches)
notifier.send_notification(
f"Total amount of errors with branches: {len(branches)}")
def test_wrongInitExtensions(self):
languages = ["Go", "TotallyNotJava", "TotallyNotCSS"]
writeLinesFile(_filename, languages, appendWithNewLine=True)
with self.assertRaises(TypeError):
extensions(_filename)
def test_prepare_list_with_errors(self, _, repo_list, repositories, error):
with self.assertRaises(error):
writeLinesFile(repo_list, repositories, appendWithNewLine=True)
prepare_list(repo_list, _rep_count, _email_notify)