class HaloFeedback(Feedback):
def __init__(self, message=None):
self.spinner = Halo(text=message or '', spinner='dots')
if message and not default_config.debug:
self.spinner.start()
def update_message(self, message):
super().update_message(message)
if not self.spinner._spinner_id and not default_config.debug:
self.spinner.start()
self.spinner.text = (message + ' ...') if self.message else ''
def succeeded(self):
self.spinner.text = self.message
self.spinner.succeed()
def errored(self, error):
# self.spinner.text = str(error) if error else self.message
self.spinner.text = f'{self.message} ... {colored(str(error), "red")}'
self.spinner.fail()
sys.exit(1)
def warning(self, warning):
self.spinner.text = f'{self.message} ... {colored(str(warning), "yellow")}'
self.spinner.warn()
def info(self, message):
self.spinner.info(message)
def exists(self, error):
self.warning('exists')
def _check_for_updates_with_appimageupdatetool(self,
path_appimageupdate,
show_spinner=True):
path_to_old_appimage = self.appdata().get('path')
if show_spinner:
spinner = Halo('Checking for updates', spinner='dots')
spinner.start()
_check_update_command = shlex.split(
"{au} --check-for-update {app}".format(
au=path_appimageupdate,
app=path_to_old_appimage,
))
_check_update_proc = subprocess.Popen(_check_update_command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
e_code = _check_update_proc.wait(600)
if e_code == 0:
if show_spinner:
spinner.succeed("Already up-to-date!")
return
elif e_code == 1:
if show_spinner:
spinner.info("Updates found")
else:
if show_spinner:
spinner.fail("Update information is not embedded within the "
"AppImage. ")
spinner.fail("Consider informing the AppImage author to add a "
".zsync file")
spinner.fail(
"Alternatively, pass the --no-appimageupdate option")
spinner.stop()
def populate_locations(countries):
spinner = Halo(spinner='dots')
spinner.info(f'Posting {len(countries.index)} locations from ISO-3166...')
for index, country in countries.iterrows():
newLocation = Location(country['countryCode'], country['countryName'])
_ = prepare_resource('locations', newLocation.countryCode,
'findByCountryCode', newLocation.serialize())
def test_info(self):
"""Test info method
"""
spinner = Halo(stream=self._stream)
spinner.start('foo')
spinner.info()
output = self._get_test_output()['text']
pattern = re.compile(r'(ℹ|¡) foo', re.UNICODE)
self.assertRegexpMatches(output[-1], pattern)
spinner.stop()
def mint_used_ids(mirids_file_name, mirid_controller_url):
spinner = Halo(spinner='dots')
spinner.info('Minting used mirids from old database in the new one...')
with open(mirids_file_name) as mirids_file:
lines = mirids_file.read().splitlines()
miriam_array = []
for mirid in lines:
if len(mirid) != 12:
spinner.fail(f'Invalid miriam: [{mirid}]')
else:
miriam_array.append(int(mirid.split(':')[1]))
first_miriam = miriam_array[0]
last_miriam = miriam_array[-1]
spinner.info(f'Found [{len(miriam_array)}] MIR ids, spanning from [{first_miriam}] to [{last_miriam}]...')
spinner.start(f'Launching {last_miriam} async load requests...')
loop = asyncio.get_event_loop()
future = asyncio.ensure_future(async_mint(first_miriam, last_miriam, f'{mirid_controller_url}loadId/'))
loop.run_until_complete(future)
spinner.succeed(f'Done.')
spinner = Halo(spinner='dots')
spinner.info('Returning mirids in gaps between used ones...')
unused_miriams = list(find_empty_ranges(miriam_array))
spinner.info(f'Found [{len(unused_miriams)}] empty MIR id blocks: {unused_miriams}')
for empty_block in unused_miriams:
spinner.info(f'Returning MIR ids in range [{empty_block}]')
return_miriam_array = []
for mir_id in range(empty_block[0], empty_block[1]):
return_miriam_array.append(mir_id)
spinner.start(f'Launching {len(return_miriam_array)} async return requests...')
future = asyncio.ensure_future(async_mint(return_miriam_array[0], return_miriam_array[-1], f'{mirid_controller_url}returnId/'))
loop.run_until_complete(future)
spinner.succeed(f'Done!')
def submit_job(code_url, cpus, memory_mb, max_runtime_secs):
job_spec = {
"code_url": code_url,
# TODO: Later, when we add support for resource_requirements, test some real values for this
"resource_requirements": {
"cpus": cpus,
"memory_mb": memory_mb,
"max_runtime_secs": max_runtime_secs
}
}
print("Submitting job...")
res = requests.post(submit_job_url, json=job_spec)
if res.status_code != 200:
print("Could not submit job!")
sys.exit(1)
resp = res.json()
if not resp.get("success", False):
print("Could not submit job!")
sys.exit(1)
job_id = resp['job_id']
job_status_code = None
spinner = Halo(text="Waiting for job updates", spinner='dots')
spinner.start()
while True:
try:
new_job_status_code = get_job_status(job_id)
if new_job_status_code != job_status_code:
if new_job_status_code in ("UNASSIGNED", "ASSIGNED"):
spinner.info(STATUS_CODE_MESSAGES[new_job_status_code])
spinner.start("Waiting for job updates")
elif new_job_status_code == "FAILED":
spinner.fail(STATUS_CODE_MESSAGES[new_job_status_code])
spinner.start("Waiting for job updates")
else:
# the job has succeeded
spinner.stop_and_persist(
symbol='🦄'.encode('utf-8'),
text=STATUS_CODE_MESSAGES[new_job_status_code])
break
job_status_code = new_job_status_code
time.sleep(JOB_STATUS_POLL_INTERVAL_SECS)
except (KeyboardInterrupt, SystemExit):
spinner.stop()
break
def getDvScore(path_to_file):
spinner = Halo(text="Checking dv score of the cleaned data",
spinner='dots',
text_color="cyan")
spinner.start()
file = path_to_file
url = 'https://dv3.datavalidation.com/api/v2/user/me/list/create_upload_url/'
params = '?name=' + file + '&email_column_index=0&has_header=0&start_validation=false'
headers = {'Authorization': 'Bearer ' + DV_API_KEY}
s = requests.Session()
a = requests.adapters.HTTPAdapter(max_retries=3)
s.mount("https://", a)
res = s.get(url + params, headers=headers)
upload_csv_url = res.json()
files = {
'file': open(file, 'rb') #open(OUTPUT_DIR_NAME + "/" + file, 'rb')
}
list_id = s.post(upload_csv_url, headers=headers, files=files)
time.sleep(10)
dv_result_url = 'https://dv3.datavalidation.com/api/v2/user/me/list/' + list_id.json(
)
dv_result = s.get(dv_result_url, headers=headers).json()
while dv_result['status_value'] == 'PRE_VALIDATING':
dv_result = requests.get(dv_result_url, headers=headers).json()
spinner.info("Status percent complete: " +
str(dv_result['status_percent_complete']))
time.sleep(5) # sleep 5 seconds
try:
dv_result = requests.get(dv_result_url, headers=headers).json()
percent = lambda count: round(
(count / dv_result['subscriber_count']), 2) * 100
spinner.succeed("Done checking dv score")
print("The grade summary is: ")
for score_name, score_value in dv_result['grade_summary'].items():
print('%-3s : ' % (score_name) + str(percent(score_value)))
except:
if (dv_result['subscriber_count'] == 0):
print("Empty list of emails were sent for dv validation!")
print("Perhaps no new email to check dv?")
print("Program terminated")
return 0
else:
print("Something goes wrong")
spinner.stop()
def add_aks(name,force):
'''
\b
Example:
\b
# Add all AKS clusters from Azure Subscription.
kubeasy aks add -n all
\b
# Add specific AKS clusters from Azure Subscription.
kubeasy aks add -n <aksCluster>
'''
spinner = Halo(text=colorama.Fore.GREEN + 'Logging into Azure using Azure CLI..', spinner='dots',color='yellow')
spinner.start()
if not login('azure',spinner):
spinner.fail(colorama.Fore.RED + 'Azure login failed')
sys.exit(1)
spinner.stop()
spinner = Halo(text=colorama.Fore.GREEN + 'Getting Kubernetes Configuration for {}'.format(name), spinner='dots',color='yellow')
spinner.start()
if name == 'all':
for key in get_K8SList('azure'):
if (not _isExist(key) or (_isExist(key) and force)):
addConfig(spinner,'azure',key)
else:
spinner.info(colorama.Fore.GREEN + '\"{}\" is already configured for the Kubeasy, Cheers ! '.format(key))
elif (not _isExist(name) or (_isExist(name) and force)):
addConfig(spinner,'azure',name)
else:
spinner.info(colorama.Fore.GREEN + '\"{}\" is already configured for the Kubeasy, Cheers !'.format(name))
spinner.stop()
def wait_then_open(url):
"""
Waits for a bit then opens a URL. Useful for waiting for a proxy to come up, and then open the URL.
"""
spinner = Halo(text=colorama.Fore.GREEN +
'Opening Kubernetes Dashboard for {} --> {}'.format(
get_current_context(), url),
spinner='dots',
color='yellow')
spinner.start()
time.sleep(3)
webbrowser.open_new_tab(url)
spinner.info(
colorama.Fore.GREEN +
'Running dashboard for {}, Press CTRL+C to stop the port forwarding..'.
format(get_current_context()))
def _run_test(cls, test_dict: dict):
"""
A generic method handling the run of both disruptive and non
disruptive tests.
"""
spinner = Halo(spinner='dots', text_color='yellow')
tc_class = test_dict["testClass"]
volume_type = test_dict["volType"]
mname = test_dict["moduleName"][:-3]
tc_log_path = (f"{cls.base_log_path+test_dict['modulePath'][5:-3]}/"
f"{volume_type}/{mname}.log")
# to calculate time spent to execute the test
start = time.time()
spinner.succeed(text=f"Running test case : {mname}-{volume_type}")
runner_thread_obj = RunnerThread(tc_class, cls.param_obj, volume_type,
mname, cls.logger, cls.env_obj,
tc_log_path, cls.log_level)
test_stats = runner_thread_obj.run_thread()
test_stats['timeTaken'] = time.time() - start
test_stats['tcNature'] = test_dict['tcNature']
spinner.clear()
result_text = f"{test_dict['moduleName'][:-3]}-{test_dict['volType']}"
if test_stats['testResult'][0] is True:
test_stats['testResult'] = "PASS"
result_text += " PASS"
spinner = Halo(spinner='dots', text_color='green')
spinner.succeed(text=f"{mname}-{volume_type} Succeeded")
elif test_stats['testResult'][0] is False:
result_text += " FAIL"
test_stats['testResult'] = "FAIL"
spinner = Halo(spinner='dots', text_color='red')
spinner.fail(f"{mname}-{volume_type} Failed")
else:
result_text += " SKIP"
test_stats['testResult'] = "SKIP"
spinner = Halo(spinner='dots', text_color='cyan')
spinner.info(f"{mname}-{volume_type} SKIP")
test_stats['component'] = tc_log_path.split('/')[-4]
result_value = {test_dict["moduleName"][:-3]: test_stats}
cls.job_result_queue.put(result_value)
def add_gke(name,force):
'''
\b
Example:
\b
# Add all GKE clusters .
kubeasy gke add -n all
\b
# Add specific GKE clusters.
kubeasy gke add -n <gkeCluster>
'''
spinner = Halo(text=colorama.Fore.GREEN + 'Logging into Google Cloud using gcloud ..', spinner='dots',color='yellow')
spinner.start()
if not login('google',spinner):
spinner.fail(colorama.Fore.RED + 'Google Cloud login failed')
sys.exit(1)
spinner.stop()
spinner = Halo(text=colorama.Fore.GREEN + 'Getting Kubernetes Configuration for {}'.format(name), spinner='dots',color='yellow')
spinner.start()
if name == 'all':
for key in get_K8SList('google'):
if (not _isExist(key) or (_isExist(key) and force)):
addConfig(spinner,'google',key)
else:
spinner.info(colorama.Fore.GREEN + '\"{}\" is already configured for the Kubeasy, Cheers ! '.format(key))
elif (not _isExist(name) or (_isExist(name) and force)):
addConfig(spinner,'google',name)
else:
spinner.info(colorama.Fore.GREEN + '\"{}\" is already configured for the Kubeasy, Cheers !'.format(name))
spinner.stop()
def curl(src, dest):
""" Installs `src` to path `dest` """
spinner = Halo(text="curl {}".format(dest),
spinner="dots",
placement="right")
spinner.start()
if os.path.exists(dest):
spinner.info("{} already exists".format(dest))
return
try:
sh.curl("-fLo", dest, src)
spinner.succeed()
except sh.ErrorReturnCode as err:
err_message = "\n\t" + err.stderr.replace("\n", "\n\t")
logging.error("Error downloading file `%s`: %s", src, err_message)
spinner.fail()
def init_config(config_file_name):
spinner = Halo(text='Initializing config', spinner='dots')
spinner.start()
# Decides where are we working on.
env = os.getenv('ENV') or 'DEV'
spinner.info(f'Using {env} environment.')
# Fetches config.
config_file_contents = configparser.ConfigParser()
config_file_contents.read_file(open(config_file_name))
config['data_origin_url'] = config_file_contents.get('DEFAULT', 'DataOriginURL')
config['destination_url'] = config_file_contents.get(env, 'DestinationURL')
config['keycloak_url'] = config_file_contents.get(env, 'KeycloakURL')
config['mirid_controller_url'] = config_file_contents.get(env, 'mirIDControllerURL')
config['EMPTY_FIELD_LITERAL'] = config_file_contents.get(env, 'EmptyFieldLiteral')
def main(config):
print("\n[Time] %s" %
datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
print("开始创建登记任务")
spinner = Halo(text='Loading', spinner='dots')
spinner.start('正在创建登记实例...')
ci = clockin(config)
spinner.succeed('创建登记实例成功')
spinner.start(text='登录...')
try:
log = ci.login()
if log == 'un_uid_and_continue':
spinner.info('学号不存在，测试登录')
elif log == 'un_uid_and_exit':
spinner.fail('学号不存在，请修改配置')
return
spinner.succeed('登陆成功')
except Exception as err:
spinner.fail(str(err))
return
if log == 'is_writted':
spinner.info('今日已登记，无需再次登记')
return
# log = ci.for_iter()
# if log == 'is_writted':
# spinner.info('今日已登记，无需再次登记')
# return
spinner.start(text='打卡中...')
try:
ci.fillin()
spinner.succeed('打卡成功')
time.sleep(5)
ci.driver.close()
except Exception as err:
spinner.fail(str(err))
spinner.fail('打卡失败')
return
def curl(src, dest):
""" Installs `src` to path `dest` """
spinner = Halo(
text="curl {}".format(dest),
spinner="dots",
placement="right"
)
spinner.start()
if os.path.exists(dest):
spinner.info("{} already exists".format(dest))
return
try:
sh.curl("-fLo", dest, src)
spinner.succeed()
except sh.ErrorReturnCode as err:
err_message = "\n\t" + err.stderr.replace("\n", "\n\t")
logging.error(
"Error downloading file `%s`: %s", src, err_message
)
spinner.fail()
def add_students(
github_students: List[str] = typer.Argument(
metavar="student_handles",
default=...,
help="list of student handles separated by white space",
),
github_team: Optional[str] = typer.Option(
None, "--team", help="invite to the specfic team under organization",
),
yes: bool = opt_all_yes,
dry: bool = opt_dry,
github_token: Optional[str] = opt_github_token,
github_organization: Optional[str] = opt_gh_org,
):
"""
Invite students to join our Github organization
"""
ensure_config_exists()
def fallback(val, fallback_value):
return val if val else fallback_value
# Handle default value manually because we'll change our config after app starts up
github_token: str = fallback(
github_token, app_context.config.github.personal_access_token
)
github_organization: str = fallback(
github_organization, app_context.config.github.organization
)
github_team: str = fallback(
github_team, app_context.config.add_students.default_team_slug
)
safety = SafetyActor(dry=dry)
safety.ensure_gh_token(github_token)
# TODO: use logging lib to log messages
spinner = Halo(stream=sys.stderr)
if dry:
spinner.info("Dry run")
if not (
yes or typer.confirm(f"Add students to {github_organization}/{github_team}?")
):
raise typer.Abort()
spinner.info("fetch existing team members from GitHub")
team = Team(
dry=dry,
org=github_organization,
team_slug=github_team,
github_token=github_token,
)
num_member = len(team.members.keys())
spinner.succeed(f" target team: {github_team} ({num_member} members) ")
existed_members = set(team.members.keys())
outside_users = list(set(github_students) - existed_members)
spinner.info("Check valid Github users")
invalid_handles = invalid_user_handles(
outside_users, github_token=github_token, safety=safety, spinner=spinner
)
if len(invalid_handles) != 0:
print("non-existed github user handles:")
# control strings take space
print_table(invalid_handles)
non_member_valid_users = list(set(outside_users) - set(invalid_handles))
print(f"Users to add (total:{len(non_member_valid_users)})")
print_table(non_member_valid_users)
print("-" * 30)
spinner.info("start to invite users")
success_user, failed_users = invite_user_to_team(
team=team, users=non_member_valid_users, spinner=spinner
)
if len(failed_users) > 0:
print("Users failed to add")
print_table(failed_users)
spinner.succeed("Add students successfully")
def main(override_args=None):
"""Method to start the script"""
logger.debug(f'START: get_threats_from_query_hash.py')
# Load initial args
parser = BaseScripts.start(
'Retrieve a list of response from a given query hash.')
parser.add_argument(
'--query_fields',
help=
'fields to be retrieved from the threat (default: only the hashkey)\n'
'If an atom detail isn\'t present in a particular atom, empty string is returned.',
nargs='+',
default=['threat_hashkey'],
)
parser.add_argument(
'--list',
help=
'Turn the output in a list (require query_fields to be a single element)',
action='store_true',
)
required_named = parser.add_argument_group('required arguments')
required_named.add_argument(
'query_hash',
help=
'the query hash from which to retrieve the response hashkeys or a path to the query body json file',
)
if override_args:
args = parser.parse_args(override_args)
else:
args = parser.parse_args()
configure_logging(args.loglevel)
if len(args.query_fields) > 1 and args.list:
parser.error(
"List output format is only available if a single element is queried (via query_fields)"
)
query_body = {}
query_hash = args.query_hash
if len(query_hash) != 32 or os.path.exists(query_hash):
try:
with open(query_hash, 'r') as query_body_file:
query_body = json.load(query_body_file)
except FileNotFoundError:
logger.error(
f"Couldn't understand the given value as a query hash or path to query body: {query_hash}"
)
exit(1)
# Load api_endpoints and tokens
dtl = Datalake(env=args.env, log_level=args.loglevel)
logger.debug(
f'Start to search for threat from the query hash:{query_hash}')
spinner = None
if logger.isEnabledFor(logging.INFO):
spinner = Halo(text=f'Creating bulk task', spinner='dots')
spinner.start()
task = dtl.BulkSearch.create_task(query_body=query_body,
query_hash=query_hash,
query_fields=args.query_fields)
if spinner:
spinner.text = f'Waiting for bulk task {task.uuid} response'
response = task.download_sync()
original_count = response.get('count', 0)
if spinner:
spinner.succeed()
spinner.info(
f'Number of threat that have been retrieved: {original_count}')
formatted_output = format_output(response, args.list)
if args.output:
with open(args.output, 'w') as output:
output.write(formatted_output)
else:
logger.info(formatted_output)
if args.output:
logger.info(f'Threats saved in {args.output}')
else:
logger.info('Done')
def filterScrape(need, category, page):
spinner = Halo(text='Scraping content', spinner='dots', animation='bounce')
headers = {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
}
output_dic = OrderedDict()
found = 0
try:
while (found < need):
spinner.start()
url = "https://ctftime.org/writeups?page={}&hidden-tags={}".format(
page, category)
spinner.text = "Scraping Page: {}".format(page)
response = requests.get(url, headers=headers)
soup = BeautifulSoup(response.content, 'html.parser')
count_per_page = 0
for tr in soup.find_all('tr')[1:]:
tds = tr.find_all('td')
w_no = tds[4].a["href"]
task_name = tds[1].text
writeup_url = "https://ctftime.org/" + w_no
r = requests.get(writeup_url, headers=headers)
spinner.text = "Parsing {} ({})".format(
w_no,
task_name.encode('ascii', 'ignore').decode('ascii'))
spinner.color = "red"
if (len(task_name) > 30):
task_name = task_name[:27] + '...'
flag = 0
original_url = ""
new_soup = BeautifulSoup(r.content, 'lxml')
a = new_soup.find_all('a')
for link in a:
if link.text == "Original writeup":
original_url = link['href']
if (len(original_url) <= 125):
flag = 1
break
if flag == 1:
if (task_name in output_dic):
output_dic[task_name] += '\n' + original_url
else:
output_dic[task_name] = original_url
count_per_page += 1
found += 1
else:
if task_name not in output_dic:
count_per_page += 1
found += 1
output_dic[task_name] = writeup_url
if (found == need):
break
else:
continue
if (count_per_page == 0):
spinner.fail("Page {} doesn't exist.".format(page))
spinner.info("Try decreasing the Page Seed or limit")
spinner.info("Try changing the category")
print(
"Such as : Change 'rev' -> 'reverse engineering' to get more results"
)
break
else:
spinner.succeed(
"Gathered writeups for {} tasks from page {}".format(
count_per_page, page))
spinner.color = "cyan"
page += 1
return output_dic
except (KeyboardInterrupt, SystemExit):
spinner.warn('Program exited unexpectedly')
exit()
def announce_grade(
homework_prefix: str = typer.Argument(
..., help="prefix of the target homework"),
feedback_source_repo: Optional[str] = typer.Option(
None, show_default=True, help="Repo contains students' feedbacks"),
only_id: Optional[str] = typer.Option(default=None,
help="only id to announce"),
token: Optional[str] = opt_github_token,
org: str = opt_gh_org,
dry: bool = typer.Option(
False, "--dry", help="dry run, do not publish result to the remote"),
yes: bool = opt_all_yes,
):
"""Announce student grades to each hw repo"""
ensure_config_exists()
def fallback(val, fallback_value):
return val if val else fallback_value
# Handle default value manually because we'll change our config after app starts up
token: str = fallback(token,
app_context.config.github.personal_access_token)
org: str = fallback(org, app_context.config.github.organization)
feedback_source_repo: str = fallback(
feedback_source_repo,
app_context.config.announce_grade.feedback_source_repo)
ensure_gh_token(token)
if not (yes or
typer.confirm(f"Add annouce_grade to {org}/{homework_prefix}?")):
raise typer.Abort()
# TODO: use logging lib to log messages
spinner = Halo(stream=sys.stderr)
student_feedback_title = f"Grade for {homework_prefix}"
gstudents = Gstudents()
feedback_vars = gstudents.left_join(homework_prefix)
# Clone feedback repo & set needed variables
cur = Path(".")
for d in cur.glob("feedback-tmp-*"):
shutil.rmtree(d)
spinner.info("delete dated folder")
root_folder = Path(
tempfile.mkdtemp(
prefix="feedback-tmp-{}-".format(
datetime.now().strftime("%b%d%H%M%S")),
dir=".",
))
spinner.succeed(f"Create tmp folder {root_folder}")
feedback_repo_path = root_folder / "feedbacks"
spinner.info(f"cloning feeback source repo : {feedback_source_repo}")
_, t = measure_time(sp.run)(
[
"git",
"clone",
f"https://github.com/{org}/{feedback_source_repo}.git",
feedback_repo_path.name,
],
cwd=root_folder,
)
spinner.succeed(
f"cloning feeback source repo : {feedback_source_repo} ... {t:4.2f} sec"
)
client = httpx.AsyncClient(headers=httpx.Headers(
{
"User-Agent": "GitHubClassroomUtils/1.0",
"Authorization": "token " + token,
# needed for the check-suites request
"Accept": "application/vnd.github.antiope-preview+json",
}))
hw_path = feedback_repo_path / homework_prefix / "reports"
# generate feedbacks
fbs, t = measure_time(gen_feedbacks)(homework_prefix, hw_path,
feedback_vars)
spinner.succeed(f"Generate content for feedbacks ... {t:5.3f} sec")
# handle only_id
if only_id:
try:
# detect possible buggy condition
info = gstudents.get_student(only_id)
except RuntimeError as e:
print(" *=" * 30)
print("Warning!")
print(e)
return
only_repo_name = get_hw_repo_name(homework_prefix,
info["github_handle"])
fbs = list(filter(lambda fb: fb["repo_name"] == only_repo_name, fbs))
async def push_to_remote(feedback_title, feedbacks):
# push to remote
async def push_feedback(fb):
request_body = {"title": feedback_title, "body": fb["value"]}
try:
issue_num = await find_existing_issue(client, org,
fb["repo_name"],
feedback_title)
except BaseException:
print(f'error on {fb["repo_name"]}')
return
if issue_num:
request_body["state"] = "open" # reopen issue
url = f"https://api.github.com/repos/{org}/{fb['repo_name']}/issues/{issue_num}"
await edit_issue_async(client, url, issue_num, request_body)
else:
url = f"https://api.github.com/repos/{org}/{fb['repo_name']}/issues"
await create_issue_async(client, url, request_body)
print(f'success {fb["repo_name"]}')
async with trio.open_nursery() as nursery:
for fb in feedbacks:
nursery.start_soon(push_feedback, fb)
# print out target repos
print("repo to announce grade:")
pprint([fb["repo_name"] for fb in fbs])
if dry:
spinner.succeed("DRYRUN: skip push to remote")
else:
if typer.confirm("Do you want to continue?", default=False):
_, t = measure_time(trio.run)(push_to_remote,
student_feedback_title, fbs)
spinner.succeed(f"Push feedbacks to remote ... {t:5.2f} sec")
else:
spinner.warn("You refused to publish to remote")
spinner.succeed("finished announce grade")
return
def announce_grade(homework_prefix, token, dry, org, only_id,
feedback_source_repo):
"""announce student grades to each hw repo"""
ensure_gh_token(token)
# TODO: use logging lib to log messages
spinner = Halo(stream=sys.stderr)
student_feedback_title = f"Grade for {homework_prefix}"
gstudents = Gstudents()
feedback_vars = gstudents.left_join(homework_prefix)
# Clone feedback repo & set needed variables
cur = Path(".")
for d in cur.glob("feedback-tmp-*"):
shutil.rmtree(d)
spinner.info("delete dated folder")
root_folder = Path(
tempfile.mkdtemp(
prefix="feedback-tmp-{}-".format(
datetime.now().strftime("%b%d%H%M%S")),
dir=".",
))
spinner.succeed(normal.txt("Create tmp folder ").kw(root_folder).to_str())
feedback_repo_path = root_folder / "feedbacks"
spinner.info(f"cloning feeback source repo : {feedback_source_repo}")
_, t = measure_time(sp.run)(
[
"git",
"clone",
f"https://github.com/{org}/{feedback_source_repo}.git",
feedback_repo_path.name,
],
cwd=root_folder,
)
spinner.succeed(
f"cloning feeback source repo : {feedback_source_repo} ... {t:4.2f} sec"
)
client = httpx.AsyncClient(headers=httpx.Headers(
{
"User-Agent": "GitHubClassroomUtils/1.0",
"Authorization": "token " + token,
# needed for the check-suites request
"Accept": "application/vnd.github.antiope-preview+json",
}))
hw_path = feedback_repo_path / homework_prefix / "reports"
# generate feedbacks
fbs, t = measure_time(gen_feedbacks)(homework_prefix, hw_path,
feedback_vars)
spinner.succeed(f"Generate content for feedbacks ... {t:5.3f} sec")
# handle only_id
if only_id:
try:
# detect possible buggy condition
info = gstudents.get_student(only_id)
except RuntimeError as e:
print(" *=" * 30)
print("Warning!")
print(e)
return
only_repo_name = get_hw_repo_name(homework_prefix,
info["github_handle"])
fbs = list(filter(lambda fb: fb["repo_name"] == only_repo_name, fbs))
async def push_to_remote(feedback_title, feedbacks):
# push to remote
async def push_feedback(fb):
request_body = {"title": feedback_title, "body": fb["value"]}
try:
issue_num = await find_existing_issue(client, org,
fb["repo_name"],
feedback_title)
except BaseException:
print(f'error on {fb["repo_name"]}')
return
if issue_num:
request_body["state"] = "open" # reopen issue
url = f"https://api.github.com/repos/{org}/{fb['repo_name']}/issues/{issue_num}"
await edit_issue_async(client, url, issue_num, request_body)
else:
url = f"https://api.github.com/repos/{org}/{fb['repo_name']}/issues"
await create_issue_async(client, url, request_body)
print(f'success {fb["repo_name"]}')
async with trio.open_nursery() as nursery:
for fb in feedbacks:
nursery.start_soon(push_feedback, fb)
# print out target repos
print("repo to announce grade:")
pprint([fb["repo_name"] for fb in fbs])
if dry:
spinner.succeed("DRYRUN: skip push to remote")
else:
if click.confirm("Do you want to continue?", default=False):
_, t = measure_time(trio.run)(push_to_remote,
student_feedback_title, fbs)
spinner.succeed(f"Push feedbacks to remote ... {t:5.2f} sec")
else:
spinner.warn("You refused to publish to remote")
spinner.succeed("finished announce grade")
return
print("Could not submit job!")
sys.exit(1)
job_id = resp['job_id']
job_status_code = None
spinner = Halo(text="Waiting for job updates", spinner='dots')
spinner.start()
while True:
try:
new_job_status_code = get_job_status(job_id)
if new_job_status_code != job_status_code:
if new_job_status_code in ("UNASSIGNED", "ASSIGNED"):
spinner.info(STATUS_CODE_MESSAGES[new_job_status_code])
spinner.start("Waiting for job updates")
elif new_job_status_code == "FAILED":
spinner.fail(STATUS_CODE_MESSAGES[new_job_status_code])
spinner.start("Waiting for job updates")
else:
# the job has succeeded
spinner.stop_and_persist(
symbol='🦄'.encode('utf-8'),
text=STATUS_CODE_MESSAGES[new_job_status_code])
break
job_status_code = new_job_status_code
time.sleep(JOB_STATUS_POLL_INTERVAL_SECS)
except (KeyboardInterrupt, SystemExit):
spinner.stop()
break
def file(path, template_file, load_vars=lambda: {}):
""" Installs a template file using symlinks.
If a file already exists at the specified path and it is not a symbolic
link, then this function will print an error and return. If the file is
a symlink to the `build` directory of your dotfiles repo, then this will
check to see if the template has been modified since the file was last
built.
Args:
path (str): Filesystem path where we should install the filled out
template file.
template_file (str): The filename of the template to install. The
file should be located in the $ROOT/templates directory of this
repository.
load_vars (func): A function that will be run when the file is built to
fill in template information. This is passed in as a function so
that user input is only asked for when the file is built.
"""
spinner = Halo(text=path, spinner="dots", placement="right")
spinner.start()
if os.path.exists(path) and not os.path.islink(path):
print("Error: {} exists and is not a soft link".format(path))
spinner.fail()
return
try:
# Load template as a Jinja2 Template
template_path = os.path.join(
ROOT, os.path.join("templates", template_file)
)
template_mtime = os.path.getmtime(template_path)
with open(template_path, "r") as template_file:
template = Template(template_file.read())
build_path = os.path.join(
ROOT, os.path.join("build", os.path.basename(path))
)
if not os.path.exists(build_path):
build_mtime = 0
else:
build_mtime = os.path.getmtime(build_path)
# Build the template if the template has been modified since last build
if template_mtime > build_mtime:
# TODO (plemons): I should only do this if I actually need user
# input. Theoretically, the load_vars function could just read
# from a config file making this unnecessary
spinner.info("Asking for user input for {}".format(path))
if not os.path.exists(os.path.dirname(build_path)):
os.makedirs(os.path.dirname(build_path))
with open(build_path, 'w') as outfile:
outfile.write(template.render(**load_vars()))
path = os.path.expanduser(path)
dirpath = os.path.dirname(path)
if not os.path.exists(dirpath):
os.makedirs(dirpath)
if os.path.islink(path):
os.unlink(path)
os.symlink(build_path, path)
spinner.succeed()
except OSError as err:
print(err)
spinner.fail()
class ResourceReporter:
def __init__(self):
self.spinner = Halo(text='', spinner='dots')
def progress(self, resource):
self.spinner.start(text='[ {0: <10}] {1}'.format(
resource.status, resource.description))
def succeed(self, resource):
self.spinner.succeed(text='[ {0: <10}] {1} [{2}].'.format(
resource.status, resource.description, resource.resource_id))
def fail(self, resource):
self.spinner.fail(text='[ {0: <10}] {1}'.format(
resource.status, resource.description))
def warn(self, text):
self.spinner.warn(text=text)
def info(self, resource):
self.spinner.info(text='[ {} ] {} [{}].'.format(
resource.status, resource.description, resource.resource_id))
def report_stack_creation(self, name, resources, stack_id):
cf = boto3.session.Session().resource('cloudformation')
stack = cf.Stack(stack_id)
rmap = {r.name: r for r in resources}
completed = set()
for r in resources:
if r.status == Status.provided or r.status == Status.created:
self.succeed(r)
completed.add(r.name)
current = None
while stack.stack_status in [
'CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS'
]:
states = {
rs.logical_resource_id: rs
for rs in stack.resource_summaries.all()
}
if current:
rs = states.get(current)
r = rmap.get(current)
if rs.resource_status in ['CREATE_IN_PROGRESS']:
time.sleep(2)
continue
else:
self.report_completed_resource(completed, r, rs,
['CREATE_COMPLETE'],
Status.created)
current = None
for name, rs in iteritems(states):
r = rmap.get(name)
if name not in completed and r:
if rs.resource_status in ['CREATE_IN_PROGRESS']:
current = name
r.status = Status.creating
self.progress(r)
break
self.report_completed_resource(completed, r, rs,
['CREATE_COMPLETE'],
Status.created)
time.sleep(2)
stack = cf.Stack(stack_id)
states = {
rs.logical_resource_id: rs
for rs in stack.resource_summaries.all()
}
if current:
r = rmap.get(current)
rs = states.get(current)
self.report_completed_resource(completed, r, rs,
['CREATE_COMPLETE'], Status.created)
for name, rs in iteritems(states):
r = rmap.get(name)
if name not in completed and r:
self.report_completed_resource(completed, r, rs,
['CREATE_COMPLETE'],
Status.created)
if stack.stack_status in [
'CREATE_FAILED', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_FAILED',
'ROLLBACK_COMPLETE'
]:
raise EKSCliException('Failed to create EKS cluster {}: {}'.format(
name, stack.stack_status))
return stack
def report_completed_resource(self, completed, resource, resource_summary,
success_states, success):
if resource_summary.resource_status in success_states:
resource.status = success
resource.resource_id = resource_summary.physical_resource_id
self.succeed(resource)
else:
resource.status = Status.failed
self.fail(resource)
completed.add(resource.name)
def report_stack_deletion(self, name, resources, stack_id):
cf = boto3.session.Session().resource('cloudformation')
stack = cf.Stack(stack_id)
rmap = {r.name: r for r in resources}
completed = set()
for r in resources:
if r.status == Status.not_exist or r.status == Status.deleted or r.status == Status.provided:
self.succeed(r)
completed.add(r.name)
current = None
while stack.stack_status not in ['DELETE_COMPLETE', 'DELETE_FAILED']:
states = {
rs.logical_resource_id: rs
for rs in stack.resource_summaries.all()
}
if current:
rs = states.get(current)
r = rmap.get(current)
if rs.resource_status in [
'DELETE_IN_PROGRESS', 'CREATE_COMPLETE',
'UPDATE_COMPLETE'
]:
time.sleep(2)
continue
else:
self.report_completed_resource(completed, r, rs,
['DELETE_COMPLETE'],
Status.deleted)
current = None
for name, rs in iteritems(states):
r = rmap.get(name)
if name not in completed and r:
if rs.resource_status in [
'DELETE_IN_PROGRESS', 'CREATE_COMPLETE',
'UPDATE_COMPLETE'
]:
current = name
r.status = Status.deleting
self.progress(r)
break
self.report_completed_resource(completed, r, rs,
['DELETE_COMPLETE'],
Status.deleted)
time.sleep(2)
stack = cf.Stack(stack_id)
states = {
rs.logical_resource_id: rs
for rs in stack.resource_summaries.all()
}
if current:
r = rmap.get(current)
rs = states.get(current)
self.report_completed_resource(completed, r, rs,
['DELETE_COMPLETE'], Status.deleted)
for name, rs in iteritems(states):
r = rmap.get(name)
if name not in completed and r:
self.report_completed_resource(completed, r, rs,
['DELETE_COMPLETE'],
Status.deleted)
if stack.stack_status in ['DELETE_FAILED']:
raise EKSCliException('Failed to create EKS cluster {}: {}'.format(
name, stack.stack_status))
return
from model import *
from uuid import uuid4
from halo import Halo
import json
spinner = Halo(text_color="blue")
spinner.start()
try:
spinner.info(text="initializing test")
#User test
SessionID = str(uuid4())
spinner.info(text="Creating User => ")
user = User(name="usertest",
email="*****@*****.**",
password="******",
SessionID=SessionID)
user.add_user()
spinner.text_color = "green"
spinner.succeed(text="passed")
print('+++++++++++++++++++++++++')
spinner.text_color = "blue"
spinner.info(text="Recovering User =>")
document = user.find_user_by_SessionID(SessionID)
document['_id'] = str(document['_id'])
print(json.dumps(document, indent=4, sort_keys=True))
spinner.text_color = "green"
spinner.succeed(text="passed")
print('+++++++++++++++++++++++++')
spinner.text_color = "blue"
spinner.info(text="Loging User =>")
print(user.login(SessionID))
spinner.text_color = "green"
while True:
try:
spinner.start()
subreddit = reddit.subreddit('random')
domains = ['i.redd.it', 'i.imgur.com']
limit = None
print('Random Subreddit Is: ', subreddit)
submissions = list(subreddit.top('all', limit=limit))
submission = random.choice(submissions)
if submission.domain in domains:
im = pyimgur.Imgur(imgur_id)
uploaded_image = im.upload_image(url=submission.url)
with open('links.txt', "a") as f:
f.write(uploaded_image.link + "\n")
reddit.validate_on_submit = True
subreddit.submit(submission.title, url=uploaded_image.link)
spinner.succeed('success')
elif submission.domain not in domains:
spinner.info('domain is not in domains :(')
except Exception as e:
exc = str(str(e))
spinner.fail(text=exc)
time.sleep(60)
except KeyboardInterrupt:
spinner.warn(text='shutting down :(')
quit()
def _update_with_appimageupdatetool(self,
path_appimageupdate,
path,
update_old_data=True,
show_spinner=True):
path_to_old_appimage = path
spinner = Halo('Checking for updates', spinner='dots')
if show_spinner:
spinner.start()
_check_update_command = shlex.split(
"{au} --check-for-update {app}".format(
au=path_appimageupdate,
app=path_to_old_appimage,
))
_check_update_proc = subprocess.Popen(_check_update_command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
e_code = _check_update_proc.wait(600)
if e_code == 0:
if show_spinner:
spinner.succeed("Already up-to-date!")
return
elif e_code == 1:
if show_spinner:
spinner.info("Updates found")
spinner.start("Updating {}".format(self.app))
_update_proc = subprocess.Popen(shlex.split(
"{au} --remove-old {app}".format(
au=path_appimageupdate,
app=path_to_old_appimage,
)),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
_update_proc_e_code = _update_proc.wait(5000)
_update_proc_out, _update_proc_err = \
(x.decode() for x in _update_proc.communicate())
if _update_proc_e_code == 0:
# update completed successfully
if show_spinner:
spinner.succeed("Update Successful!")
spinner.start("Setting up new AppImage")
_file = re.findall(r"Target file: (.*)", _update_proc_out)
if len(_file) == 1:
output_file = _file[0]
if show_spinner:
spinner.info("New file name is {}".format(output_file))
if update_old_data:
_cb_data = self.appdata()
_cb_data['path'] = output_file
command_wrapper_file_path = \
os.path.join(self.cfgmgr.bin, self.app)
with open(self.app_data_path, 'w') as w:
json.dump(_cb_data, w)
with open(command_wrapper_file_path, 'w') as fp:
fp.write(
COMMAND_WRAPPER.format(
path_to_appimage=output_file))
if show_spinner:
spinner.start("Configuring desktop files...")
try:
libappimage = LibAppImage()
if libappimage.is_registered_in_system(
path_to_old_appimage):
libappimage.unregister_in_system(
path_to_old_appimage)
libappimage.register_in_system(output_file)
except LibAppImageRuntimeError:
pass # TODO: add some more stuff here
except LibAppImageNotFoundError:
pass # TODO: add some more stuff here
if show_spinner:
spinner.succeed("Done!")
else:
if show_spinner:
spinner.stop()
print(_file)
raise RuntimeError("More than one link found")
else:
# Was unsuccessful
if show_spinner:
spinner.fail("Update failed! :'(")
spinner.start("Cleaning up")
print(_update_proc_out, _update_proc_err)
elif show_spinner:
spinner.fail("Update information is not embedded within the "
"AppImage. ")
spinner.fail("Consider informing the AppImage author to add a "
".zsync file")
spinner.fail("Alternatively, pass the --no-appimageupdate option")
out, err = (x.decode() for x in _check_update_proc.communicate())
print(out, err)
if show_spinner:
spinner.stop()
def add_students(student_handles, dry, token, org, team):
"""
student_handles: github user to add (usernames)
"""
if len(student_handles) == 0:
print("required handles")
return 1
github_students = student_handles
github_organization = org
github_team = team
github_token = token
ensure_gh_token(github_token)
# TODO: use logging lib to log messages
spinner = Halo(stream=sys.stderr)
if dry:
spinner.info("Dry run")
spinner.info("fetch existing team members from GitHub")
team = Team(github_organization,
team_slug=github_team,
github_token=github_token)
num_member = len(team.members.keys())
words = (normal.txt("target team: ").kw(f"{github_team}").txt("( ").kw2(
num_member).txt(" members) "))
spinner.succeed(words.to_str())
if dry:
existed_members = set()
else:
existed_members = set(team.members.keys())
outside_users = list(set(github_students) - existed_members)
# print("Users to invite:")
# print_table(outside_users, cols=5, wide=15)
spinner.info("Check valid Github users")
invalid_id = []
spinner.start()
total = len(outside_users)
for idx, u in enumerate(outside_users, start=1):
text = "" if not dry else "[skip]: "
text += f"{idx}/{total} Check valid GitHub username : {u}"
if dry:
spinner.succeed(text)
else:
if check_is_github_user(u, github_token):
spinner.succeed(text)
else:
spinner.fail(text)
invalid_id.append(u)
if len(invalid_id) != 0:
print("Find non-existed github user names:")
# control strings take space
print_table([warn.txt("i").to_str() for i in invalid_id],
cols=5,
wide=25)
non_member_valid_users = list(set(outside_users) - set(invalid_id))
# membership info
membership_infos = {key: "unknown" for key in non_member_valid_users}
total = len(non_member_valid_users)
spinner.info("Check Membership information")
for idx, username in enumerate(non_member_valid_users, start=1):
skip = "" if not dry else "[skip]: "
spinner.start(f"{skip}{idx}/{total}: {username}")
if not dry:
res = team.get_memberships(username)
if res.status_code == 200:
membership_infos[username] = res.json()["state"]
spinner.succeed()
pending_users = [
u for u in membership_infos.keys() if membership_infos[u] == "pending"
]
no_memship_users = [
u for u in membership_infos.keys() if membership_infos[u] == "unknown"
]
print(f"Users already in pending state (total:{len(pending_users)}):")
print_table(pending_users)
print(f"Users to add (total:{len(no_memship_users)})")
print_table(no_memship_users)
print("-" * 30)
failed_users = []
spinner.info("start to invite users")
for user_name in no_memship_users:
if dry:
spinner.info(f"[Skip] add user: {user_name}")
else:
if True == add_user(team, user_name=user_name):
spinner.succeed(f"add user: {user_name}")
else:
failed_users.append(user_name)
spinner.fail(f"failed to add user: {user_name}")
failed_users = list(set(failed_users))
if len(failed_users) != 0:
print("Users failed to add")
print_table(failed_users)
spinner.succeed("Adding students successfully")
def patch_project(hw_prefix, patch_branch, source_repo, token, org, only_repo,
dry):
"""Patch to student homeworks"""
ensure_git_cached()
ensure_gh_token(token)
# init
spinner = Halo(stream=sys.stderr)
if source_repo == "":
source_repo = f"tmpl-{hw_prefix}-revise"
# Check if repo already contains the patched branch. Skip if so.
# api : https://developer.github.com/v3/git/refs/#get-a-reference
res = requests.get(
f"https://api.github.com/repos/{org}/{source_repo}/git/refs/heads/{patch_branch}",
headers=github_headers(token),
)
if res.status_code != 200: # this branch not exists on the remote
spinner.fail(
f"branch : `{patch_branch}` doesn't exist on repo:{org}/{source_repo} "
)
return
cur = Path(".")
for d in cur.glob("patch-*"):
shutil.rmtree(d)
spinner.info("delete dated folder")
spinner.start(
normal.txt("Fetch issue template").kw(patch_branch).txt(" from ").kw(
source_repo).to_str())
# Fetch patch template on the source repo
issues = get_github_endpoint_paged_list(
endpoint=f"repos/{org}/{source_repo}/issues",
github_token=token,
verbose=False)
def find_target_issue() -> Optional[Dict]:
for issue in issues:
if issue["title"].strip() == patch_branch.strip():
return issue
return None
target_issue = find_target_issue()
if not target_issue:
raise Exception(
f"cannot found issue tmpl `{patch_branch}` on `{source_repo}`")
issue_tmpl_body = target_issue["body"]
spinner.succeed()
root_folder = Path(
tempfile.mkdtemp(
prefix="patch-{}-{}-".format(
patch_branch,
datetime.now().strftime("%b%d%H%M%S")),
dir=".",
))
spinner.succeed(normal.txt("Create tmp folder ").kw(root_folder).to_str())
spinner.info(
normal.txt("Fetch soure repo").kw(source_repo).txt(
" from GitHub ").to_str())
src_repo_path = root_folder / "source_repo"
sp.run(
[
"git",
"clone",
f"https://github.com/{org}/{source_repo}.git",
src_repo_path.name,
],
cwd=root_folder,
)
src_repo = Repo(src_repo_path)
sp.run(
["git", "checkout", "--track", f"origin/{patch_branch}"],
cwd=src_repo_path,
stdout=sp.DEVNULL,
stderr=sp.DEVNULL,
)
spinner.succeed()
# Pasting changed files into students repo
src_repo_git = src_repo.git
src_repo_git.checkout(patch_branch)
changed_files, renamed_files = get_changed_files(
master_commit=src_repo.heads["master"].commit,
patch_commit=src_repo.heads[patch_branch].commit,
)
spinner.start("Fetch information for homework repo")
spinner.succeed()
if only_repo is not None:
repos = [
re for re in query_matching_repos(org,
github_repo_prefix=only_repo,
github_token=token,
verbose=False)
if re["name"] == only_repo
]
repo = next(iter(repos), None)
if repo:
spinner.info(
normal.txt("Only patch to repo : ").kw(repo["name"]).to_str())
repos = [repo]
else:
repos = query_matching_repos(org,
github_repo_prefix=hw_prefix,
github_token=token,
verbose=False)
spinner.succeed()
# Patch to student repos
student_path = root_folder / "student_repos"
student_path.mkdir()
for repo_idx, r in enumerate(repos, start=1):
pre_prompt_str = (normal.txt(f"({repo_idx}/{len(repos)})").kw(
f" {r['name']} ").to_str())
spinner.start()
# Check if repo already contains the patched branch. Skip if so.
# api : https://developer.github.com/v3/git/refs/#get-a-reference
res = requests.get(
f"https://api.github.com/repos/{org}/{r['name']}/git/refs/heads/{patch_branch}",
headers=github_headers(token),
)
if res.status_code == 200: # this branch exists in the remote
spinner.text = (
pre_prompt_str +
normal.kw(" Skip ").txt("already patched ").to_str())
spinner.succeed()
continue
spinner.text = pre_prompt_str + normal.txt(" cloning repo...").to_str()
sp.run(
["git", "clone", "--depth=1", r["html_url"]],
cwd=student_path,
stdout=sp.DEVNULL,
stderr=sp.DEVNULL,
)
hw_repo_name = r["html_url"].rsplit("/")[-1]
# open a new branch & checkout to that branch
sp.run(
["git", "checkout", "-b", patch_branch],
cwd=student_path / hw_repo_name,
stdout=sp.DEVNULL,
stderr=sp.DEVNULL,
)
# copy file to student repo
for f in changed_files.keys():
(student_path / hw_repo_name / f).parent.mkdir(parents=True,
exist_ok=True)
shutil.copyfile(src=src_repo_path / f,
dst=student_path / hw_repo_name / f)
for f in renamed_files.keys():
os.remove(student_path / hw_repo_name / f)
# changed_files = get_changed_files(
# master_commit = src_repo.heads['master'].commit,
# patch_commit = src_repo.heads[patch_branch].commit
# )
# push (publish) that branch to student repo
sp.run(
["git", "add", "."],
cwd=student_path / hw_repo_name,
stdout=sp.DEVNULL,
stderr=sp.DEVNULL,
)
# Pass if no changed
student_repo = Repo(student_path / hw_repo_name)
if len(student_repo.index.diff("HEAD")) == 0:
spinner.text = (
pre_prompt_str +
normal.kw2(" Passed ").txt("Repo no change").to_str())
spinner.succeed()
continue
sp.run(
[
"git", "commit", "-m",
f":construction_worker: Patch: {patch_branch}"
],
cwd=student_path / hw_repo_name,
stdout=sp.DEVNULL,
stderr=sp.DEVNULL,
)
spinner.text = pre_prompt_str + normal.kw(
" publish patch to remote...").to_str()
if dry:
spinner.succeed(pre_prompt_str + normal.txt(" Patched ").to_str())
continue
res = sp.run(
["git", "push", "-u", "origin", patch_branch],
cwd=student_path / hw_repo_name,
stdout=sp.DEVNULL,
stderr=sp.DEVNULL,
)
if res.returncode != 0:
spinner.text = (pre_prompt_str + warn.kw(" Failed ") +
warn.txt(" Cannot push branch ").kw2(
patch_branch).txt(" to origin").to_str())
spinner.fail()
continue
# open an pull-request on students repo
# student_repo/patch-branch -> student_repo/master
body = {
"title": f"[PATCH] {patch_branch}",
"body": issue_tmpl_body,
"head": patch_branch,
"base": "master",
}
res = requests.post(
f"https://api.github.com/repos/{org}/{r['name']}/pulls",
headers=github_headers(token),
json=body,
)
if res.status_code == 201:
spinner.text = pre_prompt_str + normal.txt(" Patched ").to_str()
spinner.succeed()
else:
spinner.text = (pre_prompt_str + warn.kw(" Failed ") +
warn.txt("Cannot create PR").kw2(patch_branch).txt(
"to origin/master").to_str())
spinner.fail()
try:
info = warn.txt(" ").txt(
res.json()["errors"][0]["message"]).to_str()
print(info)
except:
pass
continue
def announce_grade(homework_prefix, token, org, only_id, feedback_source_repo):
'''announce student grades to each hw repo'''
ensure_gh_token(token)
# TODO: use logging lib to log messages
spinner = Halo(stream=sys.stderr)
student_feedback_title = f"Grade for {homework_prefix}"
gstudents = Gstudents()
feedback_vars = gstudents.left_join(homework_prefix)
# Clone feedback repo & set needed variables
cur = Path('.')
for d in cur.glob("feedback-tmp-*"):
shutil.rmtree(d)
spinner.info("delete dated folder")
root_folder = Path(
tempfile.mkdtemp(prefix="feedback-tmp-{}-".format(
datetime.now().strftime("%b%d%H%M%S")),
dir="."))
spinner.succeed(normal.txt('Create tmp folder ').kw(root_folder).to_str())
feedback_repo_path = root_folder / 'feedbacks'
spinner.start(f"cloning feeback source repo : {feedback_source_repo}")
_, t = measure_time(sp.run)([
'git',
'clone',
f'https://github.com/{org}/{feedback_source_repo}.git',
feedback_repo_path.name,
],
cwd=root_folder,
stdout=sp.DEVNULL,
stderr=sp.DEVNULL)
spinner.succeed(
f"cloning feeback source repo : {feedback_source_repo} ... {t:4.2f} sec"
)
client = httpx.AsyncClient(headers=httpx.Headers(
{
"User-Agent": "GitHubClassroomUtils/1.0",
"Authorization": "token " + token,
# needed for the check-suites request
"Accept": "application/vnd.github.antiope-preview+json"
}))
hw_path = feedback_repo_path / homework_prefix / 'reports'
# generate feedbacks
fbs, t = measure_time(gen_feedbacks)(homework_prefix, hw_path,
feedback_vars)
spinner.succeed(f"Generate content for feedbacks ... {t:5.3f} sec")
# handle only_id
if only_id:
try:
# detect possible buggy condition
info = gstudents.get_student(only_id)
except RuntimeError as e:
print(' *=' * 30)
print('Warning!')
print(e)
return
only_repo_name = get_hw_repo_name(homework_prefix,
info['github_handle'])
fbs = list(filter(lambda fb: fb['repo_name'] == only_repo_name, fbs))
async def push_to_remote(feedback_title, feedbacks):
# push to remote
async def push_feedback(fb):
request_body = {'title': feedback_title, 'body': fb['value']}
try:
issue_num = await find_existing_issue(client, org,
fb['repo_name'],
feedback_title)
except BaseException as e:
print(f'error on {fb["repo_name"]}')
return
if issue_num:
request_body['state'] = 'open' # reopen issue
url = f"https://api.github.com/repos/{org}/{fb['repo_name']}/issues/{issue_num}"
await edit_issue_async(client, url, issue_num, request_body)
else:
url = f"https://api.github.com/repos/{org}/{fb['repo_name']}/issues"
await create_issue_async(client, url, request_body)
print(f'success {fb["repo_name"]}')
async with trio.open_nursery() as nursery:
for fb in feedbacks:
nursery.start_soon(push_feedback, fb)
_, t = measure_time(trio.run)(push_to_remote, student_feedback_title, fbs)
spinner.succeed(f"Push feedbacks to remote ... {t:5.2f} sec")
spinner.succeed(f'finished announce grade')
return
def event_times(input_file, org, token, deadline, target_team):
"""
input-file: file contains list of repo-hash.
repo-hash : string in <repo>:<hash> format
hw0-ianre657:cb75e99
"""
global github_organization
global github_token
try:
parsed_repos = get_repo_infos(input_file)
except FileNotFoundError as e:
print(str(e))
return
ensure_gh_token(token)
spinner = Halo(stream=sys.stderr)
github_organization = org
github_token = token
print(f"deadline: {deadline}")
submit_deadline = iso8601.parse_date(deadline)
submit_deadline = submit_deadline.replace(tzinfo=LOCAL_TIMEZONE)
spinner.info(f"Deadline : {submit_deadline}")
success_group = []
fail_group = []
spinner.start("Start to check late submissions")
# get team membershup info
if target_team is not None:
only_team_members = set(
Team(org=github_organization,
team_slug=target_team,
github_token=github_token).members.keys())
for idx, repo in enumerate(parsed_repos, start=1):
# print("get commit time for {}".format(repo))
if target_team is not None:
import re
user_id = re.sub("hw[\d]+-", "", repo.name)
# print(f'user_id :{user_id}')
if user_id not in only_team_members:
continue
spinner.text = f"({idx}/{len(parsed_repos)}) Checking {repo.name}"
result = getRepoCommitTime(org=github_organization,
repo=repo.name,
commit_hash=repo.commit_hash)
for r in result:
# print(r)
passed, delta = is_deadline_passed(
submit_deadline, iso8601.parse_date(r.pushed_time))
if passed:
fail_group.append({
"repo-name": r.repo,
"commit-hash": r.commit_hash,
"time-passed": delta,
"last-pushtime": r.pushed_time,
})
else:
success_group.append((r, delta))
# print(f'{r}: {delta} later')
spinner.succeed("Check finished")
print("=" * 20, "REPORT", "=" * 20)
print(f"Total submissions : {len(parsed_repos)}")
print(f"late submissions: {len(fail_group)}")
print(f"Submission Deadline: {submit_deadline}")
print(tabulate(fail_group, headers="keys"))
def write(args, generator, object_info, stream_info):
"""
Write the livestream to file in real time.
Calls previously defined private methods:
SaveStream._create_skeleton()
SaveStream._get_temp_filename()
SaveStream._rename()
Calls a public method from an external module:
DisplayStream.display()
Parameters
----------
args: Namespace
Namespace object containing all arguments that were defined in the CLI
generator: Reddit object generator
object_info: str
String denoting which Reddit objects are displayed in the stream
stream_info: str
String denoting the livestream information
Returns
-------
stream_statistics: str
String denoting the livestream statistics (Reddit objects, Subreddit
or Redditor, and duration)
"""
skeleton = SaveStream._create_skeleton(args)
stream_path = SaveStream._get_temp_filename(stream_info)
SaveStream._create_temp_file(skeleton, stream_path)
with open(stream_path, "r+", encoding = "utf-8") as existing_file:
stream_data = json.load(existing_file)
start_stream = time.mktime(time.localtime())
try:
logging.info("")
logging.info("STREAMING...")
logging.info("")
for obj in generator:
DisplayStream.display(obj)
stream_data["data"].append(obj)
existing_file.seek(0)
existing_file.truncate()
json.dump(stream_data, existing_file)
except KeyboardInterrupt:
end_stream = time.mktime(time.localtime())
duration = time.strftime("%H:%M:%S", time.gmtime(end_stream - start_stream))
stream_statistics = f"Streamed {object_info} submitted {stream_info} for {duration}."
print("\n\n")
Halo().info(Fore.YELLOW + Style.BRIGHT + "ABORTING LIVESTREAM.")
logging.info("ABORTING LIVESTREAM.")
logging.info("")
Halo().info(stream_statistics)
print()
stream_data["livestream_metadata"]["stream_duration"] = duration
stream_data["livestream_metadata"]["stream_end"] = time.strftime("%H:%M:%S", time.localtime(end_stream))
stream_data["livestream_metadata"]["stream_start"] = time.strftime("%H:%M:%S", time.localtime(start_stream))
existing_file.seek(0)
existing_file.truncate()
json.dump(stream_data, existing_file, indent = 4)
save_spinner = Halo().start("Saving livestream.")
SaveStream._rename(duration, object_info, time.strftime("%H:%M:%S", time.localtime(start_stream)), stream_path)
save_spinner.info(Fore.GREEN + Style.BRIGHT + "Livestream has been saved to file.")
logging.info("Livestream has been saved to file.")
logging.info("")
print()
return stream_statistics
def patch_project(hw_prefix, patch_branch, source_repo, token, org, only_repo):
'''Patch to student homeworks'''
# init
colorama_init(autoreset=True)
spinner = Halo(stream=sys.stderr)
if source_repo == '':
source_repo = f'tmpl-{hw_prefix}-revise'
# Check if repo already contains the patched branch. Skip if so.
# api : https://developer.github.com/v3/git/refs/#get-a-reference
res = requests.get(
f"https://api.github.com/repos/{org}/{source_repo}/git/refs/heads/{patch_branch}",
headers=github_headers(token))
if res.status_code != 200: # this branch not exists on the remote
spinner.fail(
f"branch : `{patch_branch}` doesn't exist on repo:{org}/{source_repo} "
)
return
cur = Path('.')
for d in cur.glob("patch-*"):
shutil.rmtree(d)
spinner.info("delete dated folder")
spinner.start(
f"Fetch issue template {Fore.CYAN}{patch_branch} {Fore.RESET}from {Fore.CYAN}{source_repo}"
)
# Fetch patch template on the source repo
issues = get_github_endpoint_paged_list(
endpoint=f"repos/{org}/{source_repo}/issues",
github_token=token,
verbose=False)
issue_tmpl_found = False
for i in issues:
if i['title'] == patch_branch:
issue_tmpl_found = True
issue_tmpl_body = i['body']
break
if not issue_tmpl_found:
raise Exception(
f"cannot found issue tmpl `{patch_branch}` on `{source_repo}`")
spinner.succeed()
root_folder = Path(
tempfile.mkdtemp(prefix="patch-{}-{}-".format(
patch_branch,
datetime.now().strftime("%b%d%H%M%S")),
dir="."))
spinner.succeed(f"Create tmp folder {Fore.YELLOW}{root_folder}")
spinner.info(
f"Fetch source repo {Fore.CYAN}{source_repo}{Style.RESET_ALL} from GitHub"
)
src_repo_path = root_folder / "source_repo"
sp.run([
'git',
'clone',
f'https://github.com/{org}/{source_repo}.git',
src_repo_path.name,
],
cwd=root_folder)
src_repo = Repo(src_repo_path)
sp.run(['git', 'checkout', '--track', f'origin/{patch_branch}'],
cwd=src_repo_path,
stdout=sp.DEVNULL,
stderr=sp.DEVNULL)
spinner.succeed()
# Pasting changed files into students repo
src_repo_git = src_repo.git
src_repo_git.checkout(patch_branch)
changed_files, renamed_files = get_changed_files(
master_commit=src_repo.heads['master'].commit,
patch_commit=src_repo.heads[patch_branch].commit)
spinner.start("Fetch information for homework repo")
spinner.succeed()
if only_repo is not None:
repos = [
re for re in query_matching_repos(org,
github_repo_prefix=only_repo,
github_token=token,
verbose=False)
if re['name'] == only_repo
]
repo = next(iter(repos), None)
if repo:
spinner.info(
f"Only patch to repo : {Fore.YELLOW}{repo['name']}{Style.RESET_ALL}"
)
repos = [repo]
else:
repos = query_matching_repos(org,
github_repo_prefix=hw_prefix,
github_token=token,
verbose=False)
spinner.succeed()
# Patch to student repos
student_path = root_folder / "student_repos"
student_path.mkdir()
for repo_idx, r in enumerate(repos, start=1):
pre_prompt_str = f"({repo_idx}/{len(repos)}) {Fore.YELLOW}{r['name']}{Fore.RESET}"
spinner.start()
# Check if repo already contains the patched branch. Skip if so.
# api : https://developer.github.com/v3/git/refs/#get-a-reference
res = requests.get(
f"https://api.github.com/repos/{org}/{r['name']}/git/refs/heads/{patch_branch}",
headers=github_headers(token))
if res.status_code == 200: # this branch exists in the remote
spinner.text = pre_prompt_str + \
f" {Back.GREEN}{Fore.BLACK} Skip {Style.RESET_ALL} already patched"
spinner.succeed()
continue
spinner.text = pre_prompt_str + \
f" {Fore.BLUE}cloning repo..{Fore.RESET}"
sp.run(['git', 'clone', '--depth=1', r['html_url']],
cwd=student_path,
stdout=sp.DEVNULL,
stderr=sp.DEVNULL)
hw_repo_name = r['html_url'].rsplit("/")[-1]
# open a new branch & checkout to that branch
sp.run(['git', 'checkout', '-b', patch_branch],
cwd=student_path / hw_repo_name,
stdout=sp.DEVNULL,
stderr=sp.DEVNULL)
# copy file to student repo
for f in changed_files.keys():
(student_path / hw_repo_name / f).parent.mkdir(parents=True,
exist_ok=True)
shutil.copyfile(src=src_repo_path / f,
dst=student_path / hw_repo_name / f)
for f in renamed_files.keys():
os.remove(student_path / hw_repo_name / f)
# changed_files = get_changed_files(
# master_commit = src_repo.heads['master'].commit,
# patch_commit = src_repo.heads[patch_branch].commit
# )
# push (publish) that branch to student repo
sp.run(['git', 'add', '.'],
cwd=student_path / hw_repo_name,
stdout=sp.DEVNULL,
stderr=sp.DEVNULL)
# Pass if no changed
student_repo = Repo(student_path / hw_repo_name)
if len(student_repo.index.diff("HEAD")) == 0:
spinner.text = pre_prompt_str + \
f" {Back.GREEN}{Fore.BLACK} Passed {Style.RESET_ALL} Repo no change"
spinner.succeed()
continue
sp.run([
'git', 'commit', '-m',
f':construction_worker: Patch: {patch_branch}'
],
cwd=student_path / hw_repo_name,
stdout=sp.DEVNULL,
stderr=sp.DEVNULL)
spinner.text = pre_prompt_str + \
f" {Fore.BLUE}publish patch to remote..{Fore.RESET}"
res = sp.run(['git', 'push', '-u', 'origin', patch_branch],
cwd=student_path / hw_repo_name,
stdout=sp.DEVNULL,
stderr=sp.DEVNULL)
if res.returncode != 0:
spinner.text = (
pre_prompt_str +
f" {Back.RED}{Fore.BLACK} Failed {Style.RESET_ALL}" +
f" Cannot push branch {Fore.CYAN}{patch_branch}{Fore.RESET} to origin"
)
spinner.fail()
continue
# open an pull-request on students repo
# student_repo/patch-branch -> student_repo/master
body = {
"title": f"[PATCH] {patch_branch}",
"body": issue_tmpl_body,
"head": patch_branch,
"base": "master"
}
res = requests.post(
f"https://api.github.com/repos/{org}/{r['name']}/pulls",
headers=github_headers(token),
json=body)
if res.status_code == 201:
spinner.text = pre_prompt_str + \
f" {Fore.BLACK}{Back.GREEN} Patched {Style.RESET_ALL}"
spinner.succeed()
else:
spinner.text = (
pre_prompt_str +
f" {Back.RED}{Fore.BLACK} Failed {Style.RESET_ALL}" +
f" Cannot create PR {Fore.CYAN}{patch_branch}{Fore.RESET} to origin/master"
)
spinner.fail()
try:
print(f" {Fore.RED}{res.json()['errors'][0]['message']}")
except:
pass
continue
# TODO : print summary after patch
# how many success, skiped, failed
pass
