def getArguments():
if len(sys.argv) < 6:
print("Please provide at least 5 Parameter!")
return
username = sys.argv[2]
password = sys.argv[3]
baseUrl = sys.argv[4]
outputFolder = sys.argv[5]
genOutPutFolder = sys.argv[6]
os.system("echo machine " +
baseUrl.replace("http://", "").replace("https://", "") +
" >> ~/.netrc")
os.system("echo login " + username + " >> ~/.netrc")
os.system("echo password " + password + " >> ~/.netrc")
if sys.argv[1] == "bitbucket":
print("Using BitBucket as RMS")
bb.getProjects(baseUrl, username, password, outputFolder,
genOutPutFolder)
elif sys.argv[1] == "gitea":
print("Using Gitea as RMS")
gitea.getProjects(baseUrl, username, password, outputFolder,
genOutPutFolder)
elif sys.argv[1] == "github":
print("Using Github as RMS")
github.getProjects(baseUrl, username, password, outputFolder,
genOutPutFolder)
def data_loading(request, content_type, htmlname):
params = content_type_dict[content_type].copy()
repo = params["repo"]
query = params["query"].format(htmlname)
params.update({"htmlname": htmlname})
if not Github.has_already_created(repo, htmlname):
tweet_list = []
myTwitterAction = Twitter.MyTwitterAction()
if content_type == "legends":
tweet_list = myTwitterAction.search_tweet_list_universal(query,
amount=50)
elif content_type == "overseas":
tweet_list = myTwitterAction.search_tweet_list(query, amount=50)
try:
tweet_list = translate_tweet_list(tweet_list)
except:
pass
else:
tweet_list = myTwitterAction.search_tweet_list(query, amount=50)
git_json = {}
git_json.update({"tweet_list": tweet_list})
text = json.dumps(git_json, ensure_ascii=False, indent=4)
Github.upload(repo, htmlname, text)
return render(request, f"ranking/ranking.html", params)
def gen_page_dict_list():
global PAGE_DICT_LIST
PAGE_DICT_LIST = []
for book_id in Github.seach_page_list(repo):
json_str = Github.load(repo, book_id)
myJson = MyJson.MyJson()
page_dict = myJson.read(json_str)
with InsertableDAO("TAG_PAGE_LIST") as dao:
for tag_name in gen_tag_list():
condition = ((tag_name.capitalize()
in page_dict["description"])
or (tag_name.upper() in page_dict["description"])
or (tag_name.lower() in page_dict["description"])
or (tag_name in page_dict["title"])
or (tag_name.upper() in page_dict["title"])
or (tag_name.lower() in page_dict["title"])
or (tag_name.capitalize() in page_dict["title"]))
if condition:
try:
dao.insert(f"('{tag_name}', '{book_id}')")
except:
pass
PAGE_DICT_LIST.append({
"title": page_dict["title"],
"book_id": page_dict["book_id"],
"img": page_dict["img"]
})
PAGE_DICT_LIST.reverse()
def make_local_repo(self, repo_url) -> bool:
"""Downloads repo zip file to /tmp and expands."""
repo_name = Github.repo_url_to_name(repo_url)
LocalRepo.clean_up()
if not Github.download_repo(repo_url):
self.log.error(f"Failed to download {repo_name}.")
return False
self.log.debug(f"{repo_name} downloaded to local storage.")
LocalRepo.expand_repo()
return True
def gen_page_dict_list():
global PAGE_DICT_LIST
PAGE_DICT_LIST = []
for book_id in Github.seach_page_list(repo):
json_str = Github.load(repo, book_id)
myJson = MyJson.MyJson()
page_dict = myJson.read(json_str)
PAGE_DICT_LIST.append({
"title": page_dict["title"],
"book_id": page_dict["book_id"],
"img": page_dict["img"]
})
PAGE_DICT_LIST.reverse()
def genPageDict():
global category_list
page_dict = {}
for category in category_list:
category_dict = {}
for htmlname in Github.seach_page_list(repo, category):
mk = Github.load(repo, category + "/" + htmlname)
params = grep_param(mk, ["title", "description", "img"])
if "escape" in params:
continue
params.update({"category": category, "htmlname": htmlname})
category_dict[htmlname] = params
page_dict[category] = category_dict
return page_dict
def page(request, category, htmlname):
mk = Github.load(repo, category + "/" + htmlname)
tableIndex = TableIndex(mk)
mk = tableIndex.rebuild_mk()
md = markdown.Markdown()
htmltext = md.convert(mk)
params = {
"mk": mk,
"htmltext": htmltext,
"site_name": site_name,
"category": category,
"favicon": favicon,
"htmlname": htmlname,
"category_list": category_list,
}
params.update(
grep_param(mk, ["title", "description", "img", "category_script"]))
if category == "slides":
return render(request, "blog/non_base.html", params)
relation_list = []
if "category_script" in params:
category_script = params["category_script"]
category_dict = page_dict[category]
for page_name, category_dict in category_dict.items():
if eval(category_script):
relation_list.append(category_dict)
params.update({"index_table": tableIndex.index_table})
params.update({"relation_list": relation_list})
if request.GET.get("raw"):
return render(request, f"blog/raw.html", params)
return render(request, f"blog/techblog_ver3/page/page.html", params)
def page(request, htmlname, pagetype):
explain = ""
metadata = {}
for content in ranking_list:
if content["name"] == htmlname:
metadata = content
explain = metadata["description"]
if explain == "":
explain = site_explain
result = seach(request)
if result:
return result
params = {
"title": htmlname + get_pagetype_title(pagetype),
"description": explain,
"favicon": favicon,
"img": img,
"repo": repo,
"htmlname": htmlname,
"explain": explain
}
if Github.has_already_created(repo, htmlname):
return render(request, f"fanstatic/dashboard/{pagetype}", params)
else:
#処理は次のページに任せて、まずは飛ぶ
return render(request, f"fanstatic/dashboard/data_loading.html",
params)
def page(request, htmlname):
plain_html = Github.load(repo, htmlname)
#md = markdown.Markdown()
#htmltext = md.convert(mk)
params = {"main_contents": plain_html}
#/Users/minegishirei/myworking/docker/django3/app/d3js/templates/d3js/base.html
return render(request, f"d3js/base.html", params)
def main():
tag_list = Github.seach_page_list(repo)
for htmlname in tag_list:
Github.delete_page(repo, htmlname)
print("mainloop:", htmlname, "has deleted")
myTwitterAction = Twitter.MyTwitterAction()
tweet_list = myTwitterAction.search_tweet_list(
'"' + htmlname + '"' + " lang:ja min_faves:100", amount=50)
git_json = {}
git_json.update({"tweet_list": tweet_list})
text = json.dumps(git_json, ensure_ascii=False, indent=4)
Github.upload(repo, htmlname, text)
print("mainloop:", htmlname, "is created")
time.sleep(4)
def on_badresult_clicked(self):
token = self._db.get_oauth()
if not token:
self._frontend_status = "Error: run 'quickopen oauth' first"
return
# get a debug version of the query
q = self._create_query()
q.debug = True
result = self._db.search(q)
badresult = {"query": q.as_dict(), "result": result.as_dict()}
import json
badresult_text = json.dumps(badresult, indent=2)
import Github
import GithubException
try:
g = Github.Github(token)
quickopen = g.get_user("natduca").get_repo("quickopen")
title = "BadResult: " + self._filter_text
body = "```json\n" + badresult_text + "\n```"
issue = quickopen.create_issue(title, body)
self._frontend_status = "Created issue #" + str(issue.number)
except GithubException.GithubException as e:
self._frontend_status = "Error: " + str(e)
def main2():
repo = "overseas"
for htmlname in Github.seach_page_list(repo):
tweet_list = []
myTwitterAction = Twitter.MyTwitterAction()
query = htmlname + " lang:en min_faves:100"
tweet_list = myTwitterAction.search_tweet_list(query, amount=50)
tweet_list = translate_tweet_list(tweet_list)
git_json = {}
git_json.update({"tweet_list": tweet_list})
text = json.dumps(git_json, ensure_ascii=False, indent=4)
Github.delete_page(repo, htmlname)
Github.upload(repo, htmlname, text)
time.sleep(4)
def __init__(self, page_name):
super().__init__()
json_str = Github.load(repo, page_name)
myJson = MyJson.MyJson()
params = myJson.read(json_str)
params["title"] = "書籍まとめ | " + params["title"]
self.comdict.update(params)
def github_sign_up(event, context):
params = event["queryStringParameters"]
try:
user = ids.github.get_user(params["username"])
Github.add_to_members(user)
return {
"message":
"You have been added to the Github organization. Check your email to accept the invitation.",
"status": "OK"
}
except github.GithubException as e:
return {
"message": "Oh no! Something went wrong.",
"status": "ERROR",
"github_response": str(e)
}
def set_user_committees(my_committees, token):
user = User.get_member_info(user_id_token=token)
github_username = user["github_username"]
for committee in all_committees:
myfirebase.put('/members/' + user["uid"], "/committees/" + committee,
committee in my_committees)
for committee in my_committees:
github_team_id, slack_channel_id = get_committee_info(committee)
if github_team_id is not None:
Github.add_member_to_team(github_username, github_team_id)
if slack_channel_id is not None:
Slack.invite_user_to_channel(slack_channel_id,
email=user["rowan_email"])
def routine_update():
global routine_time
dt_now = datetime.datetime.now()
now = dt_now.strftime('%Y%m%d')
if routine_time != now:
routine_time = now
content_type_dict["page"].update({
"tag_list":
Github.seach_page_list("engineer_rank"),
})
content_type_dict["overseas"].update({
"tag_list":
Github.seach_page_list("overseas"),
})
content_type_dict["legends"].update({
"tag_list":
Github.seach_page_list("legends"),
})
def page(request, content_type, htmlname, pagetype):
params = content_type_dict[content_type].copy()
repo = params["repo"]
params.update({"htmlname": htmlname})
if Github.has_already_created(repo, htmlname):
return render(request, f"ranking/{pagetype}", params)
else:
#処理は次のページに任せて、まずは飛ぶ
return render(request, f"ranking/data_loading.html", params)
def page(request, htmlname):
if htmlname == "about.html":
return about(request)
if "reload" in request.GET:
Github.delete_page(repo, htmlname)
try:
upload_network_json(htmlname)
except Twitter.MyTwitterException:
return render(request, "fanstatic/dashboard/twitter_network_busy.html",
params)
params = {
"information_page_link": information_page_link,
"acount_name": htmlname,
"title": htmlname + " " + title,
"repo": repo,
"favicon": favicon,
"description": description,
"img": img
}
return render(request, "fanstatic/dashboard/twitter_network.html", params)
def content_index(request, content_type):
dt_now = datetime.datetime.now()
all_ranking_filename = content_type + dt_now.strftime('%Y%m%d%H') + ".json"
params = (content_type_dict[content_type]).copy()
repo = params["repo"]
tweet_list = []
if not Github.has_already_created("content_type", all_ranking_filename):
for tag in params["tag_list"]:
json_string = Github.load(repo, tag)
if len(json_string) < 10:
continue
tweet_list = json.loads(json_string)["tweet_list"]
tweet_list = sort_tweet_list(tweet_list)
tweet_list = tweet_list[:50]
text = json.dumps(tweet_list, ensure_ascii=False, indent=4)
Github.upload("content_type", all_ranking_filename, text)
params.update({
"all_ranking_file": all_ranking_filename,
"tweet_list": tweet_list
})
return render(request, f"ranking/index.html", params)
def sitemap(request):
acount_list = Github.seach_page_list(repo)
page_list = []
url_template_list = [
#"http://fanstatic.short-tips.info/page/{}/dashboard.html",
#"http://fanstatic.short-tips.info/page/{}/charts.html",
"http://fanstatic.short-tips.info/page/{}/word_cloud.html"
#"http://fanstatic.short-tips.info/page/{}/tables.html"
]
for acount in acount_list:
for url_base in url_template_list:
page_list.append(url_base.format(acount))
acount_list = Github.seach_page_list("twitter_network")
url_template_list = ["http://fanstatic.short-tips.info/twitter_network/{}"]
for acount in acount_list:
for url_base in url_template_list:
page_list.append(url_base.format(acount))
page_list.append("http://fanstatic.short-tips.info/twitter_network/")
page_list.append(
"http://fanstatic.short-tips.info/twitter_network/about.html")
return Sitemap.sitemap(request, page_list)
def index(request):
page_list = Github.seach_page_list(repo)
htmlname = "all_page.html"
params = {
"information_page_link": information_page_link,
"title": title,
"repo": repo,
"page_list": page_list,
"favicon": favicon,
"description": description,
"img": img
}
return render(request, "fanstatic/dashboard/twitter_network_index.html",
params)
def data_loading(request, htmlname):
result = seach(request)
if result:
return result
params = {
"title": "twitterアカウント分析サイト",
"description": site_explain,
"favicon": favicon,
"img": img,
"repo": repo,
"htmlname": htmlname,
}
if not Github.has_already_created(repo, htmlname):
myTwitterAction = Twitter.MyTwitterAction()
tweet_list = myTwitterAction.search_tweet_list(htmlname, amount=50)
git_json = {}
git_json.update({"tweet_list": tweet_list})
git_json.update({"wordcloud": genWordList(tweet_list)})
text = json.dumps(git_json, ensure_ascii=False, indent=4)
Github.upload("twitter_json", htmlname, text)
return render(request, f"fanstatic/dashboard/dashboard.html", params)
def page(request, htmlpage):
if not Github.has_already_created(repo, htmlpage):
add_to_github(htmlpage)
paramFactory = PageParamFactory()
if request.POST.get("title"):
context = {
'title': request.POST['title'],
'comment': request.POST['comment'],
'good': 0
}
params = paramFactory.build(htmlpage, context)
context = False
else:
params = paramFactory.build(htmlpage)
return render(request, "oreilly/oreilly_base.html", params)
def all_page(request):
result = seach(request)
if result:
return result
page_list = Github.seach_page_list(repo)
htmlname = "all_page.html"
params = {
"title": "twitterアカウント分析サイト",
"description":
"あなたのtwitterアカウントのアカウントをフォロワー数や年齢層からツイートコンテンツの内容までの観点で分析します。",
"favicon": favicon,
"img": img,
"repo": repo,
"htmlname": htmlname,
}
params.update({"ranking_list": page_list})
return render(request, f"fanstatic/dashboard/top/{htmlname}", params)
def page(self, request, htmlpage):
if not Github.has_already_created(repo, htmlpage):
add_to_github(htmlpage)
paramFactory = PageParamFactory()
if request.POST.get("title"):
context = {
'title': request.POST['title'],
'comment': request.POST['comment'],
'good': 0
}
params = paramFactory.build(htmlpage, context)
del request.POST["title"]
del request.POST["comment"]
else:
params = paramFactory.build(htmlpage)
return render(request, "source_code/source_code_base.html", params)
def request_oauth_token():
""" Prompts user for credentials and returns Oauth token from GitHub """
print "Requesting an oauth token from GitHub."
print "Your username and password will not be stored."
username = raw_input("GitHub Username: "******"GitHub Password: "******"public_repo"],
note="quickopen")
return auth.token
except GithubException.GithubException as e:
print "Error: " + str(e)
return None
def __init__(self, page_name, new_dict=False):
super().__init__()
def dub_check(comment_list, new_dict):
for old_dict in comment_list:
if old_dict["title"] == new_dict["title"]:
return True
return False
comment_list = []
if Github.has_already_created(repo_com, page_name):
json_str = Github.load(repo_com, page_name)
myJson = MyJson.MyJson()
comment_list = myJson.read(json_str)
if new_dict and (not dub_check(comment_list, new_dict)):
Github.delete_page(repo_com, page_name)
comment_list.append(new_dict)
json_str = myJson.write(comment_list)
Github.upload(repo_com, page_name, json_str)
else:
Github.upload(repo_com, page_name, str(comment_list))
self.comdict.update({"comment_list": comment_list})
def run(self, job):
repo_url, repo_name = job["repo_url"], Github.repo_url_to_name(
job["repo_url"])
self.log.debug(f"Inspecting {repo_name}.")
timestamp = int(datetime.now().timestamp())
if not self.make_local_repo(repo_url):
self.log.error(f"Unable to make local copy of {repo_name}.")
self.sql.record_inspection(repo_url, timestamp, "error")
self.sql.check_in(repo_url)
return None
to_inspect = self.make_inspection_list(job)
if to_inspect == []:
self.log.debug(f"Found nothing to inspect.")
self.sql.record_inspection(repo_url, timestamp, "none")
self.sql.check_in(repo_url)
return None
findings = self.inspect_docs(to_inspect)
self.log.debug(findings)
if findings == False:
self.log.error(f"Unable to inspect {repo_name}.")
self.sql.record_inspection(repo_url, timestamp, "error")
self.sql.check_in(repo_url)
return None
summary = Reports.summarize(findings)
if summary["count_typos"] == summary["count_bad_links"] == 0:
self.log.debug(f"Zero findings for {repo_name}.")
self.sql.record_inspection(repo_url, timestamp, "none")
self.sql.check_in(repo_url)
return None
metadata = Reports.metadata(repo_url, timestamp, VER, job, to_inspect)
details = Reports.details(findings)
report = Reports.prepare_report(metadata, summary, details)
self.mongo.write_report(report)
self.log.debug(f"Report written for {repo_name}.")
self.sql.record_inspection(repo_url, timestamp, "report")
self.sql.check_in(repo_url)
self.log.debug(f"Inspection of {repo_name} completed and logged.")
return None
def add_to_github(book_id):
oreilly = Oreilly.Oreilly(book_id)
params = oreilly.get_params()
myJson = MyJson.MyJson()
Github.upload(repo, book_id, myJson.write(params))
def __init__(self, page_name):
super().__init__()
json_str = Github.load(repo, page_name)
myJson = MyJson.MyJson()
self.comdict.update(myJson.read(json_str))