def master_update(table_name, id_):
if request.method == 'GET':
with Connection() as cn:
model = table_label_dict[table_name].model
obj = cn.s.query(model).filter(model.id == id_).one()
form = MasterForm.generate(model, cn, obj=obj)
return render_template('master/update.html',
form=form,
button='update',
table_name=table_name)
if request.method == 'POST':
with Connection() as cn:
model = table_label_dict[table_name].model
obj = cn.s.query(model).filter(model.id == id_).one()
form = MasterForm.generate(model, cn, obj=obj)
if not form.validate_on_submit():
return render_template('master/update.html',
form=form,
button='update',
table_name=table_name)
cn.upsert_from_form(table_label_dict[table_name].model, form)
return render_template('master/update.html',
form=form,
button='update',
table_name=table_name)
def save_all_commits(self, total_page=1):
for p in range(total_page):
commits = self.get_commits(p)
with Connection() as cn:
sha_list = [c['sha'] for c in commits]
exist_sha_set = {
sha
for sha, in cn.s.query(Commit.sha).filter(
Commit.sha.in_(sha_list)).all()
}
bulk_insert_list = []
for commit in commits:
if commit['sha'] in exist_sha_set:
continue
sha = commit['sha']
message = commit['commit']['message']
parent_a = None
parent_b = None
if len(commit['parents']) > 0:
parent_a = commit['parents'][0]['sha']
if len(commit['parents']) > 1:
parent_b = commit['parents'][1]['sha']
bulk_insert_list.append({
'sha': sha,
'message': message,
'parent_a': parent_a,
'parent_b': parent_b,
'production_reported': 0,
})
cn.s.bulk_insert_mappings(Commit, bulk_insert_list)
cn.s.commit()
def auto_switch_server(self):
instance_response = self.ec2.describe_instances()
with Connection() as cn:
server_list = cn.s.query(Server).filter(
Server.is_staging == 0).all()
db_instance_id_dict = {
server.instance_id: server
for server in server_list
}
# TODO non-reserved instance
for r in instance_response['Reservations']:
for instance in r['Instances']:
server: Optional[Server] = db_instance_id_dict.get(
instance['InstanceId'])
if not server:
continue
if server.auto_start_at and server.auto_start_at < datetime.now(
):
server.auto_start_at += timedelta(days=1)
cn.s.add(server)
if instance['State']['Name'] == 'stopped':
self.ec2.start_instances(
InstanceIds=[server.instance_id])
elif server.auto_stop_at and server.auto_stop_at < datetime.now(
):
server.auto_stop_at += timedelta(days=1)
cn.s.add(server)
if instance['State']['Name'] == 'running':
self.ec2.stop_instances(
InstanceIds=[server.instance_id])
cn.s.commit()
def master_delete(table_name, id_):
if request.method == 'GET':
with Connection() as cn:
model = table_label_dict[table_name].model
obj = cn.s.query(model).filter(model.id == id_).one()
form = MasterForm.generate(model, cn, obj=obj, freeze=True)
return render_template('master/update.html',
form=form,
button='delete',
table_name=table_name)
if request.method == 'POST':
with Connection() as cn:
model = table_label_dict[table_name].model
cn.s.query(model).filter(model.id == id_).delete(
synchronize_session=False)
cn.s.commit()
return redirect(
url_for_ep('controller.master.master_list', table_name=table_name))
def server_register():
with Connection() as cn:
cn.s.add(
Server(
instance_id=request.values['InstanceId'],
name=request.values['Name'],
private_ip=request.values['PrivateIpAddress'],
))
cn.s.commit()
return 'add server to database.'
def master_show(table_name, id_):
if request.method == 'GET':
with Connection() as cn:
model = table_label_dict[table_name].model
obj = cn.s.query(model).filter(model.id == id_).one()
form = MasterForm.generate(model, cn, obj=obj, freeze=True)
return render_template('master/update.html',
form=form,
button='',
table_name=table_name)
def master_create(table_name):
if request.method == 'GET':
with Connection() as cn:
form = MasterForm.generate(table_label_dict[table_name].model, cn)
return render_template('master/update.html',
form=form,
button='create',
table_name=table_name)
if request.method == 'POST':
with Connection() as cn:
form = MasterForm.generate(table_label_dict[table_name].model, cn)
if not form.validate_on_submit():
return render_template('master/update.html',
form=form,
button='create',
table_name=table_name)
cn.upsert_from_form(table_label_dict[table_name].model, form)
return redirect(
url_for_ep('controller.master.master_create',
table_name=table_name))
def server_list():
boto = BotoSession(region_name='ap-northeast-1',
aws_access_key_id=webapp_settings['AWS_ACCESS_KEY_ID'],
aws_secret_access_key=webapp_settings['AWS_SECRET_KEY'])
ec2 = boto.client('ec2')
instance_response = ec2.describe_instances()
with Connection() as cn:
server_list = cn.s.query(Server).all()
db_instance_id_dict = {
server.instance_id: server
for server in server_list
}
hide_server_name_set = {
hide_server.name
for hide_server in cn.s.query(HideServer).all()
}
instance_list = []
# TODO non-reserved instance
for r in instance_response['Reservations']:
for instance in r['Instances']:
name_tag = [
tag for tag in instance['Tags'] if tag['Key'] == 'Name'
]
name = name_tag[0]['Value'] if name_tag else ''
if name in hide_server_name_set:
continue
server: Optional[Server] = db_instance_id_dict.get(
instance['InstanceId'])
instance_list.append({
'InstanceId':
instance['InstanceId'],
'State':
instance['State']['Name'],
'Name':
name,
'PrivateIpAddress':
instance['NetworkInterfaces'][0]['PrivateIpAddress']
if instance['NetworkInterfaces'] else '',
'Registered':
instance['InstanceId'] in db_instance_id_dict,
'Staging':
server and server.is_staging,
'AutoStartAt':
safe_strftime(server and server.auto_start_at),
'AutoStopAt':
safe_strftime(server and server.auto_stop_at),
'Id':
server and server.id,
})
instance_list.sort(key=lambda x: x['Name'] + '____' + x['InstanceId'])
return render_template('server/list.html', instance_list=instance_list)
def send_message_list(webhook_id: int, message_list: List[str]):
with Connection() as cn:
print('sending to chat...')
webhook = cn.s.query(GoogleChatWebhook).filter(GoogleChatWebhook.id == webhook_id).one()
headers = {"Content-Type": "application/json"}
for message in message_list:
request = urllib.request.Request(
webhook.url, data=json.dumps({'text': message}).encode('utf-8'),
method='POST', headers=headers)
with urllib.request.urlopen(request) as res:
json_dict = json.load(res)
print(json_dict)
def check_newest_hash(self):
req = urllib.request.Request(webapp_settings['sha_url'])
with urllib.request.urlopen(req) as res:
sha = re.sub(r' .+', '',
res.read().decode().replace('commit ', ''))
with Connection() as cn:
temp_sha_list = self.get_sha_list(cn, sha, set())
if not temp_sha_list:
return
sha_list = []
for sha, message in temp_sha_list:
if message.find('Merge branch ') > -1:
continue
sha_list.append((sha, message))
message = '以下の内容がリリースされました！\n' + '\n'.join(
[s for _, s in sha_list])
if webapp_settings.get('google_chat_url'):
# notify to google chat
headers = {"Content-Type": "application/json"}
req = urllib.request.Request(
webapp_settings.get('google_chat_url'),
data=json.dumps({
'text': message
}).encode('utf-8'),
method='POST',
headers=headers)
with urllib.request.urlopen(req) as res:
print(res.read())
if webapp_settings.get('slack_url'):
headers = {"Content-Type": "application/json"}
req = urllib.request.Request(webapp_settings.get('slack_url'),
data=json.dumps({
'text': message
}).encode('utf-8'),
method='POST',
headers=headers)
with urllib.request.urlopen(req) as res:
print(res.read())
cn.s.query(Commit).filter(
Commit.sha.in_([id_ for id_, _ in temp_sha_list
])).update({'production_reported': 1},
synchronize_session=False)
cn.s.commit()
print(message)
def get_new_message_list(channel_id: int):
with Connection() as cn:
token, channel = cn.s.query(
SlackChannel.token,
SlackChannel.channel).filter(SlackChannel.id == channel_id).one()
user_dict = {
user.user: user.other_name
for user in cn.s.query(SlackUser).all()
}
with urllib.request.urlopen(
f'https://slack.com/api/channels.history?token={token}&channel={channel}'
) as res:
json_dict = json.load(res)
print(json_dict)
messages = sorted(json_dict['messages'], key=lambda x: x.get('ts', ''))
client_msg_id_list = [
id_ for id_, in cn.s.query(SlackMessage.client_msg_id).filter(
SlackMessage.client_msg_id.in_(
[message.get('client_msg_id')
for message in messages])).all()
]
message_list = []
insert_msg_id_list = []
for message in messages:
if not (message.get('user') and message.get('text')
and message.get('client_msg_id')):
continue
if message.get('client_msg_id') in client_msg_id_list:
continue
time_stamp = message.get('ts', '')
if time_stamp:
time_stamp = datetime.fromtimestamp(
float(time_stamp), jst).strftime('%m/%d %H:%M:%S')
text = message['text']
for user, name in user_dict.items():
text = text.replace(user, name)
message_list.append(user_dict[message['user']] + ':[' +
time_stamp + '] ' + text)
insert_msg_id_list.append(
{'client_msg_id': message['client_msg_id']})
cn.s.bulk_insert_mappings(SlackMessage, insert_msg_id_list)
cn.s.commit()
return message_list
def save_all_issues(self, total_page=1, since=None):
for p in range(total_page):
issues = self.get_issues_all(p, since)
with Connection() as cn:
number_list = [c['number'] for c in issues]
exist_issue_dict = {
issue.number: issue
for issue in cn.s.query(Issue).filter(
Issue.number.in_(number_list)).all()
}
for issue in issues:
db_issue = exist_issue_dict.get(issue['number'], Issue())
db_issue.number = issue['number']
db_issue.state = issue['state']
db_issue.title = issue['title']
db_issue.body = issue['body']
db_issue.labels = ','.join(
[label['name'] for label in issue['labels']])
db_issue.assignee = issue['assignee']['login'] if issue[
'assignee'] else None
cn.s.add(db_issue)
cn.s.commit()
def master_list(table_name):
if request.method == 'GET':
with Connection() as cn:
form = MasterSearchForm.generate(
table_label_dict[table_name].model, cn)
count = int(request.values.get('count', 100))
page = int(request.values.get('page', 0))
list_query = cn.s.query(table_label_dict[table_name].model)
# TODO filter data
list_data = [
r.__dict__
for r in list_query.limit(count).offset(count * page).all()
]
columns = [
column.name for column in
table_label_dict[table_name].model.__table__.columns
]
return render_template('master/list.html',
form=form,
list_data=list_data,
columns=columns,
table_name=table_name)
def check_and_update_pull_request(self):
basic_digest = base64.b64encode(
(webapp_settings["owner"] + ':' +
webapp_settings["token"]).encode()).decode()
# TODO too many pull_requests
req = urllib.request.Request(
f'https://api.github.com/repos/{webapp_settings["owner"]}/{webapp_settings["repo"]}/pulls?state=all',
headers={'Authorization': f'Basic {basic_digest}'})
with urllib.request.urlopen(req) as res:
pull_request_list = json.loads(res.read())
github_pull_request_list = [{
'number': p['number'],
'state': p['state'],
'sha': p['head']['sha'],
'title': p['title'],
'ref': p['head']['ref'],
'login': p['user']['login'],
} for p in pull_request_list]
boto = BotoSession(
region_name='ap-northeast-1',
aws_access_key_id=webapp_settings['AWS_ACCESS_KEY_ID'],
aws_secret_access_key=webapp_settings['AWS_SECRET_KEY'])
ec2 = boto.client('ec2')
with Connection() as cn:
db_pull_request_list = cn.s.query(PullRequest).filter(
PullRequest.number.in_(
[p['number'] for p in github_pull_request_list])).all()
# insert pull requests
db_number_dict = {p.number: p for p in db_pull_request_list}
stopped_server_list = cn.s.query(Server).filter(
Server.is_staging == 1, ~cn.s.query(PullRequest).filter(
PullRequest.server_id == Server.id, PullRequest.state
== 'open').exists()).all()
for pull_request in github_pull_request_list:
if pull_request['number'] not in db_number_dict.keys():
if not stopped_server_list:
continue
db_schema = ''
if not db_schema:
github_user = cn.s.query(GitHubUser).filter(
GitHubUser.login == pull_request['login']).first()
if not github_user:
github_user = cn.s.query(GitHubUser).first()
db_schema = github_user.db_schema
# insert
new_db_pull_request = PullRequest(
number=pull_request['number'],
state=pull_request['state'],
sha=pull_request['sha'],
title=pull_request['title'],
ref=pull_request['ref'],
is_launched=0,
db_schema=db_schema,
)
if new_db_pull_request.state == 'open':
server = stopped_server_list.pop()
new_db_pull_request.server_id = server.id
ec2.start_instances(InstanceIds=[server.instance_id])
# drop / create and copy database
try:
subprocess.check_output(
f'{webapp_settings["base_dir"]}/setup_db.sh {db_schema} {server.db_schema}'
f' {webapp_settings["mysql_user"]} {webapp_settings["mysql_pw"]}'
f' {webapp_settings["mysql_host"]} {webapp_settings["mysql_port"]}',
shell=True)
except Exception as ex:
print(ex)
print(traceback.format_exc())
raise ex
if self.token:
self.post_and_set_check_run(
new_db_pull_request, pull_request, server)
cn.s.add(new_db_pull_request)
else:
# update (if status was changed)
db_pull_request = db_number_dict[pull_request['number']]
if db_pull_request.sha != pull_request['sha']:
db_pull_request.sha = pull_request['sha']
db_pull_request.is_launched = 0
if self.token:
server = db_pull_request.server
self.post_and_set_check_run(
db_pull_request, pull_request, server)
cn.s.add(server)
if db_pull_request.state != pull_request['state']:
if db_pull_request.state == 'open':
# close
ec2.stop_instances(InstanceIds=[
db_pull_request.server.instance_id
])
db_pull_request.state = pull_request['state']
cn.s.add(db_pull_request)
cn.s.commit()
if self.token:
server_list = cn.s.query(Server, PullRequest).join(
PullRequest, Server.id == PullRequest.server_id).filter(
Server.is_staging == 1,
PullRequest.state == 'open').all()
for server, pr in server_list:
try:
req = urllib.request.Request(server.check_url)
with urllib.request.urlopen(req) as res:
print(res.read())
# when come this line, server returns 20X/30X
self.patch_check_run(pr.check_run_id)
pr.check_run_id = None
cn.s.add(pr)
cn.s.commit()
except Exception as ex:
print(ex)
print(traceback.format_exc())
if hasattr(ex, 'fp') and hasattr(
ex.fp, 'read') and callable(ex.fp.read):
print(ex.fp.read())
return github_pull_request_list
def get_watch_link_list() -> List[WatchLink]:
with Connection() as cn:
return cn.s.query(WatchLink).all()
import socket
import subprocess
from model import PullRequest, Server
from mysql_dbcon import Connection, webapp_settings
if __name__ == '__main__':
host = socket.gethostname()
ip = socket.gethostbyname(host)
need_launch = False
with Connection() as cn:
pull_requests = cn.s.query(PullRequest).join(
PullRequest.server).filter(Server.private_ip == ip,
PullRequest.state == 'open',
PullRequest.is_launched == 0).all()
if len(pull_requests) > 0:
print('set unprocessed pull request...')
pr = pull_requests[0]
res = subprocess.check_output(
f'{webapp_settings["base_dir"]}/client_pull.sh {webapp_settings["target_dir"]} {pr.sha}',
shell=True)
pr.is_launched = 1
cn.s.commit()
need_launch = True
if len(pull_requests) > 1:
print(
'delete pull requests, because there are too many pull requests...'
)
cn.s.query(PullRequest).filter(
PullRequest.id.in_([p.id for p in pull_requests[1:]