def back_ec2_create_ec2(param):
instance = ec2.run_instances(
BlockDeviceMappings=
[ # 이게 기본 부트 볼륨으로 지정이 안됨.. /dev/sda1 같은거로 바꾸고, VolumeType, IOPS 세팅 피룡함
{
"DeviceName": "/dev/sda1",
"Ebs": {
"DeleteOnTermination": True,
"VolumeSize": param["ssd"],
"VolumeType": "gp3",
"Iops": param["iops"],
"Throughput": 125,
"DeleteOnTermination": True,
}
}
],
ImageId=param["os"],
InstanceType=param["plan"],
KeyName=param["keypair"],
MaxCount=1,
MinCount=1,
Monitoring={'Enabled': True},
NetworkInterfaces=[
{
"AssociatePublicIpAddress": True,
"DeviceIndex": 0,
'SubnetId': param["subnetid"],
'Groups': param["security-group-id"],
"DeleteOnTermination": True,
},
]
# UserData='string', # StackScript if exists
)
cloud_id = param["cloudid"]
instance_id = instance["Instances"][0]["InstanceId"]
cloud = Cloud.query.filter_by(id=cloud_id).first()
cloud.aws_instance_id = instance_id
cloud.status = "Running"
secgroup = SecurityGroup.query.filter_by(
sec_group_id=param["security-group-id"][0]).first()
secgroup.associated_to = cloud_id
db.session.add(cloud)
db.session.add(secgroup)
db.session.commit()
q.enqueue(back_update_ec2_info, instance_id)
return instance
def delete_cloud(instance_id):
cloud_with_user = db.session.query(Cloud, User).join(User).filter(
and_(Cloud.user_id == current_user.id,
Cloud.id == instance_id)).first()
if cloud_with_user is not None:
try:
aws_instance_id = cloud_with_user.Cloud.aws_instance_id
print("[Debug] - {}".format(aws_instance_id))
parameter = {
"instance_id": cloud_with_user.Cloud.aws_instance_id,
}
job = q.enqueue(delete_ec2, parameter)
flash('{} was Terminated.'.format(cloud_with_user.Cloud.hostname),
'success')
except Exception as e:
db.session.rollback()
message = Markup("<strong>Error!</strong> Eroror{} ".format(e))
flash(message, 'danger')
return redirect(url_for('cloud.all_clouds'))
def add_cloud():
form = CloudForm(request.form)
plans = Plan.query.all()
os_list = Oslist.query.all()
keypair_list = db.session.query(
Keypair.id, Keypair.name).filter(Keypair.user_id == current_user.id)
sec_list = db.session.query(SecurityGroup).filter(
SecurityGroup.user_id == current_user.id)
credit_sum = db.session.query(
Balance.balance).filter_by(user_id=current_user.id).scalar()
app_name = form["appname"]
if credit_sum < 1: # 관리자 승인된 크레딧을 1원이라도 충전하지 않았을 경우
print("크레딧이 없습니다 같은 메세지")
abort(403)
if request.method == 'POST':
if form.validate_on_submit():
try:
plan_id = form.data["plan"]
sec_id = form.data["secgroup"]
get_sec_id = db.session.query(
SecurityGroup.sec_group_id).filter(
SecurityGroup.id == sec_id).scalar()
get_aws_plan = db.session.query(
Plan.aws_plan, Plan.ssd,
Plan.iops).filter(Plan.id == plan_id)[0]
param_plan = get_aws_plan[0]
param_ssd = get_aws_plan[1]
param_iops = get_aws_plan[2]
vpc_info = db.session.query(
VPC.vpc_id, VPC.inter_gw_id, VPC.default_subnet_id,
VPC.default_sec_id,
VPC.id).filter(VPC.user_id == current_user.id)[0]
aws_image = db.session.query(
Oslist.aws_image_id,
Oslist.os_name).filter(Oslist.id == form.data["os"])[0]
vpc_id = vpc_info[4]
vpc_default_subnetid = vpc_info[2]
vpc_default_secid = vpc_info[3]
keypair_id = form.data["keypair"]
if check_environment(current_user.id) == True:
# 기존 키 페어 찾아서 클라우드에 반영
get_keypair = db.session.query(
Keypair.name, Keypair.keytoken).filter(
Keypair.id == keypair_id).first()
keypairname_formatted = "{}_{}".format(
get_keypair.keytoken, get_keypair.name)
# DB에 기록
new_cloud = Cloud(form.data["Hostname"], form.data["plan"],
current_user.id, form.data["os"],
"Queued", "Requesting", "Seoul",
keypair_id, vpc_id, "Requesting",
"creating", "", sec_id)
db.session.add(new_cloud)
db.session.flush()
db.session.refresh(new_cloud)
assigned_id = new_cloud.id
print("AssignedId : {}".format(assigned_id))
db.session.commit()
parameter = {
"plan": param_plan,
"iops": param_iops,
"ssd": param_ssd,
"os": aws_image.aws_image_id,
"subnetid": vpc_default_subnetid,
"keypair": keypairname_formatted,
"security-group-id": [get_sec_id],
"cloudid": assigned_id,
"os_name": aws_image.os_name,
"hostname": new_cloud.hostname,
"vpc_id": vpc_id
}
job = q.enqueue(back_ec2_create_ec2, parameter)
print("Task ({}) added to queue at {}".format(
job.id, job.enqueued_at))
else:
raise Exception("관리자에게 문의해 주세요.")
# 2. DB 에 기록..
# new_cloud = Cloud(form.name.data, form.
message = Markup(
"<strong>Well done!</strong> Cloud Deploy Request successfully!"
)
flash(message, 'success')
return redirect(url_for('cloud.all_clouds'))
except Exception as mesg:
db.session.rollback()
# 조금있다가 생각
# if vpc_id:
# print("vpcId : {} ".format(vpc_id))
# vpc_select = db.session.query(VPC.vpc_id).filter(VPC.id == vpc_id)
# back_ec2_delete_vpc(vpc_select)
# SDK rollback implement need
message = Markup("<strong>내부 API 에러{}".format(mesg))
flash(message, 'danger')
else:
print("GET")
return render_template('cloud/add.html',
form=form,
planlist=plans,
oslist=os_list,
keypair=keypair_list,
secgroup=sec_list)
def train(bot_guid):
code, user_id = checkAuth(request)
# code = 200
# user_id = 16
if code == 200:
bot = Bot.query.filter_by(bot_guid=bot_guid).first()
if bot:
if bot.user_id == user_id:
for key in redis_db.scan_iter(match=bot_guid + '_*'):
redis_db.delete(key)
print('training')
rasa_data = {
"rasa_nlu_data": {
"common_examples": [{
"text": "Nomnffl",
"intent": "None",
"entities": []
}, {
"text": "Nekdg",
"intent": "None",
"entities": []
}, {
"text": "Fesf",
"intent": "None",
"entities": []
}, {
"text": "this is the this",
"intent": "None",
"entities": []
}, {
"text": "likesdike mike",
"intent": "None",
"entities": []
}],
"entity_synonyms": [],
"regex_features": []
}
}
intents = Intent.query.filter_by(bot_guid=bot_guid).filter(
~Intent.name.like('eliza.%')).all()
entities = Entity.query.filter_by(bot_guid=bot_guid)
ent_data = {}
words_json = {}
for entity in entities:
ent_data[entity.name] = []
if type(entity.examples) == str:
entity.examples = json.loads(entity.examples)
for example_key in entity.examples:
ent_data[entity.name].extend(
entity.examples[example_key])
rasa_data['rasa_nlu_data']['entity_synonyms'].append({
"value":
example_key,
"synonyms":
entity.examples[example_key]
})
for intent in intents:
for utterance in intent.utterances:
utt_copy = utterance
intent_name = intent.name
for word in stopWords:
utt_copy = utt_copy.replace(word, " ")
utt_words = utt_copy.split(" ")
for word in utt_words:
if word in words_json:
if intent_name in words_json[word]:
words_json[word][intent_name] += 1
else:
words_json[word][intent_name] = 1
else:
words_json[word] = {intent_name: 1}
terminals = []
common_example = {}
common_example['text'] = utterance.lower()
common_example['intent'] = intent.name
common_example['entities'] = []
new_examples = []
new_values = []
for entity in entities:
if type(entity.examples) == str:
entity.examples = json.loads(entity.examples)
for example_key in entity.examples:
utterance = utterance.lower()
ent_examples = entity.examples[example_key]
# Used to consider key as well - but key is automatically added (not verified) when retreiving so these steps are not needed
# if example_key.lower() not in ent_examples:
# if example_key in ent_examples:
# ent_examples.remove(example_key)
# ent_examples.append(example_key.lower())
for example in ent_examples:
example = example.lower()
test_utterance = ' ' + utterance + ' '
test_example = ' ' + example + ' '
if test_example in test_utterance:
#print("here")
start = utterance.find(example)
end = start + len(example)
is_valid = True
if len(terminals) > 0:
mod_terminals = [
x for x in terminals
]
for term_start, term_end, name, value in terminals:
if term_start <= start and term_end >= end:
is_valid = False
continue
if start <= term_start and end >= term_end:
mod_terminals.remove(
(term_start, term_end,
name, value))
continue
terminals = [x for x in terminals]
if is_valid:
terminals.append(
(start, end, entity.name,
example))
for terminal in terminals:
common_example['entities'].append({
"start":
terminal[0],
"end":
terminal[1],
"entity":
terminal[2],
"value":
terminal[3]
})
remaining_examples = ent_data[terminal[2]]
if terminal[3] in remaining_examples:
remaining_examples.remove(terminal[3])
values = generate(utterance, terminal[3],
remaining_examples, intent.name,
entities)
new_values += values
rasa_data['rasa_nlu_data']['common_examples'].append(
common_example)
# rasa_data['rasa_nlu_data']['common_examples'] += new_values
# print(type(new_values))
# print(len(new_values))
# child_count = 0
# for val in new_values:
# if type(val) == dict:
# child_count += 1
# print(child_count)
#print(rasa_data['rasa_nlu_data']['common_examples'])
job = q.enqueue(train_bot,
bot,
rasa_data,
words_json,
timeout=600)
job_id = job.get_id()
print(job_id)
app.logger.info('GET /api/train/' + bot_guid +
' bot training started')
return jsonify({"success": True, "job_id": job_id})
#train_bot(bot,rasa_data, words_json)
else:
app.logger.warning('GET /api/train/' + bot_guid +
' not authorized')
return jsonify({"error": "Not Authorized"}), 401
else:
app.logger.warning('GET /api/train/' + bot_guid +
' bot does not exist')
return jsonify({"error": "Bot doesn't exist"}), 404
elif code == 400:
app.logger.warning('GET /api/train/' + bot_guid +
' invalid authorization token')
return jsonify({"error": "Invalid Authorization Token"}), 400
elif code == 401:
app.logger.warning('GET /api/train/' + bot_guid +
' no authorization token sent')
return jsonify({"error": "No Authorization Token Sent"}), 401
def back_ec2_create_ec2(param):
client = boto3.client('acm')
response = client.request_certificate(
DomainName=param["hostname"] +
".some-cloud.net", # arn:aws:acm:ap-northeast-2:453409655393:certificate/bc65ddcb-9963-4d29-bd9f-f98f9569fcdd
ValidationMethod='DNS',
Options={'CertificateTransparencyLoggingPreference': 'DISABLED'},
)
cert_arn = response["CertificateArn"]
secret_key = secrets.token_hex()
print("secKey: {}".format(secret_key))
amz_docker_install = """
#!/bin/bash
mkdir -p /home/ec2-user/.manager
mkdir -p /home/ec2-user/public_html
chown -R ec2-user:ec2-user /home/ec2-user/.manager
sudo yum install docker git python3.7 -y
git clone https://github.com/kookmin-sw/capstone-2021-4 -b backend /home/ec2-user/.manager
cd capstone-2021-4/backend/receiver/
chmod +x run.sh
cd ~/
sudo usermod -a -G docker ec2-user
sudo systemctl start docker && sudo systemctl enable docker
#set secret key
echo 'export secret={}' >> /home/ec2-user/.bashrc
# permission
chown -R ec2-user:ec2-user /home/ec2-user/.manager
#start up like rc.local
""".format(secret_key)
ubuntu_docker_install = """
#!/bin/bash \n
wget https://get.docker.com\n
chmod 777 index.html
./index.html\n
sudo usermod -a -G docker ubuntu \n
sudo systemctl enable docker \n
"""
flask_install = """
#!/bin/bash
sudo service supervisord stop
cd /home/ec2-user/.manager/capstone-2021-4/backend
git reset --hard
git clean -d -f -f
git pull
cp -rf /home/ec2-user/.manager/capstone-2021-4/backend/receiver/flask.ini /etc/supervisord.d/
echo 'environment=secret={}' >> /etc/supervisord.d/flask.ini
sudo service supervisord start
""".format(secret_key)
use_userdata = ""
if param["os_name"] == "ubuntu20.04":
use_userdata = ubuntu_docker_install
elif param["os_name"] == "amazonLinux":
use_userdata = amz_docker_install
# load balancer create
elif param["os_name"] == "Amazon-Linux-flask":
print("secret install : {} ".format(flask_install))
use_userdata = flask_install
instance = ec2.run_instances(
BlockDeviceMappings=
[ # 이게 기본 부트 볼륨으로 지정이 안됨.. /dev/sda1 같은거로 바꾸고, VolumeType, IOPS 세팅 피룡함
{
"DeviceName": "/dev/sda1",
"Ebs": {
"DeleteOnTermination": True,
"VolumeSize": param["ssd"],
"VolumeType": "gp3",
"Iops": param["iops"],
"Throughput": 125,
"DeleteOnTermination": True,
}
}
],
ImageId=param["os"],
InstanceType=param["plan"],
KeyName=param["keypair"],
MaxCount=1,
MinCount=1,
Monitoring={'Enabled': True},
UserData=use_userdata,
NetworkInterfaces=[
{
"AssociatePublicIpAddress":
True, # no need to change subnet attribute 'ipv4 auto assign'
"DeviceIndex": 0,
'SubnetId': param["subnetid"],
'Groups': param["security-group-id"],
"DeleteOnTermination": True,
},
]
# UserData='string', # StackScript if exists
)
cloud_id = param["cloudid"]
instance_id = instance["Instances"][0]["InstanceId"]
cloud = Cloud.query.filter_by(id=cloud_id).first()
cloud.aws_instance_id = instance_id
cloud.status = "Running"
cloud.app_secret_access = secret_key
cloud.certificate_arn = cert_arn
secgroup = SecurityGroup.query.filter_by(
sec_group_id=param["security-group-id"][0]).first()
secgroup.associated_to = cloud_id
db.session.commit()
q.enqueue(back_update_ec2_info, instance_id)
return instance