def change_signal_config(host, port, login, password, nodeid, portid, signalid,
wavelength, mode):
connection = d.connect(host, port, login,
password) # connection to NETCONF server
namespace = '''<sdm_node xmlns="urn:cttc:params:xml:ns:yang:sdm_node">'''
signal_config = '''
<?xml version="1.0" encoding="UTF-8"?>
<config xmlns="urn:ietf:params:xml:ns:netconf:base:1.0">
<sdm_node xmlns="urn:cttc:params:xml:ns:yang:sdm_node">
<node-id>''' + nodeid + '''</node-id>
<port>
<port-id>''' + portid + '''</port-id>
<signal>
<signal-id>''' + signalid + '''</signal-id>
<wavelength>''' + wavelength + '''</wavelength>
<mode>''' + mode + '''</mode>
</signal>
</port>
</sdm_node>
</config>
'''
try:
d.edit_config(connection, signal_config, session_running,
operation_replace) # edit configuration
print("node configuration edited\nnew configuration:")
d.get_config(connection, namespace,
session_running) # get node configuration
except Exception as e:
print(e)
finally:
connection.close_session()
def change_signal_config(host, port, login, password, nodeid, location,
componentid, param1, param2, param3, param4, wdmid,
portid, signalid, wavelength, mode, core):
connection = d.connect(host, port, login,
password) # connection to NETCONF server
namespace = '''<bluespace-node xmlns="urn:cttc:params:xml:ns:yang:bluespace_node">'''
component_config = '''
<?xml version="1.0" encoding="UTF-8"?>
<config xmlns="urn:ietf:params:xml:ns:netconf:base:1.0">
<bluespace-node xmlns="urn:cttc:params:xml:ns:yang:bluespace_node">
<bluespace-node-id>''' + nodeid + '''</bluespace-node-id>
<location>''' + location + '''</location>
<components>
<component-id>''' + componentid + '''</component-id>
<analog-rof>
<param1>''' + param1 + '''</param1>
</analog-rof>
<digital-rof>
<param2>''' + param2 + '''</param2>
</digital-rof>
<optical-beam-forming>
<param3>''' + param3 + '''</param3>
</optical-beam-forming>
<ethernet>
<param4>''' + param4 + '''</param4>
</ethernet>
<sdm-wdm>
<wdm-id>''' + wdmid + '''</wdm-id>
<port>
<port-id>''' + portid + '''</port-id>
<signal>
<signal-id>''' + signalid + '''</signal-id>
<wavelength>''' + wavelength + '''</wavelength>
<mode>''' + mode + '''</mode>
<core>''' + core + '''</core>
</signal>
</port>
</sdm-wdm>
</components>
</bluespace-node>
</config>
'''
try:
d.edit_config(connection, component_config, session_running,
operation_replace) # edit configuration
print("node configuration edited\nnew configuration:")
d.get_config(connection, namespace,
session_running) # get node configuration
except Exception as e:
print(e)
finally:
connection.close_session()
def get_api():
config = data.get_config()
try:
api = linode.Api(config['linode_api_key'])
return api
except:
pass
def do_rename(server):
config = data.get_config()
command = "rm %s" % str(server[0]['disk_path'])
p = subprocess.Popen(command.split(), stdout=subprocess.PIPE)
command2 = "mv %s/temp.img %s" % (
str(config['disk_directory']), str(server[0]['disk_path']))
d = subprocess.Popen(command2.split(), stdout=subprocess.PIPE)
def do_rename(server):
config = data.get_config()
command = "rm %s" % str(server[0]['disk_path'])
p = subprocess.Popen(command.split(), stdout=subprocess.PIPE)
command2 = "mv %s/temp.img %s" % (str(
config['disk_directory']), str(server[0]['disk_path']))
d = subprocess.Popen(command2.split(), stdout=subprocess.PIPE)
def get_api():
config = data.get_config()
try:
api = linode.Api(config['linode_api_key'])
return api
except:
pass
def update_chart(year_idx):
if not x_axis_selection or not y_axis_selection or year_idx is None:
return
config = get_config(
data,
x_axis_selection,
y_axis_selection,
supported_countries=SUPPORTED_COUNTRIES,
)
# one of axis is updated but slider is still not updated
year_idx = min(len(config["years"]) - 1, year_idx)
year = config["years"][year_idx]
return {
"data": [
go.Scatter(
x=[config["x_df"].loc[country, year]],
y=[config["y_df"].loc[country, year]],
mode="markers",
name="", # hide trace-39 etc
)
for country in config["countries"]
],
"layout": {
"showlegend": False,
"xaxis": get_axis(config["x_df"]),
"yaxis": get_axis(config["y_df"]),
"height": 500,
"margin": {"b": 30, "r": 0, "t": 30},
},
}
def update_year_slider(x_axis_selection, y_axis_selection):
if not x_axis_selection or not y_axis_selection:
return []
config = get_config(
data,
x_axis_selection,
y_axis_selection,
supported_countries=SUPPORTED_COUNTRIES,
)
years = config["years"]
marks = {i: year for i, year in enumerate(years)}
if len(years) > 30:
marks = {i: year if i % 5 == 0 else "" for i, year in enumerate(years)}
return [
html.Label("Year"),
# TODO: return slider with properly set properties (id, min, max, value, marks)
dcc.Slider(
id="year_slider",
min=0,
max=len(years) - 1,
value=len(years) - 1,
marks=marks,
updatemode="drag",
),
]
def append_dhcp_config(mac_address, ip, vmid):
config = data.get_config()
with open(config['dhcp_configuration'], "a") as config:
config.write("host vm%s {\n hardware ethernet %s;\n fixed-address %s;\n}" %
(str(vmid), str(mac_address), str(ip)))
config.close()
restart_dhcpd()
def create_vm(name, ram, disk_size, image, vcpu):
config = data.get_config()
create.make_config(name, "", ram, vcpu, image)
message = "Created new configuration for vm%s at %s/vm%s.xml" % (str(name), str(config['config_directory']), str(name))
create_log(message, 1)
create.make_image(name, disk_size)
message2 = "Created new disk image %s/vm%s.img of size %sGB." % (str(name), str(config['disk_directory']), str(disk_size))
create_log(message2, 1)
conn = connect()
xmlpath = "%s/vm%s.xml" % (str(config['config_directory']), str(name))
xml = ""
with open(xmlpath, "r") as file:
xml = file.read()
conn.defineXML(xml)
message3 = "Defined new domain vm%s." % str(name)
create_log(message3, 1)
newdom = conn.lookupByName("vm%s" % str(name))
newdom.create()
message4 = "Sent startup to vm%s." % str(name)
create_log(message4, 1)
def setup_training(model, batcher, emoji_vocab):
"""Does setup before starting training (run_training)"""
train_dir = os.path.join(FLAGS.log_root, "train")
if not os.path.exists(train_dir): os.makedirs(train_dir)
epoch_ckpt_dir = os.path.join(FLAGS.log_root, "epoch_ckpt")
if not os.path.exists(epoch_ckpt_dir): os.makedirs(epoch_ckpt_dir)
model.build_graph() # build the graph
saver = tf.train.Saver(max_to_keep=3) # keep 3 checkpoints at a time
epoch_saver = tf.train.Saver(max_to_keep=99) # keep 3 checkpoints at a time
pretrain_saver = tf.train.Saver(var_list={v.name[:-2]: v for v in tf.trainable_variables() if v.name.startswith('image_encoder') and 'logits' not in v.name}) # keep 3 checkpoints at a time
sv = tf.train.Supervisor(logdir=train_dir,
is_chief=True,
saver=saver,
summary_op=None,
save_summaries_secs=60,
save_model_secs=600,
global_step=model.global_step)
summary_writer = sv.summary_writer
tf.logging.info("Preparing or waiting for session...")
sess_context_manager = sv.prepare_or_wait_for_session(config=get_config())
tf.logging.info("Created session.")
tf.logging.info("Loading pretrained parameters.")
pretrain_saver.restore(sess_context_manager, FLAGS.inception_ckpt)
try:
run_training(model, batcher, sess_context_manager, summary_writer, epoch_saver, emoji_vocab)
except KeyboardInterrupt:
tf.logging.info("Caught keyboard interrupt on worker. Stopping supervisor...")
sv.stop()
def make_services():
config = data.get_config()
data.make_service(config['dhcp_service'], 0)
data.make_service("celery", 0)
data.make_service("gunicorn", 0)
data.make_service("rabbitmq-server", 0)
data.make_service("mongod", 0)
check_services()
def make_services():
config = data.get_config()
data.make_service(config['dhcp_service'], 0)
data.make_service("celery", 0)
data.make_service("gunicorn", 0)
data.make_service("rabbitmq-server", 0)
data.make_service("mongod", 0)
check_services()
def append_dhcp_config(mac_address, ip, vmid):
config = data.get_config()
with open(config['dhcp_configuration'], "a") as config:
config.write(
"host vm%s {\n hardware ethernet %s;\n fixed-address %s;\n}" %
(str(vmid), str(mac_address), str(ip)))
config.close()
restart_dhcpd()
def do_resize(server, direction):
config = data.get_config()
if direction == 1:
# we are resizing up
command = "virt-resize --expand /dev/sda2 %s %s/temp.img" % (str(server[0]['disk_path']), str(config['disk_directory']))
elif direction == 0:
# we are resizing down
command = "virt-resize --shrink /dev/sda2 %s %s/temp.img" % (str(server[0]['disk_path']), str(config['disk_directory']))
p = subprocess.call(command.split(), stdout=subprocess.PIPE)
def create_node_config(host, port, login, password, config_file, session,
operation, namespace):
connection = d.connect(host, port, login,
password) # connection to NETCONF server
try:
f = open(config_file) # open configuration file
d.edit_config(connection, f.read(), session,
operation) # create node configuration
f.close()
print("new node configuration created\nnew configuration:")
d.get_config(connection, namespace, session) # get node configuration
except Exception as e:
print(e)
finally:
connection.close_session()
def load_emb_from_idx2path(data_dir, idx2path, vector_dir, ver_vector_dir, force=True):
args = get_config()
fmodel = FaceModel(args)
emb_data = {}
for name, paths in idx2path.items():
_embs = []
good_file_names = []
good_embs = []
for path in paths:
file_name = split(path)[1]
bfile_name = splitext(file_name)[0]
emb_path = join(vector_dir, name, bfile_name + '.pkl')
if not exists(join(vector_dir, name)):
mkdir(join(vector_dir, name))
if not exists(emb_path):
if exists(join(ver_vector_dir, bfile_name[:bfile_name.rfind('_')], bfile_name + '.pkl')):
emb_path = join(ver_vector_dir, bfile_name[:bfile_name.rfind('_')], bfile_name + '.pkl')
with open(emb_path, 'rb') as f:
_emb = pickle.load(f)
else:
pdb.set_trace()
_img = cv2.imread(path)
# pdb.set_trace()
# i_input = fmodel.get_input(_img)
# if i_input is not None:
# _img = i_input
# else:
# pdb.set_trace()
# exit(0)
try:
detected, _emb = fmodel.get_feature(_img)
except:
detected = True
_emb = fmodel.get_feature(_img)
if (force == False and detected ==True) or force == True:
with open(emb_path, 'wb') as f:
pickle.dump(_emb, f)
else:
with open(emb_path, 'rb') as f:
_emb = pickle.load(f)
if force == False:
if detected == True:
good_file_names.append(file_name)
good_embs.append(_emb)
_embs.append(_emb)
if len(good_file_names) > 0:
data[name] = good_file_names
emb_data[name] = good_embs
else:
emb_data[name] = _embs
if force == False and len(good_file_names) == 0:
pdb.set_trace()
# pdb.set_trace()
return emb_data
def start_novnc(port, last):
config = data.get_config()
command = "python %s/utils/websockify --web %s 61%s localhost:%s -D --cert %s" % (str(config['novnc_directory']), str(config['novnc_directory']), str(last), str(port), str(config['pem_location']))
command = command.replace('\n', '')
# command = "/srv/noVNC/utils/websockify 127.0.0.1:60%s --vnc 127.0.0.1:%s --web /srv/noVNC/utils -D" % (str(last), str(port))
p = subprocess.Popen(command.split(), stdout=subprocess.PIPE)
output = p.communicate()[0]
message = "Opened noVNC on port 60%s" % str(last)
create_log(message, 1)
port = "61%s" % str(last)
return port
def identify(data, ide_model, vector_dir, k, output):
args = get_config()
face_model = FaceModel(args)
data_items = data.items()
top_5s = []
for name, paths in data_items:
path = paths[0]
file_name = split(path)[1]
print('file_name: ', file_name)
bfile_name = splitext(file_name)[0]
emb_path = join(vector_dir, name, bfile_name + '.pkl')
if not exists(join(vector_dir, name)):
mkdir(join(vector_dir, name))
if not exists(emb_path):
_img = cv2.imread(path)
_img = cv2.resize(_img, (112,112))
_emb = face_model.get_feature(_img)
with open(emb_path, 'wb') as f:
pickle.dump(_emb, f)
else:
with open(emb_path, 'rb') as f:
_emb = pickle.load(f)
candidates, is_sames = ide_model.identify(_emb, k)
top_5 = [0] * k
is_novelty = False
for i, (candidate, is_same) in enumerate(zip(candidates, is_sames)):
if i == 0:
if is_same == True:
top_5[i] = candidate
else:
top_5[i] = 1000
is_novelty = True
if is_novelty == False:
top_5[-1] = 1000
top_5[1:-1] = candidates[1:-1]
else:
top_5[1:] = candidates[0:-1]
top_5s.append(top_5)
with open(output, 'w') as f:
f.write('image,label\n')
for i, ((name, paths), top_5) in enumerate(zip(data_items,top_5s)):
path = paths[0]
file_name = split(path)[1]
bfile_name = splitext(file_name)[0]
f.write(bfile_name + '.png,' + ' '.join([str(e) for e in top_5]))
if i < len(top_5s) - 1:
f.write('\n')
def rebuild_dhcp_config():
config = data.get_config()
ips = data.get_ipaddress_allocated_all()
ipranges = data.get_all_iprange()
create_log("Rebuilding DHCPD configuration to unassign IP.", 1)
with open(config['dhcp_configuration'], "w") as config:
config.write("option domain-name-servers 8.8.8.8, 8.8.4.4;\n\n")
for range in ipranges:
line = "subnet %s netmask %s {\n range %s %s;\n option routers %s;\n}\n" % (str(range['subnet']), str(range['netmask']), str(range['startip']), str(range['endip']), str(range['gateway']))
config.write(line)
for ip in ips:
vm = data.get_server_id(ip['server_id'])
config.write("host vm%s {\n hardware ethernet %s;\n fixed-address %s;\n}\n" % (str(vm[0]['_id']), str(vm[0]['mac_address']), str(ip['ip'])))
config.close()
restart_dhcpd()
def update_config(vm):
config = data.get_config()
path = "%s/vm%s.xml" % (str(config['config_directory']), str(vm[0]['_id']))
try:
os.remove(path)
except:
message2 = "Tried to delete config at %s but it wasn't there." % path
create_log(message2, 2)
pass
message1 = "Deleted config for vm%s at %s/vm%s.xml" % (str(vm), str(config['config_directory']), str(vm[0]['_id']))
create_log(message1, 1)
create.make_config(vm[0]['_id'], "", str(vm[0]['ram']), str(vm[0]['vcpu']), vm[0]['disk_image'])
message = "Created new configuration for vm%s at %s/vm%s.xml" % (str(vm[0]['_id']), str(config['config_directory']), str(vm[0]['_id']))
create_log(message, 1)
def import_images():
config = data.get_config()
filesystem_images = [ f for f in os.listdir(config['image_directory']) if os.path.isfile(os.path.join(config['image_directory'],f)) ]
message = "Image sync initated."
create_log(message, 1)
for image in filesystem_images:
images = data.get_all_images()
state = 0
for existing_image in images:
if os.path.splitext(image)[0] == existing_image['name']:
state = 1
if state == 0:
image_name = os.path.splitext(image)[0]
image_path = "%s/%s" % (str(config['image_directory']), str(image))
image_size = os.path.getsize(image_path) / (1024*1024)
message2 = "Found new image at %s, adding to the DB" % str(image_path)
create_log(message2, 1)
data.make_image(image_name, image_path, image_size)
def run_test(model, batcher, emoji_vocab):
import numpy as np
model.build_graph()
fp = None
sess = tf.Session(config=data.get_config())
saver = tf.train.Saver()
ckpt_path = data.load_ckpt(saver, sess)
tf.get_default_graph().finalize()
nn = datetime.now().strftime('%m-%d-%H-%M')
all_logits = []
all_one_hot_lables = []
train_step = train_epoch = 0
counter = 0
while True:
batch = batcher.next_batch()
if batch is None:
break
results = model.run_decode_step(sess, batch)
train_step = results['global_step']
train_epoch = results['global_epoch']
if fp is None:
fp = open(os.path.join(FLAGS.log_root, "predict-ep%dstep%d-" % (train_epoch, train_step) + nn + '.txt'), 'w', encoding='utf8')
sticker_logits = results['sticker_logits']
one_hot_targets = np.eye(FLAGS.sticker_candidates)[batch.sticker_selection_label]
for i in range(FLAGS.batch_size):
logits = sticker_logits[i].tolist()
one_hot = one_hot_targets[i].tolist()
for l, o in zip(logits, one_hot):
fp.write('%.3f\t%d\n' % (l, o))
all_logits.append(l)
all_one_hot_lables.append(o)
counter += 1
fp.close()
metrics = evaluation(all_logits, all_one_hot_lables, FLAGS.sticker_candidates)
metrics_string = []
for k in list(metrics.keys()):
metrics_string.append('%s:%.3f' % (k, metrics[k]))
mf = open(os.path.join(FLAGS.log_root, "metric.txt"), 'a+', encoding='utf8')
mf.write('%s\n' % ' | '.join(metrics_string))
mf.close()
def redefine_vm(vm):
config = data.get_config()
conn = connect()
dom = conn.lookupByName("vm%s" % str(vm))
try:
dom.destroy()
except:
dom.undefine()
xmlpath = "%s/vm%s.xml" % (str(config['config_directory']), str(vm))
xml = ""
with open(xmlpath, "r") as file:
xml = file.read()
conn.defineXML(xml)
message3 = "Redefined domain vm%s." % str(vm)
create_log(message3, 1)
def import_images():
config = data.get_config()
filesystem_images = [f for f in os.listdir(config['image_directory']) if
os.path.isfile(
os.path.join(config['image_directory'], f))]
message = "Image sync initated."
create_log(message, 1)
for image in filesystem_images:
images = data.get_all_images()
state = 0
for existing_image in images:
if os.path.splitext(image)[0] == existing_image['name']:
state = 1
if state == 0:
image_name = os.path.splitext(image)[0]
image_path = "%s/%s" % (str(config['image_directory']), str(image))
image_size = os.path.getsize(image_path) / (1024 * 1024)
message2 = "Found new image at %s, adding to the DB" % str(
image_path)
create_log(message2, 1)
data.make_image(image_name, image_path, image_size)
def restart_dhcpd():
config = data.get_config()
command = "service %s restart" % str(config['dhcp_service'])
ip = subprocess.Popen(command.split(), stdout=subprocess.PIPE)
create_log("Restarted DHCPD.", 1)
def make_image(name, disk_size):
config = data.get_config()
command = "qemu-img create %s/vm%s.img %sG" % (str(
config['disk_directory']), str(name), str(disk_size))
subprocess.Popen(command.split())
def connect():
config = data.get_config()
conn=libvirt.open("%s:///system" % config['system_type'])
message = "Connection established with server."
create_log(message, 1)
return conn
# Created by Mahdi Parastesh, February and March 2021
# Github: https://github.com/fulcrum1378
# All rights reserved.
import simple_http_server.server as server
import socket
import webbrowser
import data
data.get_personal()
data.get_config()
import analyze, watch
print("SQLite:", data.do_connect() is not None)
print("Mofid:", data.init_mofid(), "\n")
data.init_analyzer(analyze.Analyzer())
data.init_watcher(watch.Watcher())
my_ip = socket.gethostbyname(socket.gethostname())
webbrowser.open("http://" + str(my_ip) + ":1399/")
while True:
try:
server.scan("", r".*controller.*")
server.start(host=my_ip, port=1399)
except Exception as e:
print("ERROR STARTING SERVER:", e)
my_ip = input("PLEASE ENTER YOUR CORRECT IP ADDRESS:")
else:
break
def make_image(name, disk_size):
config = data.get_config()
command = "qemu-img create %s/vm%s.img %sG" % (
str(config['disk_directory']), str(name), str(disk_size))
subprocess.Popen(command.split())
type=float,
help='nms_threshold')
parser.add_argument('--keep_top_k', default=750, type=int, help='keep_top_k')
parser.add_argument('-s',
'--show_image',
action="store_true",
default=False,
help='show detection results')
parser.add_argument('--vis_thres',
default=0.5,
type=float,
help='visualization_threshold')
parser.add_argument('--cfg', default='mot', help='config mode')
args = parser.parse_args()
cfg = get_config(args.cfg)
def check_keys(model, pretrained_state_dict):
ckpt_keys = set(pretrained_state_dict.keys())
model_keys = set(model.state_dict().keys())
used_pretrained_keys = model_keys & ckpt_keys
unused_pretrained_keys = ckpt_keys - model_keys
missing_keys = model_keys - ckpt_keys
print('Missing keys:{}'.format(len(missing_keys)))
print('Unused checkpoint keys:{}'.format(len(unused_pretrained_keys)))
print('Used keys:{}'.format(len(used_pretrained_keys)))
assert len(
used_pretrained_keys) > 0, 'load NONE from pretrained checkpoint'
return True
def update_hist_chart(x_axis_selection, y_axis_selection, year_idx):
if not x_axis_selection or not y_axis_selection or year_idx is None:
return
config = get_config(
data,
x_axis_selection,
y_axis_selection,
supported_countries=SUPPORTED_COUNTRIES,
)
# one of axis is updated but slider is still not updated
year_idx = min(len(config["years"]) - 1, year_idx)
year = config["years"][year_idx]
return [
html.Div(
className="row",
children=[
html.Div(
dcc.Graph(
id="y_hist",
figure={
"data": [
go.Histogram(
x=data[y_axis_selection][year],
nbinsx=10,
opacity=0.5,
)
],
"layout": {
"title": y_axis_selection,
"titlefont": {
"size": 12
},
"height": 250,
"margin": {
"l": 30,
"b": 30,
"t": 30,
"r": 30
},
"yaxis": {
"showticklabels": False
},
},
},
)),
html.Div(
dcc.Graph(
id="x_hist",
figure={
"data": [
go.Histogram(
x=data[x_axis_selection][year],
nbinsx=10,
opacity=0.5,
)
],
"layout": {
"title": x_axis_selection,
"titlefont": {
"size": 12
},
"height": 250,
"margin": {
"l": 30,
"b": 30,
"t": 30,
"r": 30
},
"yaxis": {
"showticklabels": False
},
},
},
)),
],
)
]
def make_config(name, disk_path, ram, vcpu, image, bootdev="hd"):
config = data.get_config()
domain = et.Element("domain")
domain.set("type", "kvm")
namexml = et.SubElement(domain, "name")
namexml.text = "vm%s" % str(name)
cpuxml = et.SubElement(domain, "cpu")
cputopologyxml = et.SubElement(cpuxml, "topology")
cputopologyxml.set("sockets", "1")
cputopologyxml.set("cores", "4")
cputopologyxml.set("threads", "4")
uuidxml = et.SubElement(domain, "uuid")
uuidxml.text = str(uuid.uuid1())
memoryxml = et.SubElement(domain, "memory")
memoryxml.set("unit", "MB")
memoryxml.text = ram
cmemoryxml = et.SubElement(domain, "currentMemory")
cmemoryxml.set("unit", "MB")
cmemoryxml.text = ram
vcpuxml = et.SubElement(domain, "vcpu")
vcpuxml.set("placement", "static")
vcpuxml.text = vcpu
osxml = et.SubElement(domain, "os")
typexml = et.SubElement(osxml, "type")
typexml.text = "hvm"
bootdev1xml = et.SubElement(osxml, "boot")
bootdev1xml.set("dev", bootdev)
bootmenuxml = et.SubElement(osxml, "bootmenu")
bootmenuxml.set("enabled", "true")
featurexml = et.SubElement(domain, "features")
acpixml = et.SubElement(featurexml, "acpi")
apicxml = et.SubElement(featurexml, "apic")
paexml = et.SubElement(featurexml, "pae")
clockxml = et.SubElement(domain, "clock")
clockxml.set("offset", "utc")
onpoweroffxml = et.SubElement(domain, "on_poweroff")
onpoweroffxml.text = "destroy"
onrebootxml = et.SubElement(domain, "on_reboot")
onrebootxml.text = "restart"
oncrashxml = et.SubElement(domain, "on_crash")
oncrashxml.text = "restart"
devicesxml = et.SubElement(domain, "devices")
emulatorxml = et.SubElement(devicesxml, "emulator")
if config['distribution'] == "centos":
emulatorxml.text = "/usr/libexec/qemu-kvm"
else:
emulatorxml.text = "/usr/bin/kvm"
disk1xml = et.SubElement(devicesxml, "disk")
disk1xml.set("type", "file")
disk1xml.set("device", "disk")
disk1driverxml = et.SubElement(disk1xml, "driver")
disk1driverxml.set("name", "qemu")
disk1driverxml.set("type", "raw")
disk1driverxml.set("cache", "none")
disk1sourcexml = et.SubElement(disk1xml, "source")
disk1sourcexml.set("file", "%s/vm%s.img" %
(str(config['disk_directory']), str(name)))
disk1targetxml = et.SubElement(disk1xml, "target")
disk1targetxml.set("dev", "hda")
disk1addressxml = et.SubElement(disk1xml, "address")
disk1addressxml.set("type", "drive")
disk1addressxml.set("controller", "0")
disk1addressxml.set("bus", "0")
disk1addressxml.set("target", "0")
disk1addressxml.set("unit", "0")
disk2xml = et.SubElement(devicesxml, "disk")
disk2xml.set("type", "file")
disk2xml.set("device", "cdrom")
disk2sourcexml = et.SubElement(disk2xml, "source")
disk2sourcexml.set("file", "%s/%s.iso" %
(str(config['image_directory']), str(image)))
disk2driverxml = et.SubElement(disk2xml, "driver")
disk2driverxml.set("name", "qemu")
disk2driverxml.set("type", "raw")
disk2targetxml = et.SubElement(disk2xml, "target")
disk2targetxml.set("dev", "hdc")
disk2targetxml.set("bus", "ide")
disk2readonlyxml = et.SubElement(disk2xml, "readyonly")
disk2addressxml = et.SubElement(disk2xml, "address")
disk2addressxml.set("type", "drive")
disk2addressxml.set("controller", "0")
disk2addressxml.set("bus", "1")
disk2addressxml.set("target", "0")
disk2addressxml.set("unit", "0")
networkxml = et.SubElement(devicesxml, "interface")
networkxml.set("type", "network")
networksourcexml = et.SubElement(networkxml, "source")
networksourcexml.set("network", "default")
graphicsxml = et.SubElement(devicesxml, "graphics")
graphicsxml.set("type", "vnc")
graphicsxml.set("port", "-1")
graphicsxml.set("passwd", str(name))
interfacexml = et.SubElement(domain, "interface")
interfacesourcexml = et.SubElement(interfacexml, "source")
interfacesourcexml.set("bridge", "virbr0")
interfacemodelxml = et.SubElement(interfacexml, "model")
interfacemodelxml.set("type", "virtio")
tree = et.ElementTree(domain)
path = "%s/vm%s.xml" % (str(config['config_directory']), str(name))
tree.write(path)
def get_token():
config = data.get_config()
return config['do_api_key']
parser.add_argument('--num_workers', default=4, type=int, help='Number of workers used in dataloading')
parser.add_argument('--ngpu', default=1, type=int, help='gpus')
parser.add_argument('--lr', '--learning-rate', default=1e-3, type=float, help='initial learning rate')
parser.add_argument('--momentum', default=0.9, type=float, help='momentum')
parser.add_argument('--resume_net', default=None, help='resume net for retraining')
parser.add_argument('--resume_epoch', default=0, type=int, help='resume iter for retraining')
parser.add_argument('-max', '--max_epoch', default=300, type=int, help='max epoch for retraining')
parser.add_argument('--weight_decay', default=5e-4, type=float, help='Weight decay for SGD')
parser.add_argument('--gamma', default=0.1, type=float, help='Gamma update for SGD')
parser.add_argument('--save_folder', default='./weights/', help='Location to save checkpoint models')
args = parser.parse_args()
if not os.path.exists(args.save_folder):
os.mkdir(args.save_folder)
cfg = get_config(args.config_name)
img_dim = 1024 # only 1024 is supported
rgb_mean = (104, 117, 123) # bgr order
num_classes = 2
num_gpu = args.ngpu
num_workers = args.num_workers
batch_size = args.batch_size
momentum = args.momentum
weight_decay = args.weight_decay
initial_lr = args.lr
gamma = args.gamma
max_epoch = args.max_epoch
training_dataset = args.training_dataset
save_folder = args.save_folder
gpu_train = cfg['gpu_train']
def get_config_cached(*args, **kwargs):
return get_config(*args, **kwargs)
from data import load_data, get_config, get_countires_mapping
from utils import options
# Loading data
data = load_data()
# Options
DATA_CHOICES = sorted(list(data.keys()))
DEFAULT_X_AXIS = "Life expectancy"
DEFAULT_Y_AXIS = "GDP per capita"
COUNTIRES_MAPPING = get_countires_mapping()
SUPPORTED_COUNTRIES = COUNTIRES_MAPPING.keys()
init_config = get_config(data,
DEFAULT_X_AXIS,
DEFAULT_Y_AXIS,
supported_countries=SUPPORTED_COUNTRIES)
YEARS = init_config["years"]
# Creating app
app = dash.Dash(__name__)
app.title = "Poor man's Gapminder"
# Add Materialize CSS for friendly styling
app.css.append_css({
"external_url":
"https://cdnjs.cloudflare.com/ajax/libs/materialize/0.100.2/css/materialize.min.css"
})
app.scripts.append_script({
"external_url":
"https://cdnjs.cloudflare.com/ajax/libs/materialize/0.100.2/js/materialize.min.js"
def make_temp_disk(size):
config = data.get_config()
command = "qemu-img create %s/temp.img %sG" % (str(config['disk_directory']), str(size))
p = subprocess.Popen(command.split(), stdout=subprocess.PIPE)
def get_token():
config = data.get_config()
return config['do_api_key']
def make_config(name, disk_path, ram, vcpu, image, bootdev="hd"):
config = data.get_config()
domain = et.Element("domain")
domain.set("type", "kvm")
namexml = et.SubElement(domain, "name")
namexml.text = "vm%s" % str(name)
cpuxml = et.SubElement(domain, "cpu")
cputopologyxml = et.SubElement(cpuxml, "topology")
cputopologyxml.set("sockets", "1")
cputopologyxml.set("cores", "4")
cputopologyxml.set("threads", "4")
uuidxml = et.SubElement(domain, "uuid")
uuidxml.text = str(uuid.uuid1())
memoryxml = et.SubElement(domain, "memory")
memoryxml.set("unit", "MB")
memoryxml.text = ram
cmemoryxml = et.SubElement(domain, "currentMemory")
cmemoryxml.set("unit", "MB")
cmemoryxml.text = ram
vcpuxml = et.SubElement(domain, "vcpu")
vcpuxml.set("placement", "static")
vcpuxml.text = vcpu
osxml = et.SubElement(domain, "os")
typexml = et.SubElement(osxml, "type")
typexml.text = "hvm"
bootdev1xml = et.SubElement(osxml, "boot")
bootdev1xml.set("dev", bootdev)
bootmenuxml = et.SubElement(osxml, "bootmenu")
bootmenuxml.set("enabled", "true")
featurexml = et.SubElement(domain, "features")
acpixml = et.SubElement(featurexml, "acpi")
apicxml = et.SubElement(featurexml, "apic")
paexml = et.SubElement(featurexml, "pae")
clockxml = et.SubElement(domain, "clock")
clockxml.set("offset", "utc")
onpoweroffxml = et.SubElement(domain, "on_poweroff")
onpoweroffxml.text = "destroy"
onrebootxml = et.SubElement(domain, "on_reboot")
onrebootxml.text = "restart"
oncrashxml = et.SubElement(domain, "on_crash")
oncrashxml.text = "restart"
devicesxml = et.SubElement(domain, "devices")
emulatorxml = et.SubElement(devicesxml, "emulator")
if config['distribution'] == "centos":
emulatorxml.text = "/usr/libexec/qemu-kvm"
else:
emulatorxml.text = "/usr/bin/kvm"
disk1xml = et.SubElement(devicesxml, "disk")
disk1xml.set("type", "file")
disk1xml.set("device", "disk")
disk1driverxml = et.SubElement(disk1xml, "driver")
disk1driverxml.set("name", "qemu")
disk1driverxml.set("type", "raw")
disk1driverxml.set("cache", "none")
disk1sourcexml = et.SubElement(disk1xml, "source")
disk1sourcexml.set(
"file", "%s/vm%s.img" % (str(config['disk_directory']), str(name)))
disk1targetxml = et.SubElement(disk1xml, "target")
disk1targetxml.set("dev", "hda")
disk1addressxml = et.SubElement(disk1xml, "address")
disk1addressxml.set("type", "drive")
disk1addressxml.set("controller", "0")
disk1addressxml.set("bus", "0")
disk1addressxml.set("target", "0")
disk1addressxml.set("unit", "0")
disk2xml = et.SubElement(devicesxml, "disk")
disk2xml.set("type", "file")
disk2xml.set("device", "cdrom")
disk2sourcexml = et.SubElement(disk2xml, "source")
disk2sourcexml.set(
"file", "%s/%s.iso" % (str(config['image_directory']), str(image)))
disk2driverxml = et.SubElement(disk2xml, "driver")
disk2driverxml.set("name", "qemu")
disk2driverxml.set("type", "raw")
disk2targetxml = et.SubElement(disk2xml, "target")
disk2targetxml.set("dev", "hdc")
disk2targetxml.set("bus", "ide")
disk2readonlyxml = et.SubElement(disk2xml, "readyonly")
disk2addressxml = et.SubElement(disk2xml, "address")
disk2addressxml.set("type", "drive")
disk2addressxml.set("controller", "0")
disk2addressxml.set("bus", "1")
disk2addressxml.set("target", "0")
disk2addressxml.set("unit", "0")
networkxml = et.SubElement(devicesxml, "interface")
networkxml.set("type", "network")
networksourcexml = et.SubElement(networkxml, "source")
networksourcexml.set("network", "default")
graphicsxml = et.SubElement(devicesxml, "graphics")
graphicsxml.set("type", "vnc")
graphicsxml.set("port", "-1")
graphicsxml.set("passwd", str(name))
interfacexml = et.SubElement(domain, "interface")
interfacesourcexml = et.SubElement(interfacexml, "source")
interfacesourcexml.set("bridge", "virbr0")
interfacemodelxml = et.SubElement(interfacexml, "model")
interfacemodelxml.set("type", "virtio")
tree = et.ElementTree(domain)
path = "%s/vm%s.xml" % (str(config['config_directory']), str(name))
tree.write(path)
