def video_upload():
target = os.path.join(APP_ROOT, 'files/')
clean_data(target)
print(target)
if not os.path.isdir(target):
os.mkdir(target)
print(request.files.getlist("file"))
for upload in request.files.getlist("file"):
print(upload)
print("{} is the file name".format(upload.filename))
filename = upload.filename
print(filename + "ana henaaa")
ext = os.path.splitext(filename)[1]
if (ext == ".mp4"):
print("File supported moving on...")
else:
return render_template("Error.html",
message="""The application supports only mp4
videos, this format is not supported""")
destination = "".join([target, filename])
print("Accept incoming file:", filename)
print("Save it to:", destination)
upload.save(destination)
pre_process(target, destination, filename)
generate_video(target, filename)
return render_template("complete_video.html", value=filename)
def test_multi_param_remove_dummy():
name = f'ckt_{get_test_id()}'.upper()
netlist = multi_param_ckt(name)
constraints = [{
"constraint": "PowerPorts",
"ports": ["D"]
}, {
"constraint": "GroundPorts",
"ports": ["S"]
}]
example = build_example(name, netlist, constraints)
ckt_library = compiler_input(example, name, pdk_path, config_path)
all_modules = set([name])
available_modules = set([
module.name for module in ckt_library
if isinstance(module, SubCircuit)
])
assert available_modules == all_modules, f"{available_modules}"
ckt = ckt_library.find(name)
assert ckt
assert ckt.get_element(
"MI1"), f"all instances{[ele.name for ele in ckt.elements]}"
assert ckt.get_element("MI1").parameters["NFIN"] == "16"
assert ckt.get_element("MI2")
assert ckt.get_element("MI2").parameters["NFIN"] == "24"
assert ckt.get_element("MI3")
assert ckt.get_element("MI3").parameters["NFIN"] == "24"
assert ckt.get_element("MI4")
assert ckt.get_element("MI4").parameters["NFIN"] == "64"
clean_data(name)
def test_symm_net():
name = f'ckt_{get_test_id()}'
netlist = ota_six(name)
constraints = [
{"constraint": "IsDigital", "isTrue": True}
]
example = build_example(name, netlist, constraints)
ckt_library = compiler_input(example, name, pdk_path, config_path)
ckt = ckt_library.find(name)
G = Graph(ckt)
pairs, pinsA, pinsB = symmnet_device_pairs(G, 'VIN', 'VIP', list(), None, True)
assert pairs == {'VIN': 'VIP', 'MN4': 'MN3'}
assert pinsA == ['MN4/G', 'VIN']
assert pinsB == ['MN3/G', 'VIP']
pairs, pinsA, pinsB = symmnet_device_pairs(G, 'VIN', 'VIP', [{'MN3', 'MN4'}], None)
assert pairs == {'VIN': 'VIP', 'MN4': 'MN3'}
pairs, pinsA, pinsB = symmnet_device_pairs(G, 'VIN', 'VIP', ['MN3'], None)
assert pairs is None
pairs, pinsA, pinsB = symmnet_device_pairs(G, "VIN", "VIP", ["MN4"], None)
assert pairs is None
pairs, pinsA, pinsB = symmnet_device_pairs(G, "VIN", "VIP", list(), ["MN4"])
assert pairs is None
pairs, pinsA, pinsB = symmnet_device_pairs(G, "VIN", "VIP", list(), ["MN3"])
assert pairs is None
pairs, pinsA, pinsB = symmnet_device_pairs(G, "IBIAS", "TAIL", list(), ["MN3"])
assert pairs is None
pairs, pinsA, pinsB = symmnet_device_pairs(G, "VON", "VOP", list(), ["MN3"])
assert pairs is None
pairs, pinsA, pinsB = symmnet_device_pairs(G, "VIN", "VON", list(), ["MN3"])
assert pairs is None
clean_data(name)
def test_add_symmetry_const():
name = f'ckt_{get_test_id()}'
netlist = ota_six(name)
constraints = [
{"constraint": "IsDigital", "isTrue": True}
]
example = build_example(name, netlist, constraints)
ckt_library = compiler_input(example, name, pdk_path, config_path)
ckt = ckt_library.find(name)
with set_context(ckt.constraints):
x = constraint.SymmetricBlocks(direction="V", pairs=[["MN4", "MN3"]])
const_pairs = {"MN4": "MN3"} # skip dictionary element
with pytest.raises(KeyError):
add_or_revert_const(const_pairs, ckt.constraints, list())
assert len(ckt.constraints) == 1
const_pairs = [["MN4", "MN3"]]
add_or_revert_const(const_pairs, ckt.constraints, list())
assert len(ckt.constraints) == 2
assert ckt.constraints[1] == x
const_pairs = [["MN4", "MN5"]] # Skip unequal size
add_or_revert_const(const_pairs, ckt.constraints, list())
assert len(ckt.constraints) == 2
const_pairs = [["VIN", "VIP"]] # Skip net
add_or_revert_const(const_pairs, ckt.constraints, list())
assert len(ckt.constraints) == 2
clean_data(name)
def test_array_gen_ro_f():
name = f'ckt_{get_test_id()}'
netlist = ring_oscillator_flat(name)
constraints = [{
"constraint": "DoNotUseLib",
"libraries": ["STAGE2_INV", "INV", "DP_PMOS", "DP_NMOS"]
}]
example = build_example(name, netlist, constraints)
ckt_library = compiler_input(example, name, pdk_path, config_path)
ckt = ckt_library.find(name)
assert ckt, f"No ckt {name} found in library"
array_cl = process_arrays(ckt, dict())
array1 = array_cl.find_array('VCCX', ['VSSX'])
assert array1 == [['MP0', 'MN0'], ['MP1', 'MN1'], ['MP2', 'MN2'],
['MP3', 'MN3'], ['MP4', 'MN4']]
array_cl.add_align_block_const()
array_cl.add_new_array_hier()
assert ckt.get_element("X_ARRAY_HIER_VCCX")
assert ckt_library.find("ARRAY_HIER_VCCX")
assert ckt_library.find(
"ARRAY_TEMPLATE"
), f"{set([inst.name for inst in ckt_library.find('ARRAY_TEMPLATE').elements])}"
assert set([
inst.name for inst in ckt_library.find("ARRAY_TEMPLATE").elements
]) == {'MP0', 'MN0'}
array_insts = [
'X_ARRAY_TEMPLATE', 'X_ARRAY_TEMPLATE1', 'X_ARRAY_TEMPLATE2',
'X_ARRAY_TEMPLATE3', 'X_ARRAY_TEMPLATE4'
]
assert [
inst.name for inst in ckt_library.find("ARRAY_HIER_VCCX").elements
] == array_insts
clean_data(name)
def do_load_dataset(ws):
""" Socket endpoint to receive command to load a dataset
"""
while not ws.closed:
datasetName = ws.receive()
if datasetName:
print("Load dataset: {}".format(datasetName))
X, y, labels = datasets.load_dataset(datasetName)
metadata = {
'dataset_name': datasetName,
'n_total': X.shape[0],
'original_dim': X.shape[1],
'reduced_dim': 2,
'shape_X': X.shape,
'type_X': X.dtype.name,
'shape_y': y.shape,
'type_y': y.dtype.name
}
utils.clean_data() # In dev mode: flush all data in redis
utils.set_dataset_metadata(metadata)
utils.set_ndarray(name='X_original', arr=X)
utils.set_ndarray(name='y_original', arr=y)
utils.set_to_db(key='labels', str_value=json.dumps(labels))
n_neighbors = 100 #int(0.05 * X.shape[0])
info = datasets.pre_calculate(X, k=n_neighbors)
ws.send(json.dumps(info))
def test_preprocessing_SD():
name = f'ckt_{get_test_id()}'.upper()
netlist = nested_swap_SD(name)
constraints = constraints = [{
"constraint": "PowerPorts",
"ports": ["D"]
}, {
"constraint": "GroundPorts",
"ports": ["S"]
}, {
"constraint": "KeepDummyHierarchies",
"isTrue": True
}]
example = build_example(name, netlist, constraints)
ckt_library = compiler_input(example, name, pdk_path, config_path)
all_modules = set([name, "PARAM_MOS", "P_MOS"])
available_modules = set([
module.name for module in ckt_library
if isinstance(module, SubCircuit)
])
assert available_modules == all_modules, f"{available_modules}"
assert ckt_library.find("P_MOS").get_element(
"MN1").parameters["NFIN"] == "12"
assert ckt_library.find("P_MOS").get_element("MN1").pins == {
"D": "D",
"G": "G",
"S": "S",
"B": "B",
}
clean_data(name)
def test_multi_param_skip():
name = f'ckt_{get_test_id()}'.upper()
netlist = multi_param_ckt_with_existing_name(name)
constraints = [{
"constraint": "PowerPorts",
"ports": ["D"]
}, {
"constraint": "GroundPorts",
"ports": ["S"]
}, {
"constraint": "KeepDummyHierarchies",
"isTrue": True
}]
example = build_example(name, netlist, constraints)
ckt_library = compiler_input(example, name, pdk_path, config_path)
all_modules = set([name, "PARAM_MOS", "PARAM_MOS_1", "PARAM_MOS_2"])
available_modules = set([
module.name for module in ckt_library
if isinstance(module, SubCircuit)
])
assert available_modules == all_modules, f"{available_modules}"
assert ckt_library.find("PARAM_MOS").parameters["TF"] == "16"
assert ckt_library.find("PARAM_MOS_1").parameters["TF"] == "32"
assert ckt_library.find("PARAM_MOS_2").parameters["TF"] == "24"
assert ckt_library.find("PARAM_MOS").get_element(
"MN1").parameters["NFIN"] == "16"
assert ckt_library.find("PARAM_MOS_1").get_element(
"MN2").parameters["NFIN"] == "32"
assert ckt_library.find("PARAM_MOS_2").get_element(
"MN1").parameters["NFIN"] == "24"
clean_data(name)
def image_upload():
target = os.path.join(APP_ROOT, 'static/photo/')
clean_data(target)
print(target)
if not os.path.isdir(target):
os.mkdir(target)
print(request.files.getlist("file"))
for upload in request.files.getlist("file"):
print(upload)
print("{} is the file name".format(upload.filename))
filename = upload.filename
ext = os.path.splitext(filename)[1]
if (ext == ".jpg") or (ext == ".png"):
print("File supported moving on...")
else:
return render_template(
"Error.html",
message="""The application supports only jpg and png
images, this format is not supported""")
destination = "".join([target, filename])
upload.save(destination)
full_filename = os.path.join(app.config['UPLOAD_FOLDER'], filename)
return render_template("complete_image.html",
user_image=full_filename,
filename=filename)
def build_query(data_path, w2v_path, vocab_path, k):
'''
构建查询
Args:
data_path str 查询文件路径
model_path str 词向量模型路径
vocab_path str 词典路径
k int 返回前k个相近词
Returns:
query_list list 已扩展的查询列表
'''
# 载入词向量模型，词典模型
w2v_model = load_model(w2v_path)
vocab = pickle_load(vocab_path)
query_list = []
# 解析xml文档
qurey_dict = {'disease': [], 'gene': [], 'demographic': [], 'other': []}
query_dict = xml_parse(data_path, qurey_dict, 1)
disease_field_list = query_dict['disease']
gene_field_list = query_dict['gene']
demographic_field_list = query_dict['demographic']
other_field_list = query_dict['other']
del query_dict
# 遍历查询
for i in range(len(disease_field_list)):
query_tmp_list = []
# 获取一条查询的查询词
disease_field_list[i] = preprocess(disease_field_list[i])
disease_list = disease_field_list[i].split(' ')
gene_field_list[i] = preprocess(gene_field_list[i])
gene_list = gene_field_list[i].split(' ')
other_list = preprocess(other_field_list[i])
other_list = other_field_list[i].split(' ')
demographic_list = demographic_split(demographic_field_list[i])
# 对原始查询进行词性还原与去停用词操作
disease_clean_list = clean_data(disease_list)
gene_clean_list = clean_data(gene_list)
demographic_clean_list = clean_data(demographic_list)
other_clean_list = clean_data(other_list)
# 查询扩展(含词干还原和去停用词操作)
query_tmp_list.append(
query_extension(disease_clean_list, w2v_model, vocab, k))
query_tmp_list.append(
query_extension(gene_clean_list, w2v_model, vocab, k))
query_tmp_list.append(
query_extension(other_clean_list, w2v_model, vocab, k))
tmp_dict = {}
for tmp in demographic_clean_list:
tmp_dict[tmp] = []
query_tmp_list.append(tmp_dict)
query_list.append(query_tmp_list)
return query_list
def _create_issue(self):
# create issue on the server
uri = "repos/%s/issues" % self.repo
url = github.url(uri)
data = utils.clean_data(copy.deepcopy(self.data), ["state"])
if not data or len(data.keys()) == 0:
utils.log("New issues require title/body")
return
data, status = github.request(url, "post", data)
if not status:
utils.log(data)
return
# update attributes as needed for object
self.number = str(data["number"])
self.data["user"] = data["user"]["login"]
self.url = data["html_url"]
self.issue_uri = "repos/%s/issues/%s" % (self.repo, self.number)
self.comments.number = self.number
# clean up hash
del i_hash["%s/%s" % (self.repo, "new")]
i_hash["%s/%s" % (self.repo, self.number)] = self
# delete the old buffer that we don't need any more
vim.command("silent new")
vim.command("bdelete %s" % self.buffer_name)
def test_array_gen_ro():
name = f'ckt_{get_test_id()}'
netlist = ring_oscillator(name)
constraints = []
example = build_example(name, netlist, constraints)
ckt_library = compiler_input(example, name, pdk_path, config_path)
ckt = ckt_library.find(name)
assert ckt, f"No ckt {name} found in library"
array_cl = process_arrays(ckt, dict())
array1 = array_cl.find_array('VCCX', ['VSSX'])
assert array1 == ['XI0', 'XI1', 'XI2', 'XI3', 'XI4']
array_cl.add_align_block_const()
with set_context(ckt.constraints):
x = constraint.Align(line="h_center", instances=array1)
assert ckt.constraints == [x]
clean_data(name)
def handle_data(self, data):
if self.is_text:
if self.auto_clean:
data = clean_data(data)
print(data)
self.doc.write(data)
def index():
df = clean_data()
params = {'name': 'Friday', 'task': 'Fun envent', 'df': df.head()}
return render_template('index.html', params=params)
#
# if __name__ == '__main__':
# app.run()
def test_clean_data(self):
data = {
"key1": "",
"key2": [],
"no_key": "a",
"key": "B"
}
_data = utils.clean_data(data, ["no_key"])
def test_merge_parallel():
# TODO Do not identify array when setup set as false
name = f'ckt_{get_test_id()}'.upper()
netlist = ota_six(name)
constraints = [{
"constraint": "PowerPorts",
"ports": ["VCCX"]
}, {
"constraint": "GroundPorts",
"ports": ["VSSX"]
}, {
"constraint": "MergeParallelDevices",
"isTrue": False
}]
example = build_example(name, netlist, constraints)
generate_hierarchy(example, name, out_path, False, pdk_path, False)
clean_data(name)
pass
def test_dont_constrain_clk():
# TODO Do not constrain clock connected devices
name = f'ckt_{get_test_id()}'.upper()
netlist = ota_six(name)
constraints = [{
"constraint": "PowerPorts",
"ports": ["VCCX"]
}, {
"constraint": "GroundPorts",
"ports": ["VSSX"]
}, {
"constraint": "ClockPorts",
"ports": ["vin"]
}]
example = build_example(name, netlist, constraints)
generate_hierarchy(example, name, out_path, False, pdk_path, False)
clean_data(name)
pass
def test_array_gen_ro_fh():
name = f'ckt_{get_test_id()}'
netlist = ring_oscillator_flat(name)
constraints = [{"constraint": "DoNotUseLib", "libraries": ["STAGE2_INV"]}]
example = build_example(name, netlist, constraints)
ckt_library = compiler_input(example, name, pdk_path, config_path)
ckt = ckt_library.find(name)
assert ckt, f"No ckt {name} found in library"
array_cl = process_arrays(ckt, dict())
array1 = array_cl.find_array('VCCX', ['VSSX'])
assert array1 == [
'X_INV_MN0_MP0', 'X_INV_MN1_MP1', 'X_INV_MN2_MP2', 'X_INV_MN3_MP3',
'X_INV_MN4_MP4'
]
array_cl.add_align_block_const()
with set_context(ckt.constraints):
x = constraint.Align(line="h_center", instances=array1)
assert ckt.constraints[-1] == x
clean_data(name)
def get_faculty(url):
if url:
resp = requests.get(url)
if resp.status_code == 200:
soup = BeautifulSoup(resp.content, 'html.parser')
faculty = {}
main_form = soup.find('form', attrs={'id': 'filter-form'})
for element in main_form:
faculty_el = element.find(
'select', attrs={'id': 'TimeTableForm_faculty'})
# Перебираем key/value для faculty
getKeyValueOption(faculty, faculty_el)
# Очистка данных
clean_data(faculty)
return faculty
def get_group(url):
if url:
resp = requests.get(url)
if resp.status_code == 200:
soup = BeautifulSoup(resp.content, 'html.parser')
group = {}
main_form = soup.find('form', attrs={'id': 'filter-form'})
for element in main_form:
group_el = element.find('select',
attrs={'id': 'TimeTableForm_group'})
# Перебираем key/value для group
getKeyValueOption(group, group_el)
# Очистка данных
clean_data(group)
return group
def test_ota_six():
name = f'ckt_{get_test_id()}'.upper()
netlist = ota_six(name)
constraints = [{
"constraint": "PowerPorts",
"ports": ["VCCX"]
}, {
"constraint": "GroundPorts",
"ports": ["VSSX"]
}]
example = build_example(name, netlist, constraints)
ckt_library = compiler_input(example, name, pdk_path, config_path)
all_modules = set([name, "SCM_NMOS", "SCM_PMOS", "DP_NMOS_B"])
available_modules = set([
module.name for module in ckt_library
if isinstance(module, SubCircuit)
])
assert available_modules == all_modules, f"{available_modules}"
clean_data(name)
def do_reset():
print("[Reset]Receive Reset command from client. Do reset!")
# set a flag to denote it's time to break all running thread
utils.update_server_status({'stop': True})
# let the tsnex thread to jump out of waiting status
utils.continue_server()
# stop all threads
print("[Reset]Stopping the running threads ... ")
if (shared_states['thread_tsnex']):
shared_states['thread_tsnex'].join(timeout=0.5)
if (shared_states['thread_pubsub']):
shared_states['thread_pubsub'].join(timeout=0.5)
time.sleep(1)
print("[Reset]Threads stopped")
utils.clean_data()
print("[Reset]Done!")
def build_subwords_vocab(target_vocab_size=10000):
train_neg = import_data('./train/neg')
train_pos = import_data('./train/pos')
train_raw = train_neg + train_pos
train_clean = [clean_data(t) for t in train_raw]
vocab_encoder = tfds.features.text.SubwordTextEncoder.build_from_corpus(train_clean, target_vocab_size)
vocab_encoder.save_to_file('vocab')
print(vocab_encoder.vocab_size)
def test_array_vga_equal():
name = f'ckt_{get_test_id()}'
netlist = variable_gain_amplifier_equal(name)
constraints = list()
example = build_example(name, netlist, constraints)
ckt_library = compiler_input(example, name, pdk_path, config_path)
ckt = ckt_library.find(name)
assert ckt, f"No ckt {name} found in library"
FindConst(ckt)
all_arrays = [
module.name for module in ckt_library
if isinstance(module, SubCircuit) and 'ARRAY' in module.name
]
ARRAY_HIER = ckt_library.find("ARRAY_HIER_VOUT_VGA1")
assert ARRAY_HIER, f"ARRAY_HIER_VOUT_VGA1 not found in {all_arrays}"
TEMPLATE = ckt_library.find("ARRAY_TEMPLATE")
assert TEMPLATE, f"TEMPLATE not found in {all_arrays}"
insts = [inst.name for inst in TEMPLATE.elements]
assert set(insts) == {'X_DP_NMOS_B_M00_M01', 'MSW0'}
clean_data(name)
async def fetch_messages(self, db: AsyncIOMotorClient) -> None:
"""
Iterate by messages array, find specific message in db by message_id
and replace message_id in array by message document from db
Args:
db: db client instance.
"""
for idx, message in enumerate(self.messages):
_message = await db.messages.find_one({"message_id": message})
self.messages[idx] = clean_data(_message)
def test_top_param():
name = f'ckt_{get_test_id()}'.upper()
netlist = mos_ckt(name)
constraints = [{
"constraint": "PowerPorts",
"ports": ["D"]
}, {
"constraint": "GroundPorts",
"ports": ["S"]
}]
example = build_example(name, netlist, constraints)
ckt_library = compiler_input(example, name, pdk_path, config_path)
all_modules = set([name])
available_modules = set([
module.name for module in ckt_library
if isinstance(module, SubCircuit)
])
assert available_modules == all_modules, f"{available_modules}"
assert ckt_library.find(name).get_element("MN1")
assert ckt_library.find(name).get_element("MN1").parameters["NFIN"] == "12"
clean_data(name)
def _get_issues(self, **kwargs):
self.issues = [] #(number, title, @username, url)
uri = "repos/%s/issues" % self.repo
# prepare kwargs for getting the correct issues
kwargs = utils.clean_data(copy.deepcopy(kwargs), ("repo"), (("label", "labels"),))
url = github.url(uri, kwargs)
data, status = github.request(url, "get")
# this generates the visible list for issues that we will be handling
for ih in data: # ih = issue_hash
issue = (ih["number"], ih["title"], ih["user"]["login"], ih["state"], ih["url"], [l.get("name") for l in ih["labels"]])
self.issues.append(issue)
def get_data(ner_path, tsv_path):
if NEW_DATA:
raw_data = read_data(ner_path, verbose=VERBOSE)
else:
raw_data = pd.read_csv(tsv_path, sep="\t")
marked_data = sentence_marker(raw_data, verbose=VERBOSE)
data = clean_data(marked_data, verbose=VERBOSE)
return data
def test_dont_const():
name = f'ckt_{get_test_id()}'.upper()
netlist = ota_six(name)
constraints = [{
"constraint": "PowerPorts",
"ports": ["VCCX"]
}, {
"constraint": "GroundPorts",
"ports": ["VSSX"]
}, {
"constraint": "AutoConstraint",
"isTrue": False
}]
example = build_example(name, netlist, constraints)
generate_hierarchy(example, name, out_path, False, pdk_path, False)
gen_const_path = out_path / f'{name}.verilog.json'
with open(gen_const_path, "r") as fp:
gen_const = next(x for x in json.load(fp)['modules']
if x['name'] == name)["constraints"]
assert len(gen_const) == 3, f"{gen_const}"
clean_data(name)
def predict_raw(self, thestring):
"""
Return raw vector from model prediction
"""
t0 = time.time()
statement_arr = np.array([" ".join(clean_data(thestring))])
test_sequences = self.tokenizer.texts_to_sequences(statement_arr)
test_sequences_matrix = sequence.pad_sequences(test_sequences,maxlen=self.max_len)
predvec = self.model.predict(test_sequences_matrix)[0]
t1 = time.time()
print("Got prediction for {} in {:.2}secs".format(statement_arr,t1-t0))
return predvec
async def fetch_contacts(self, db: AsyncIOMotorClient) -> None:
"""
Iterate by contacts array, find specific user in db by user_id
and replace user_id in array by user document from db
Args:
db: db client instance.
"""
for idx, contact in enumerate(self.contacts):
_contact = await db.users.find_one({"user_id": contact})
del _contact['contacts']
del _contact['chat_id']
self.contacts[idx] = clean_data(_contact)
def main():
"""
主函数
"""
# 加载数据
raw_data = pd.read_csv(os.path.join(config.dataset_path, 'german_credit_data.csv'))
# 清洗数据
cln_data = utils.clean_data(raw_data)
# 分割数据集
train_data, test_data = train_test_split(cln_data, test_size=1 / 4, random_state=10)
# 数据查看
utils.inspect_dataset(train_data, test_data)
# 特征工程
print('\n===================== 特征工程 =====================')
X_train, y_train = utils.transform_data(train_data)
X_test, y_test = utils.transform_data(test_data)
# 构建训练测试数据
# 数据建模及验证
print('\n===================== 数据建模及验证 =====================')
model_name_param_dict = {'kNN': [5, 11, 15],
'LR': [0.01, 1, 100]}
# 比较结果的DataFrame
results_df = pd.DataFrame(columns=['Accuracy (%)', 'Time (s)'],
index=list(model_name_param_dict.keys()))
results_df.index.name = 'Model'
for model_name, param_range in model_name_param_dict.items():
_, best_acc, mean_duration = utils.train_test_model(X_train, y_train, X_test, y_test,
param_range, model_name)
results_df.loc[model_name, 'Accuracy (%)'] = best_acc * 100
results_df.loc[model_name, 'Time (s)'] = mean_duration
results_df.to_csv(os.path.join(config.output_path, 'model_comparison.csv'))
# 模型及结果比较
print('\n===================== 模型及结果比较 =====================')
plt.figure(figsize=(10, 4))
ax1 = plt.subplot(1, 2, 1)
results_df.plot(y=['Accuracy (%)'], kind='bar', ylim=[60, 100], ax=ax1, title='Accuracy(%)', legend=False)
ax2 = plt.subplot(1, 2, 2)
results_df.plot(y=['Time (s)'], kind='bar', ax=ax2, title='Time(s)', legend=False)
plt.tight_layout()
plt.savefig(os.path.join(config.output_path, 'pred_results.png'))
plt.show()
def test_ota_dont_swap():
# check drain gate swap
name = f'ckt_{get_test_id()}'.upper()
netlist = ota_six_flip(name)
constraints = [{
"constraint": "PowerPorts",
"ports": ["VCCX"]
}, {
"constraint": "GroundPorts",
"ports": ["VSSX"]
}, {
"constraint": "FixSourceDrain",
"isTrue": False
}]
example = build_example(name, netlist, constraints)
ckt_library = compiler_input(example, name, pdk_path, config_path)
all_modules = set([name, 'SCM_NMOS', 'SCM_PMOS'])
available_modules = set([
module.name for module in ckt_library
if isinstance(module, SubCircuit)
])
assert available_modules == all_modules, f"{available_modules}"
clean_data(name)
def save_city(city_data):
"""
This function parsed the line and save a city in database.
"""
# split the string on new line symbols
city_data = city_data.split('\n')
temp_city = None
count_new_obj = 0
for line in city_data:
parsed_data = clean_data(line)
if parsed_data:
try:
temp_city = City()
temp_city.code = parsed_data[0]
temp_city.name = parsed_data[1]
temp_city.save()
count_new_obj+= 1
except IntegrityError:
pass
print '{0} new cities added'.format(count_new_obj)
def save_hotel(data):
"""
Parse data and save a hotel instance if not exist.
"""
# split the string on new line symbols
data = data.split('\n')
temp_hotel = None
count_new_obj = 0
for line in data:
parsed_data = clean_data(line)
if parsed_data:
try:
city = City.objects.get(code=parsed_data[0])
temp_hotel = Hotel()
temp_hotel.city = city
temp_hotel.code = parsed_data[1]
temp_hotel.name = parsed_data[2]
temp_hotel.save()
count_new_obj+=1
except IntegrityError, Hotel.DoesNotExist:
pass
layers=[
Layer("Tanh", units=12),
Layer("Softmax")],
learning_rate=0.005,
n_iter=25)
# gs = GridSearchCV(nn, param_grid={
# 'learning_rate': [0.05, 0.01, 0.005, 0.001],
# 'hidden0__units': [4, 8, 12,100],
# 'hidden0__type': ["Rectifier", "Sigmoid", "Tanh"]})
# gs.fit(X_train, y_train)
# print(gs.best_estimator_)
nn.fit(X_train, y_train)
predicted = nn.predict(X_test).flatten()
labels = y_test
return predicted, labels
if __name__ == "__main__":
features = ['zcr', 'rms', 'sc', 'sr', 'sf','mfcc']
X, y = read_features(features)
X = clean_data(X)
# train_score, test_score = train_nolearn_model(X, y)
# print("train score: ", train_score)
# print("test score: ", test_score)
predicted, true_value = train_sknn(X,y)
print("predicted: ", predicted)
print("true_value: ", true_value)
print("accuracy: ", np.sum(predicted == true_value) / float(len(predicted)))
#the test accuracy is only 38%, need more tunning!!!
{'filetype': {'$regex': 'PE32.*'}}]}, timeout=False)):
try:
logger.info('[%s] Processing sample %s' % (sampleno,
sample['sha256']))
sample_key = {'_id': sample['_id']}
job_key = {'md5': sample['md5']}
# download sample file
logger.debug('[%s] Downloading data' % sampleno)
pe = pefile.PE(data=get_file(db, sha256=sample['sha256']))
# Do analysis
logger.debug('[%s] Analysing PE headers' % sampleno)
peheader = clean_data(pe.dump_dict())
logger.debug('[%s] Analysing PE signatures' % sampleno)
peid = signatures.match_all(pe, ep_only=True)
# Store results
logger.debug('[%s] Storing PEDump results into MongoDB' % sampleno)
db.fs.files.update(sample_key, {'$set': {'pe': peheader}},
upsert=True)
logger.debug('[%s] Storing PEiD results into MongoDB' % sampleno)
db.fs.files.update(sample_key, {'$set': {'peid': peid}},
upsert=True)
{'filetype': {'$regex': 'PDF.*'}}]}, timeout=False)):
try:
logger.info('[%s] Processing sample %s' % (sampleno,
sample['sha256']))
sample_key = {'_id': sample['_id']}
job_key = {'md5': sample['md5']}
# download sample file
logger.debug('[%s] Downloading data' % sampleno)
data = get_file(db, sha256=sample['sha256'])
# Do analysis
logger.debug('[%s] Analysing PDF' % sampleno)
pdfid = clean_data(get_pdfid(data))
# Store results
if pdfid:
logger.debug('[%s] Storing results into MongoDB'
% sampleno)
db.fs.files.update(sample_key,
{'$set': {'pdfid': pdfid}},
upsert=True)
logger.info('[%s] Metadata updated' % sampleno)
except Exception, e:
logger.exception(e)
pass
logger.info('Sleeping %s minutes' % SLEEPTIME)
def _save_issue(self):
# get ready for the patch operation
url = github.url(self.issue_uri)
data = utils.clean_data(copy.deepcopy(self.data), ["number", "user", "labels"])
data, status = github.request(url, "patch", data)
sample_key = {'sample_id': sample['_id']}
job_key = {'md5': sample['md5']}
parameters = {'resource': sample['sha256'],
'apikey': api_key}
logger.debug('[%s] Analysing' % sampleno)
r = requests.post(url, data=parameters, proxies=proxy)
VTjson = None
logger.debug('[%s] Response headers: %s' % (sampleno,
r.headers))
logger.debug('[%s] Response content: %s' % (sampleno,
r.content))
try:
VTjson = clean_data(r.json())
except Exception:
try:
VTjson = clean_data(json.loads(r.text))
except Exception:
logger.debug('[%s] Unknown response: %s'
% (sampleno, r.content))
if VTjson:
if VTjson['response_code'] == 1:
logger.debug('[%s] Storing results into MongoDB'
% sampleno)
db.fs.files.update(sample_key,
{'$set': {'virustotal': VTjson}},
upsert=True)
logger.info('[%s] Metadata updated' % sampleno)
with exiftool.ExifTool() as et:
logger.debug('[%s] Downloading data' % sampleno)
filename = os.path.join('/', 'tmp', sample['sha256'])
get_file(db, filename=filename, sha256=sample['sha256'])
logger.debug('[%s] Analysing' % sampleno)
metadata = et.get_metadata(filename)
logger.debug('[%s] Deleting temporary file' % sampleno)
os.remove(filename)
logger.debug('[%s] Storing results into MongoDB'
% sampleno)
for exifkey in uselessexifkey:
del metadata[exifkey]
metadata = clean_data(metadata)
db.fs.files.update(sample_key,
{'$set': {'exif': metadata}},
upsert=True)
logger.info('[%s] Metadata updated' % sampleno)
except Exception, e:
logger.exception(e)
pass
logger.info('Sleeping %s minutes' % SLEEPTIME)
time.sleep(SLEEPTIME * 60) # Sleep 5 minutes between runs
