def video_upload(): target = os.path.join(APP_ROOT, 'files/') clean_data(target) print(target) if not os.path.isdir(target): os.mkdir(target) print(request.files.getlist("file")) for upload in request.files.getlist("file"): print(upload) print("{} is the file name".format(upload.filename)) filename = upload.filename print(filename + "ana henaaa") ext = os.path.splitext(filename)[1] if (ext == ".mp4"): print("File supported moving on...") else: return render_template("Error.html", message="""The application supports only mp4 videos, this format is not supported""") destination = "".join([target, filename]) print("Accept incoming file:", filename) print("Save it to:", destination) upload.save(destination) pre_process(target, destination, filename) generate_video(target, filename) return render_template("complete_video.html", value=filename)
def test_multi_param_remove_dummy(): name = f'ckt_{get_test_id()}'.upper() netlist = multi_param_ckt(name) constraints = [{ "constraint": "PowerPorts", "ports": ["D"] }, { "constraint": "GroundPorts", "ports": ["S"] }] example = build_example(name, netlist, constraints) ckt_library = compiler_input(example, name, pdk_path, config_path) all_modules = set([name]) available_modules = set([ module.name for module in ckt_library if isinstance(module, SubCircuit) ]) assert available_modules == all_modules, f"{available_modules}" ckt = ckt_library.find(name) assert ckt assert ckt.get_element( "MI1"), f"all instances{[ele.name for ele in ckt.elements]}" assert ckt.get_element("MI1").parameters["NFIN"] == "16" assert ckt.get_element("MI2") assert ckt.get_element("MI2").parameters["NFIN"] == "24" assert ckt.get_element("MI3") assert ckt.get_element("MI3").parameters["NFIN"] == "24" assert ckt.get_element("MI4") assert ckt.get_element("MI4").parameters["NFIN"] == "64" clean_data(name)
def test_symm_net(): name = f'ckt_{get_test_id()}' netlist = ota_six(name) constraints = [ {"constraint": "IsDigital", "isTrue": True} ] example = build_example(name, netlist, constraints) ckt_library = compiler_input(example, name, pdk_path, config_path) ckt = ckt_library.find(name) G = Graph(ckt) pairs, pinsA, pinsB = symmnet_device_pairs(G, 'VIN', 'VIP', list(), None, True) assert pairs == {'VIN': 'VIP', 'MN4': 'MN3'} assert pinsA == ['MN4/G', 'VIN'] assert pinsB == ['MN3/G', 'VIP'] pairs, pinsA, pinsB = symmnet_device_pairs(G, 'VIN', 'VIP', [{'MN3', 'MN4'}], None) assert pairs == {'VIN': 'VIP', 'MN4': 'MN3'} pairs, pinsA, pinsB = symmnet_device_pairs(G, 'VIN', 'VIP', ['MN3'], None) assert pairs is None pairs, pinsA, pinsB = symmnet_device_pairs(G, "VIN", "VIP", ["MN4"], None) assert pairs is None pairs, pinsA, pinsB = symmnet_device_pairs(G, "VIN", "VIP", list(), ["MN4"]) assert pairs is None pairs, pinsA, pinsB = symmnet_device_pairs(G, "VIN", "VIP", list(), ["MN3"]) assert pairs is None pairs, pinsA, pinsB = symmnet_device_pairs(G, "IBIAS", "TAIL", list(), ["MN3"]) assert pairs is None pairs, pinsA, pinsB = symmnet_device_pairs(G, "VON", "VOP", list(), ["MN3"]) assert pairs is None pairs, pinsA, pinsB = symmnet_device_pairs(G, "VIN", "VON", list(), ["MN3"]) assert pairs is None clean_data(name)
def test_add_symmetry_const(): name = f'ckt_{get_test_id()}' netlist = ota_six(name) constraints = [ {"constraint": "IsDigital", "isTrue": True} ] example = build_example(name, netlist, constraints) ckt_library = compiler_input(example, name, pdk_path, config_path) ckt = ckt_library.find(name) with set_context(ckt.constraints): x = constraint.SymmetricBlocks(direction="V", pairs=[["MN4", "MN3"]]) const_pairs = {"MN4": "MN3"} # skip dictionary element with pytest.raises(KeyError): add_or_revert_const(const_pairs, ckt.constraints, list()) assert len(ckt.constraints) == 1 const_pairs = [["MN4", "MN3"]] add_or_revert_const(const_pairs, ckt.constraints, list()) assert len(ckt.constraints) == 2 assert ckt.constraints[1] == x const_pairs = [["MN4", "MN5"]] # Skip unequal size add_or_revert_const(const_pairs, ckt.constraints, list()) assert len(ckt.constraints) == 2 const_pairs = [["VIN", "VIP"]] # Skip net add_or_revert_const(const_pairs, ckt.constraints, list()) assert len(ckt.constraints) == 2 clean_data(name)
def test_array_gen_ro_f(): name = f'ckt_{get_test_id()}' netlist = ring_oscillator_flat(name) constraints = [{ "constraint": "DoNotUseLib", "libraries": ["STAGE2_INV", "INV", "DP_PMOS", "DP_NMOS"] }] example = build_example(name, netlist, constraints) ckt_library = compiler_input(example, name, pdk_path, config_path) ckt = ckt_library.find(name) assert ckt, f"No ckt {name} found in library" array_cl = process_arrays(ckt, dict()) array1 = array_cl.find_array('VCCX', ['VSSX']) assert array1 == [['MP0', 'MN0'], ['MP1', 'MN1'], ['MP2', 'MN2'], ['MP3', 'MN3'], ['MP4', 'MN4']] array_cl.add_align_block_const() array_cl.add_new_array_hier() assert ckt.get_element("X_ARRAY_HIER_VCCX") assert ckt_library.find("ARRAY_HIER_VCCX") assert ckt_library.find( "ARRAY_TEMPLATE" ), f"{set([inst.name for inst in ckt_library.find('ARRAY_TEMPLATE').elements])}" assert set([ inst.name for inst in ckt_library.find("ARRAY_TEMPLATE").elements ]) == {'MP0', 'MN0'} array_insts = [ 'X_ARRAY_TEMPLATE', 'X_ARRAY_TEMPLATE1', 'X_ARRAY_TEMPLATE2', 'X_ARRAY_TEMPLATE3', 'X_ARRAY_TEMPLATE4' ] assert [ inst.name for inst in ckt_library.find("ARRAY_HIER_VCCX").elements ] == array_insts clean_data(name)
def do_load_dataset(ws): """ Socket endpoint to receive command to load a dataset """ while not ws.closed: datasetName = ws.receive() if datasetName: print("Load dataset: {}".format(datasetName)) X, y, labels = datasets.load_dataset(datasetName) metadata = { 'dataset_name': datasetName, 'n_total': X.shape[0], 'original_dim': X.shape[1], 'reduced_dim': 2, 'shape_X': X.shape, 'type_X': X.dtype.name, 'shape_y': y.shape, 'type_y': y.dtype.name } utils.clean_data() # In dev mode: flush all data in redis utils.set_dataset_metadata(metadata) utils.set_ndarray(name='X_original', arr=X) utils.set_ndarray(name='y_original', arr=y) utils.set_to_db(key='labels', str_value=json.dumps(labels)) n_neighbors = 100 #int(0.05 * X.shape[0]) info = datasets.pre_calculate(X, k=n_neighbors) ws.send(json.dumps(info))
def test_preprocessing_SD(): name = f'ckt_{get_test_id()}'.upper() netlist = nested_swap_SD(name) constraints = constraints = [{ "constraint": "PowerPorts", "ports": ["D"] }, { "constraint": "GroundPorts", "ports": ["S"] }, { "constraint": "KeepDummyHierarchies", "isTrue": True }] example = build_example(name, netlist, constraints) ckt_library = compiler_input(example, name, pdk_path, config_path) all_modules = set([name, "PARAM_MOS", "P_MOS"]) available_modules = set([ module.name for module in ckt_library if isinstance(module, SubCircuit) ]) assert available_modules == all_modules, f"{available_modules}" assert ckt_library.find("P_MOS").get_element( "MN1").parameters["NFIN"] == "12" assert ckt_library.find("P_MOS").get_element("MN1").pins == { "D": "D", "G": "G", "S": "S", "B": "B", } clean_data(name)
def test_multi_param_skip(): name = f'ckt_{get_test_id()}'.upper() netlist = multi_param_ckt_with_existing_name(name) constraints = [{ "constraint": "PowerPorts", "ports": ["D"] }, { "constraint": "GroundPorts", "ports": ["S"] }, { "constraint": "KeepDummyHierarchies", "isTrue": True }] example = build_example(name, netlist, constraints) ckt_library = compiler_input(example, name, pdk_path, config_path) all_modules = set([name, "PARAM_MOS", "PARAM_MOS_1", "PARAM_MOS_2"]) available_modules = set([ module.name for module in ckt_library if isinstance(module, SubCircuit) ]) assert available_modules == all_modules, f"{available_modules}" assert ckt_library.find("PARAM_MOS").parameters["TF"] == "16" assert ckt_library.find("PARAM_MOS_1").parameters["TF"] == "32" assert ckt_library.find("PARAM_MOS_2").parameters["TF"] == "24" assert ckt_library.find("PARAM_MOS").get_element( "MN1").parameters["NFIN"] == "16" assert ckt_library.find("PARAM_MOS_1").get_element( "MN2").parameters["NFIN"] == "32" assert ckt_library.find("PARAM_MOS_2").get_element( "MN1").parameters["NFIN"] == "24" clean_data(name)
def image_upload(): target = os.path.join(APP_ROOT, 'static/photo/') clean_data(target) print(target) if not os.path.isdir(target): os.mkdir(target) print(request.files.getlist("file")) for upload in request.files.getlist("file"): print(upload) print("{} is the file name".format(upload.filename)) filename = upload.filename ext = os.path.splitext(filename)[1] if (ext == ".jpg") or (ext == ".png"): print("File supported moving on...") else: return render_template( "Error.html", message="""The application supports only jpg and png images, this format is not supported""") destination = "".join([target, filename]) upload.save(destination) full_filename = os.path.join(app.config['UPLOAD_FOLDER'], filename) return render_template("complete_image.html", user_image=full_filename, filename=filename)
def build_query(data_path, w2v_path, vocab_path, k): ''' 构建查询 Args: data_path str 查询文件路径 model_path str 词向量模型路径 vocab_path str 词典路径 k int 返回前k个相近词 Returns: query_list list 已扩展的查询列表 ''' # 载入词向量模型,词典模型 w2v_model = load_model(w2v_path) vocab = pickle_load(vocab_path) query_list = [] # 解析xml文档 qurey_dict = {'disease': [], 'gene': [], 'demographic': [], 'other': []} query_dict = xml_parse(data_path, qurey_dict, 1) disease_field_list = query_dict['disease'] gene_field_list = query_dict['gene'] demographic_field_list = query_dict['demographic'] other_field_list = query_dict['other'] del query_dict # 遍历查询 for i in range(len(disease_field_list)): query_tmp_list = [] # 获取一条查询的查询词 disease_field_list[i] = preprocess(disease_field_list[i]) disease_list = disease_field_list[i].split(' ') gene_field_list[i] = preprocess(gene_field_list[i]) gene_list = gene_field_list[i].split(' ') other_list = preprocess(other_field_list[i]) other_list = other_field_list[i].split(' ') demographic_list = demographic_split(demographic_field_list[i]) # 对原始查询进行词性还原与去停用词操作 disease_clean_list = clean_data(disease_list) gene_clean_list = clean_data(gene_list) demographic_clean_list = clean_data(demographic_list) other_clean_list = clean_data(other_list) # 查询扩展(含词干还原和去停用词操作) query_tmp_list.append( query_extension(disease_clean_list, w2v_model, vocab, k)) query_tmp_list.append( query_extension(gene_clean_list, w2v_model, vocab, k)) query_tmp_list.append( query_extension(other_clean_list, w2v_model, vocab, k)) tmp_dict = {} for tmp in demographic_clean_list: tmp_dict[tmp] = [] query_tmp_list.append(tmp_dict) query_list.append(query_tmp_list) return query_list
def _create_issue(self): # create issue on the server uri = "repos/%s/issues" % self.repo url = github.url(uri) data = utils.clean_data(copy.deepcopy(self.data), ["state"]) if not data or len(data.keys()) == 0: utils.log("New issues require title/body") return data, status = github.request(url, "post", data) if not status: utils.log(data) return # update attributes as needed for object self.number = str(data["number"]) self.data["user"] = data["user"]["login"] self.url = data["html_url"] self.issue_uri = "repos/%s/issues/%s" % (self.repo, self.number) self.comments.number = self.number # clean up hash del i_hash["%s/%s" % (self.repo, "new")] i_hash["%s/%s" % (self.repo, self.number)] = self # delete the old buffer that we don't need any more vim.command("silent new") vim.command("bdelete %s" % self.buffer_name)
def test_array_gen_ro(): name = f'ckt_{get_test_id()}' netlist = ring_oscillator(name) constraints = [] example = build_example(name, netlist, constraints) ckt_library = compiler_input(example, name, pdk_path, config_path) ckt = ckt_library.find(name) assert ckt, f"No ckt {name} found in library" array_cl = process_arrays(ckt, dict()) array1 = array_cl.find_array('VCCX', ['VSSX']) assert array1 == ['XI0', 'XI1', 'XI2', 'XI3', 'XI4'] array_cl.add_align_block_const() with set_context(ckt.constraints): x = constraint.Align(line="h_center", instances=array1) assert ckt.constraints == [x] clean_data(name)
def handle_data(self, data): if self.is_text: if self.auto_clean: data = clean_data(data) print(data) self.doc.write(data)
def index(): df = clean_data() params = {'name': 'Friday', 'task': 'Fun envent', 'df': df.head()} return render_template('index.html', params=params) # # if __name__ == '__main__': # app.run()
def test_clean_data(self): data = { "key1": "", "key2": [], "no_key": "a", "key": "B" } _data = utils.clean_data(data, ["no_key"])
def test_merge_parallel(): # TODO Do not identify array when setup set as false name = f'ckt_{get_test_id()}'.upper() netlist = ota_six(name) constraints = [{ "constraint": "PowerPorts", "ports": ["VCCX"] }, { "constraint": "GroundPorts", "ports": ["VSSX"] }, { "constraint": "MergeParallelDevices", "isTrue": False }] example = build_example(name, netlist, constraints) generate_hierarchy(example, name, out_path, False, pdk_path, False) clean_data(name) pass
def test_dont_constrain_clk(): # TODO Do not constrain clock connected devices name = f'ckt_{get_test_id()}'.upper() netlist = ota_six(name) constraints = [{ "constraint": "PowerPorts", "ports": ["VCCX"] }, { "constraint": "GroundPorts", "ports": ["VSSX"] }, { "constraint": "ClockPorts", "ports": ["vin"] }] example = build_example(name, netlist, constraints) generate_hierarchy(example, name, out_path, False, pdk_path, False) clean_data(name) pass
def test_array_gen_ro_fh(): name = f'ckt_{get_test_id()}' netlist = ring_oscillator_flat(name) constraints = [{"constraint": "DoNotUseLib", "libraries": ["STAGE2_INV"]}] example = build_example(name, netlist, constraints) ckt_library = compiler_input(example, name, pdk_path, config_path) ckt = ckt_library.find(name) assert ckt, f"No ckt {name} found in library" array_cl = process_arrays(ckt, dict()) array1 = array_cl.find_array('VCCX', ['VSSX']) assert array1 == [ 'X_INV_MN0_MP0', 'X_INV_MN1_MP1', 'X_INV_MN2_MP2', 'X_INV_MN3_MP3', 'X_INV_MN4_MP4' ] array_cl.add_align_block_const() with set_context(ckt.constraints): x = constraint.Align(line="h_center", instances=array1) assert ckt.constraints[-1] == x clean_data(name)
def get_faculty(url): if url: resp = requests.get(url) if resp.status_code == 200: soup = BeautifulSoup(resp.content, 'html.parser') faculty = {} main_form = soup.find('form', attrs={'id': 'filter-form'}) for element in main_form: faculty_el = element.find( 'select', attrs={'id': 'TimeTableForm_faculty'}) # Перебираем key/value для faculty getKeyValueOption(faculty, faculty_el) # Очистка данных clean_data(faculty) return faculty
def get_group(url): if url: resp = requests.get(url) if resp.status_code == 200: soup = BeautifulSoup(resp.content, 'html.parser') group = {} main_form = soup.find('form', attrs={'id': 'filter-form'}) for element in main_form: group_el = element.find('select', attrs={'id': 'TimeTableForm_group'}) # Перебираем key/value для group getKeyValueOption(group, group_el) # Очистка данных clean_data(group) return group
def test_ota_six(): name = f'ckt_{get_test_id()}'.upper() netlist = ota_six(name) constraints = [{ "constraint": "PowerPorts", "ports": ["VCCX"] }, { "constraint": "GroundPorts", "ports": ["VSSX"] }] example = build_example(name, netlist, constraints) ckt_library = compiler_input(example, name, pdk_path, config_path) all_modules = set([name, "SCM_NMOS", "SCM_PMOS", "DP_NMOS_B"]) available_modules = set([ module.name for module in ckt_library if isinstance(module, SubCircuit) ]) assert available_modules == all_modules, f"{available_modules}" clean_data(name)
def do_reset(): print("[Reset]Receive Reset command from client. Do reset!") # set a flag to denote it's time to break all running thread utils.update_server_status({'stop': True}) # let the tsnex thread to jump out of waiting status utils.continue_server() # stop all threads print("[Reset]Stopping the running threads ... ") if (shared_states['thread_tsnex']): shared_states['thread_tsnex'].join(timeout=0.5) if (shared_states['thread_pubsub']): shared_states['thread_pubsub'].join(timeout=0.5) time.sleep(1) print("[Reset]Threads stopped") utils.clean_data() print("[Reset]Done!")
def build_subwords_vocab(target_vocab_size=10000): train_neg = import_data('./train/neg') train_pos = import_data('./train/pos') train_raw = train_neg + train_pos train_clean = [clean_data(t) for t in train_raw] vocab_encoder = tfds.features.text.SubwordTextEncoder.build_from_corpus(train_clean, target_vocab_size) vocab_encoder.save_to_file('vocab') print(vocab_encoder.vocab_size)
def test_array_vga_equal(): name = f'ckt_{get_test_id()}' netlist = variable_gain_amplifier_equal(name) constraints = list() example = build_example(name, netlist, constraints) ckt_library = compiler_input(example, name, pdk_path, config_path) ckt = ckt_library.find(name) assert ckt, f"No ckt {name} found in library" FindConst(ckt) all_arrays = [ module.name for module in ckt_library if isinstance(module, SubCircuit) and 'ARRAY' in module.name ] ARRAY_HIER = ckt_library.find("ARRAY_HIER_VOUT_VGA1") assert ARRAY_HIER, f"ARRAY_HIER_VOUT_VGA1 not found in {all_arrays}" TEMPLATE = ckt_library.find("ARRAY_TEMPLATE") assert TEMPLATE, f"TEMPLATE not found in {all_arrays}" insts = [inst.name for inst in TEMPLATE.elements] assert set(insts) == {'X_DP_NMOS_B_M00_M01', 'MSW0'} clean_data(name)
async def fetch_messages(self, db: AsyncIOMotorClient) -> None: """ Iterate by messages array, find specific message in db by message_id and replace message_id in array by message document from db Args: db: db client instance. """ for idx, message in enumerate(self.messages): _message = await db.messages.find_one({"message_id": message}) self.messages[idx] = clean_data(_message)
def test_top_param(): name = f'ckt_{get_test_id()}'.upper() netlist = mos_ckt(name) constraints = [{ "constraint": "PowerPorts", "ports": ["D"] }, { "constraint": "GroundPorts", "ports": ["S"] }] example = build_example(name, netlist, constraints) ckt_library = compiler_input(example, name, pdk_path, config_path) all_modules = set([name]) available_modules = set([ module.name for module in ckt_library if isinstance(module, SubCircuit) ]) assert available_modules == all_modules, f"{available_modules}" assert ckt_library.find(name).get_element("MN1") assert ckt_library.find(name).get_element("MN1").parameters["NFIN"] == "12" clean_data(name)
def _get_issues(self, **kwargs): self.issues = [] #(number, title, @username, url) uri = "repos/%s/issues" % self.repo # prepare kwargs for getting the correct issues kwargs = utils.clean_data(copy.deepcopy(kwargs), ("repo"), (("label", "labels"),)) url = github.url(uri, kwargs) data, status = github.request(url, "get") # this generates the visible list for issues that we will be handling for ih in data: # ih = issue_hash issue = (ih["number"], ih["title"], ih["user"]["login"], ih["state"], ih["url"], [l.get("name") for l in ih["labels"]]) self.issues.append(issue)
def get_data(ner_path, tsv_path): if NEW_DATA: raw_data = read_data(ner_path, verbose=VERBOSE) else: raw_data = pd.read_csv(tsv_path, sep="\t") marked_data = sentence_marker(raw_data, verbose=VERBOSE) data = clean_data(marked_data, verbose=VERBOSE) return data
def test_dont_const(): name = f'ckt_{get_test_id()}'.upper() netlist = ota_six(name) constraints = [{ "constraint": "PowerPorts", "ports": ["VCCX"] }, { "constraint": "GroundPorts", "ports": ["VSSX"] }, { "constraint": "AutoConstraint", "isTrue": False }] example = build_example(name, netlist, constraints) generate_hierarchy(example, name, out_path, False, pdk_path, False) gen_const_path = out_path / f'{name}.verilog.json' with open(gen_const_path, "r") as fp: gen_const = next(x for x in json.load(fp)['modules'] if x['name'] == name)["constraints"] assert len(gen_const) == 3, f"{gen_const}" clean_data(name)
def predict_raw(self, thestring): """ Return raw vector from model prediction """ t0 = time.time() statement_arr = np.array([" ".join(clean_data(thestring))]) test_sequences = self.tokenizer.texts_to_sequences(statement_arr) test_sequences_matrix = sequence.pad_sequences(test_sequences,maxlen=self.max_len) predvec = self.model.predict(test_sequences_matrix)[0] t1 = time.time() print("Got prediction for {} in {:.2}secs".format(statement_arr,t1-t0)) return predvec
async def fetch_contacts(self, db: AsyncIOMotorClient) -> None: """ Iterate by contacts array, find specific user in db by user_id and replace user_id in array by user document from db Args: db: db client instance. """ for idx, contact in enumerate(self.contacts): _contact = await db.users.find_one({"user_id": contact}) del _contact['contacts'] del _contact['chat_id'] self.contacts[idx] = clean_data(_contact)
def main(): """ 主函数 """ # 加载数据 raw_data = pd.read_csv(os.path.join(config.dataset_path, 'german_credit_data.csv')) # 清洗数据 cln_data = utils.clean_data(raw_data) # 分割数据集 train_data, test_data = train_test_split(cln_data, test_size=1 / 4, random_state=10) # 数据查看 utils.inspect_dataset(train_data, test_data) # 特征工程 print('\n===================== 特征工程 =====================') X_train, y_train = utils.transform_data(train_data) X_test, y_test = utils.transform_data(test_data) # 构建训练测试数据 # 数据建模及验证 print('\n===================== 数据建模及验证 =====================') model_name_param_dict = {'kNN': [5, 11, 15], 'LR': [0.01, 1, 100]} # 比较结果的DataFrame results_df = pd.DataFrame(columns=['Accuracy (%)', 'Time (s)'], index=list(model_name_param_dict.keys())) results_df.index.name = 'Model' for model_name, param_range in model_name_param_dict.items(): _, best_acc, mean_duration = utils.train_test_model(X_train, y_train, X_test, y_test, param_range, model_name) results_df.loc[model_name, 'Accuracy (%)'] = best_acc * 100 results_df.loc[model_name, 'Time (s)'] = mean_duration results_df.to_csv(os.path.join(config.output_path, 'model_comparison.csv')) # 模型及结果比较 print('\n===================== 模型及结果比较 =====================') plt.figure(figsize=(10, 4)) ax1 = plt.subplot(1, 2, 1) results_df.plot(y=['Accuracy (%)'], kind='bar', ylim=[60, 100], ax=ax1, title='Accuracy(%)', legend=False) ax2 = plt.subplot(1, 2, 2) results_df.plot(y=['Time (s)'], kind='bar', ax=ax2, title='Time(s)', legend=False) plt.tight_layout() plt.savefig(os.path.join(config.output_path, 'pred_results.png')) plt.show()
def test_ota_dont_swap(): # check drain gate swap name = f'ckt_{get_test_id()}'.upper() netlist = ota_six_flip(name) constraints = [{ "constraint": "PowerPorts", "ports": ["VCCX"] }, { "constraint": "GroundPorts", "ports": ["VSSX"] }, { "constraint": "FixSourceDrain", "isTrue": False }] example = build_example(name, netlist, constraints) ckt_library = compiler_input(example, name, pdk_path, config_path) all_modules = set([name, 'SCM_NMOS', 'SCM_PMOS']) available_modules = set([ module.name for module in ckt_library if isinstance(module, SubCircuit) ]) assert available_modules == all_modules, f"{available_modules}" clean_data(name)
def save_city(city_data): """ This function parsed the line and save a city in database. """ # split the string on new line symbols city_data = city_data.split('\n') temp_city = None count_new_obj = 0 for line in city_data: parsed_data = clean_data(line) if parsed_data: try: temp_city = City() temp_city.code = parsed_data[0] temp_city.name = parsed_data[1] temp_city.save() count_new_obj+= 1 except IntegrityError: pass print '{0} new cities added'.format(count_new_obj)
def save_hotel(data): """ Parse data and save a hotel instance if not exist. """ # split the string on new line symbols data = data.split('\n') temp_hotel = None count_new_obj = 0 for line in data: parsed_data = clean_data(line) if parsed_data: try: city = City.objects.get(code=parsed_data[0]) temp_hotel = Hotel() temp_hotel.city = city temp_hotel.code = parsed_data[1] temp_hotel.name = parsed_data[2] temp_hotel.save() count_new_obj+=1 except IntegrityError, Hotel.DoesNotExist: pass
layers=[ Layer("Tanh", units=12), Layer("Softmax")], learning_rate=0.005, n_iter=25) # gs = GridSearchCV(nn, param_grid={ # 'learning_rate': [0.05, 0.01, 0.005, 0.001], # 'hidden0__units': [4, 8, 12,100], # 'hidden0__type': ["Rectifier", "Sigmoid", "Tanh"]}) # gs.fit(X_train, y_train) # print(gs.best_estimator_) nn.fit(X_train, y_train) predicted = nn.predict(X_test).flatten() labels = y_test return predicted, labels if __name__ == "__main__": features = ['zcr', 'rms', 'sc', 'sr', 'sf','mfcc'] X, y = read_features(features) X = clean_data(X) # train_score, test_score = train_nolearn_model(X, y) # print("train score: ", train_score) # print("test score: ", test_score) predicted, true_value = train_sknn(X,y) print("predicted: ", predicted) print("true_value: ", true_value) print("accuracy: ", np.sum(predicted == true_value) / float(len(predicted))) #the test accuracy is only 38%, need more tunning!!!
{'filetype': {'$regex': 'PE32.*'}}]}, timeout=False)): try: logger.info('[%s] Processing sample %s' % (sampleno, sample['sha256'])) sample_key = {'_id': sample['_id']} job_key = {'md5': sample['md5']} # download sample file logger.debug('[%s] Downloading data' % sampleno) pe = pefile.PE(data=get_file(db, sha256=sample['sha256'])) # Do analysis logger.debug('[%s] Analysing PE headers' % sampleno) peheader = clean_data(pe.dump_dict()) logger.debug('[%s] Analysing PE signatures' % sampleno) peid = signatures.match_all(pe, ep_only=True) # Store results logger.debug('[%s] Storing PEDump results into MongoDB' % sampleno) db.fs.files.update(sample_key, {'$set': {'pe': peheader}}, upsert=True) logger.debug('[%s] Storing PEiD results into MongoDB' % sampleno) db.fs.files.update(sample_key, {'$set': {'peid': peid}}, upsert=True)
{'filetype': {'$regex': 'PDF.*'}}]}, timeout=False)): try: logger.info('[%s] Processing sample %s' % (sampleno, sample['sha256'])) sample_key = {'_id': sample['_id']} job_key = {'md5': sample['md5']} # download sample file logger.debug('[%s] Downloading data' % sampleno) data = get_file(db, sha256=sample['sha256']) # Do analysis logger.debug('[%s] Analysing PDF' % sampleno) pdfid = clean_data(get_pdfid(data)) # Store results if pdfid: logger.debug('[%s] Storing results into MongoDB' % sampleno) db.fs.files.update(sample_key, {'$set': {'pdfid': pdfid}}, upsert=True) logger.info('[%s] Metadata updated' % sampleno) except Exception, e: logger.exception(e) pass logger.info('Sleeping %s minutes' % SLEEPTIME)
def _save_issue(self): # get ready for the patch operation url = github.url(self.issue_uri) data = utils.clean_data(copy.deepcopy(self.data), ["number", "user", "labels"]) data, status = github.request(url, "patch", data)
sample_key = {'sample_id': sample['_id']} job_key = {'md5': sample['md5']} parameters = {'resource': sample['sha256'], 'apikey': api_key} logger.debug('[%s] Analysing' % sampleno) r = requests.post(url, data=parameters, proxies=proxy) VTjson = None logger.debug('[%s] Response headers: %s' % (sampleno, r.headers)) logger.debug('[%s] Response content: %s' % (sampleno, r.content)) try: VTjson = clean_data(r.json()) except Exception: try: VTjson = clean_data(json.loads(r.text)) except Exception: logger.debug('[%s] Unknown response: %s' % (sampleno, r.content)) if VTjson: if VTjson['response_code'] == 1: logger.debug('[%s] Storing results into MongoDB' % sampleno) db.fs.files.update(sample_key, {'$set': {'virustotal': VTjson}}, upsert=True) logger.info('[%s] Metadata updated' % sampleno)
with exiftool.ExifTool() as et: logger.debug('[%s] Downloading data' % sampleno) filename = os.path.join('/', 'tmp', sample['sha256']) get_file(db, filename=filename, sha256=sample['sha256']) logger.debug('[%s] Analysing' % sampleno) metadata = et.get_metadata(filename) logger.debug('[%s] Deleting temporary file' % sampleno) os.remove(filename) logger.debug('[%s] Storing results into MongoDB' % sampleno) for exifkey in uselessexifkey: del metadata[exifkey] metadata = clean_data(metadata) db.fs.files.update(sample_key, {'$set': {'exif': metadata}}, upsert=True) logger.info('[%s] Metadata updated' % sampleno) except Exception, e: logger.exception(e) pass logger.info('Sleeping %s minutes' % SLEEPTIME) time.sleep(SLEEPTIME * 60) # Sleep 5 minutes between runs