def run(self): program = [] while len(self.__stack): print("Stack: " + list_to_string(self.__stack)) want = self.__stack.pop(0) print("Want " + want) if want == "token_eos" or want == "epsilon": continue if want.startswith("token_keyword"): program.append(want[len("token_keyword_"):]) continue if want == "token_identifier": # FIXME: choose from applicable identifiers program.append("a") continue if want == "token_number": program.append("0") continue if want.startswith("token_"): program.append(Scanner.tokenNameToString(want)) continue applicable_rules = [r for r in rules if r.left == want] # In order to not grow the stack, give the simplest production a lot of weight... if random() > 0.7: ix = 0 else: ix = floor(random() * len(applicable_rules)) print(list_to_string(applicable_rules)) print(ix) print("Rule: " + str(applicable_rules[ix])) self.__stack = applicable_rules[ix].right + self.__stack print("Program so far: ") print(list_to_string(program, "", "", " ")) return program
def get_ticker(self, pairs): """Get ticker infomation Args: pairs: a string containing asset pairs delimited by "," Returns: A dictionary containing ticker information See https://www.kraken.com/help/api#get-ticker-info for documentation Raises: RuntimeError: An error occurs when failed to retrieve server information """ if pairs is None: raise Exception("KrakenAPIHandle.get_ticker() failed: pairs cannot be None") conn = self.new_connection() pair_string = util.list_to_string(pairs) param = urllib.parse.urlencode({'pair': pair_string}) conn.request('POST', self.api_link.ticker, param) resp = json.loads(self._read_response(conn.getresponse()).decode()) if len(resp["error"]) != 0: raise RuntimeError("KrakenAPIHandle.get_ticker(): %s" % util.list_to_string(resp["error"])) return resp["result"]
def host_inspect(target, extensive): if core.is_valid_ip(target): msg = "Ip Validation OK" logger.debug(msg) msg = "[+] Valid ip" logger.info(msg) msg = "[*] Performing hostname conversion" logger.info(msg) try: value = core.get_host_by_ip(target) util.list_to_string(value) except: msg = "[-] ERROR: Cannot resolve hostname" logger.error(msg) elif core.is_valid_hostname(target): msg = "Host Validation OK" logger.debug(msg) msg = "[+] Valid host" logger.info(msg) msg = "[*] Performing ip conversion" logger.info(msg) try: value = core.get_host_by_name(target) util.list_to_string(value) except: msg = "[-] ERROR: Cannot resolve hostname" logger.error(msg) else: msg = "[-] ERROR: You must provide a valid target. Given: "+ target showhelp() logger.error(msg) sys.exit(1) db = GEOIPFILE geo = core.ip_to_country(core.get_ip(target), db) if geo: msg = "[+] The host is situated in "+geo logger.info(msg) else: msg = "[-] Cannot geolocalize the host" logger.warning(msg) if extensive: msg = "Extensive probing" logger.debug(msg) msg = "[*] Starting extensive information gathering" logger.info(msg) whois = core.get_extensive_data(target, 0) info = core.get_extensive_data(target, 1) dns = core.get_extensive_data(target, 2)
def host_inspect(target, extensive): if core.is_valid_ip(target): msg = "Ip Validation OK" logger.debug(msg) msg = "[+] Valid ip" logger.info(msg) msg = "[*] Performing hostname conversion" logger.info(msg) try: value = core.get_host_by_ip(target) util.list_to_string(value) except: msg = "[-] ERROR: Cannot resolve hostname" logger.error(msg) elif core.is_valid_hostname(target): msg = "Host Validation OK" logger.debug(msg) msg = "[+] Valid host" logger.info(msg) msg = "[*] Performing ip conversion" logger.info(msg) try: value = core.get_host_by_name(target) util.list_to_string(value) except: msg = "[-] ERROR: Cannot resolve hostname" logger.error(msg) else: msg = "[-] ERROR: You must provide a valid target. Given: " + target showhelp() logger.error(msg) sys.exit(1) db = GEOIPFILE geo = core.ip_to_country(core.get_ip(target), db) if geo: msg = "[+] The host is situated in " + geo logger.info(msg) else: msg = "[-] Cannot geolocalize the host" logger.warning(msg) if extensive: msg = "Extensive probing" logger.debug(msg) msg = "[*] Starting extensive information gathering" logger.info(msg) whois = core.get_extensive_data(target, 0) info = core.get_extensive_data(target, 1) dns = core.get_extensive_data(target, 2)
def get_ohlc(self, pair, interval=1, since=None): """Get ticker infomation Args: pair: an asset pair string interval: the time interval of ohlc (Available values: 1 (default), 5, 15, 30, 60, 240, 1440, 10080, 21600) since: return committed OHLC data since given id Returns: A dictionary containing ticker information See https://www.kraken.com/help/api#get-ticker-info for documentation Raises: RuntimeError: An error occurs when failed to retrieve server information """ if pair is None: return conn = self.new_connection() if since is None: param = urllib.parse.urlencode({'pair': pair, 'interval': interval}) else: param = urllib.parse.urlencode({'pair': pair, 'interval': interval, 'since': since}) conn.request('POST', self.api_link.ohlc, param) resp = json.loads(self._read_response(conn.getresponse()).decode()) if len(resp["error"]) != 0: raise RuntimeError("KrakenAPIHandle.get_ohlc(): %s" % util.list_to_string(resp["error"])) return resp["result"]
def __init__(self, parameter_types, return_type): assert (isinstance(return_type, Type)) assert (isinstance(parameter_types, TypeList) or parameter_types is None) parameter_types = parameter_types or TypeList(None) name = list_to_string( parameter_types.items) + " -> " + str(return_type) super().__init__(name) self.return_type = return_type self.parameter_types = parameter_types.items
def get_orderbook(self, pairs, count=10): if pairs is None: return conn = self.new_connection() pair_string = util.list_to_string(pairs) param = urllib.parse.urlencode({'pair': pair_string, 'count': count}) conn.request('POST', self.api_link.orderbook, param) resp = json.loads(self._read_response(conn.getresponse()).decode()) if len(resp["error"]) != 0: raise RuntimeError("KrakenAPIHandle.get_orderbook(): Unable to retrieve orderbook") return resp["result"]
def main(): print("Hello! Welcome~") print("Please enter the kind of function you would like to use.") print("Currently supported functions are: \n" + list_to_string(FUNCTIONS_SUPPORTED)) sel = input("Select an option: ") # Apply logic based on selection if (sel == "add"): print("Specify two numbers, a and b") a = input("a: ") b = input("b: ") # Cast numbers to float via function, then add res = Calculator.add(float(a), float(b)) print("Result: " + str(res)) elif (sel == "sub"): print("Specify two numbers, a and b") a = input("a: ") b = input("b: ") res = Calculator.sub(float(a), float(b)) print("Result " + str(res)) elif (sel == "mul"): print("Specify two numbers, a and b") a = input("a: ") b = input("b: ") res = Calculator.mul(float(a), float(b)) print("Result " + str(res)) elif (sel == "div"): print("Specify two numbers, a and b") a = input("a: ") b = input("b: ") res = Calculator.div(float(a), float(b)) print("Result " + str(res)) elif (sel == "power"): print("Specify two numbers, a and b") a = input("a: ") b = input("b: ") res = Calculator.power(float(a), float(b)) print("Result " + str(res)) else: print("Function \"" + sel + "\" not implemented.")
def get_time(self): """Get Kraken server time Returns: A dictionary containing server time See https://www.kraken.com/help/api#get-server-time for documentation Raises: RuntimeError: An error occurs when failed to retrieve server information """ conn = self.new_connection() conn.request('GET',self.api_link.time) resp = json.loads(self._read_response(conn.getresponse()).decode()) if len(resp["error"]) != 0: raise RuntimeError("KrakenAPIHandle.get_time(): %s" % util.list_to_string(resp["error"])) return resp["result"]
def get_asset_pairs(self): """Get information of all asset pairs available on Kraken exchange Returns: A dictionary containing all asset pair information See https://www.kraken.com/help/api#get-asset-info for documentation Raises: RuntimeError: An error occurs when failed to retrieve server information """ conn = self.new_connection() conn.request('GET', self.api_link.assetpair) resp = json.loads(self._read_response(conn.getresponse()).decode()) if len(resp["error"]) != 0: raise RuntimeError("KrakenAPIHandle.get_asset_pairs(): %s" % util.list_to_string(resp["error"])) return resp["result"]
def make_next_move(self): """ Get the next square that the AI wants to expose on the board :return: The coordinates of a square that the AI wants to expose """ print "finding move. Queue = " pprint(self.uncover_queue) print "Groups = " + util.list_to_string(self.groups) # If first click: if self.first_click: self.first_click = False x = self.minefield_width / 2 y = self.minefield_height / 2 else: if not self.uncover_queue: print "No moves can be deduced! Guessing not yet implemented" # take an educated guess click = self.uncover_queue.pop(0) x = click[0] y = click[1] # Is a little long to make anonymous def callback(degree): square = self.squares[x][y] self.groups_to_check += square.uncover(degree, False) mines = square.degree squares = [] for neighbor in square.neighbors: if neighbor.covered: squares += [neighbor] elif neighbor.mine: mines -= 1 group = Group(square.coordinates, squares, mines, self.dissolve_group) self.groups += [group] self.groups_to_check += [group] while self.groups_to_check: self.groups_to_check += self.groups_to_check.pop(0).check() self.query(x, y, callback)
def get_xml_data(file_path, filename): root = ET.parse(file_path + filename) # 文书情况 condition = "正常" # 事实 facts = "" # 证据 evidence = "" # 是否被证实 is_identified = "-1" # 证据关键词 evidence_keyword = "" # 证据分词 evidence_divide = "" # 事实分词列表 divide_fact_list = [] # 关键词列表 key_words_list = [] # 返回数组 return_list = [] # 证据段 zjds = root.getiterator('ZJD') for zjd in zjds: if zjd.attrib.has_key('value'): # 整合原告+被告的证据 evidence += zjd.attrib['value'] # print "证据段:", evidence if len(zjds) == 0: condition = "无证据" print filename, "无证据" return evidence = delete_space(evidence) evidence_divide = get_divide_result(evidence) evidence_keyword = get_evidence_keyword(evidence) # 原告诉称段 ygscds = root.getiterator('YGSCD') for ygscd in ygscds: if ygscd.attrib.has_key('value'): all_fact = ygscd.attrib['value'].strip() # print "原告诉称段:", all_fact # fact_all = ygscd.attrib['value'] # facts = fact_all.split('。') if len(ygscds) == 0: condition = "无原告诉称" print filename, "无原告诉称" return # 查明事实段 cmssds = root.getiterator('CMSSD') for cmssd in cmssds: if cmssd.attrib.has_key('value'): fact_identify = cmssd.attrib['value'] # print "查明事实段:", fact_identify if len(cmssds) == 0: condition = "无查明事实" print filename, "无查明事实" return fact_identify = delete_space(fact_identify) # 删除空格 all_fact = delete_space(all_fact) # 根据句号划分 facts = all_fact.split('。') # 删除诉求部分 facts = delete_appeal(facts) for fact in facts: # 删去事实中的脏数据。中间有空白项,就跳过 if (fact == "") or (fact == "……") or (len(fact) < 10): continue # 获得分词结果 fact_divide = list_to_string(get_divide_list(fact)) #获得关键词 fact_keyword = list_to_string(get_key_word_list(fact)) # 判断是否被认定 match_rate = get_match_rate(fact, fact_identify) identify = judge_relevance(match_rate) if identify: is_identified = 1 else: is_identified = 0 # [fact, fact_divide, fact_keyword, evidence, evidence_divide, evidence_keyword, is_identified] a_set_of_data = [ fact, fact_divide, fact_keyword, evidence, evidence_divide, evidence_keyword, is_identified ] return_list.append(a_set_of_data) return return_list
# FIXME: choose from applicable identifiers program.append("a") continue if want == "token_number": program.append("0") continue if want.startswith("token_"): program.append(Scanner.tokenNameToString(want)) continue applicable_rules = [r for r in rules if r.left == want] # In order to not grow the stack, give the simplest production a lot of weight... if random() > 0.7: ix = 0 else: ix = floor(random() * len(applicable_rules)) print(list_to_string(applicable_rules)) print(ix) print("Rule: " + str(applicable_rules[ix])) self.__stack = applicable_rules[ix].right + self.__stack print("Program so far: ") print(list_to_string(program, "", "", " ")) return program if __name__ == "__main__": f = Fuzzer(rules) program = f.run() print("Program:") print(list_to_string(program, "", "", " "))
def __str__(self): return self.left + " -> " + list_to_string(self.right)
def get_evidence_keyword(evidence): key_word_result = list_to_string(get_key_word_list(evidence)) return key_word_result
def get_divide_result(evidence): divide_result = list_to_string(get_divide_list(evidence)) return divide_result
def test_set_parser(publication_txt_path_prefix, publications_json_path, data_sets_json_path, output_filename): data_set_mention_info = [] pub_date_dict = dict() print("Loading data_sets.json file...") with open(data_sets_json_path) as json_data_sets: data_sets = json.load(json_data_sets) for data_set_info in tqdm(data_sets, total=len(data_sets)): data_set_id = data_set_info.get("data_set_id", None) name = data_set_info.get("name", None) name.encode('ascii', 'ignore') date = data_set_info.get("date", None) date.encode('ascii', 'ignore') date = date[:10] if 'None' in date: date = '1800-01-01' date = int(date[:4]) * 12 * 31 + int(date[5:7]) * 31 + int( date[8:10]) mention_list = data_set_info.get("mention_list", None) formatted_mention_list = [] name = normalize_string(name) name_words = word_tokenize(name) formatted_mention_list.append( [name_words, list_to_string((name_words))]) for mention in mention_list: mention.encode('ascii', 'ignore') mention = normalize_string(mention).strip() mention = re.sub("\s\s+", " ", mention) if all(c.islower() for c in mention) and len(mention.split()) <= 2: continue # to avoid pronoun mentions like 'data', 'time' sentences = split_into_sentences(mention) words = [] for sentence in sentences: words += word_tokenize(sentence) words = [w for w in words if len(w) < 15] if len(words) > 0: formatted_mention_list.append( [words, list_to_string(words)]) data_set_mention_info.append( [date, data_set_id, formatted_mention_list]) data_set_mention_info.sort(key=lambda x: int(x[0]), reverse=True) # set prefix to formatted publication txt files formatted_txt_path_prefix = "./formatted-data/" # set path to publications.json formatted_publications = dict() print("Tokenizing publications.json file...") # open the publications.json file with open(publications_json_path) as json_publication_file: # parse it as JSON publication_list = json.load(json_publication_file) # loop over the elements in the list for publication_info in tqdm(publication_list, total=len(publication_list)): # get information on publication: publication_id = publication_info.get("publication_id", None) title = publication_info.get("title", None) title.encode('ascii', 'ignore') text_file_name = publication_info.get("text_file_name", None) unique_identifier = publication_info.get( "unique_identifier", None) # id가 bbk로 시작하면 pub_date은 None임 if 'bbk' not in unique_identifier: pub_date = publication_info.get("pub_date", None) else: pub_date = '2200-01-01' pub_date.encode('ascii', 'ignore') pub_date = int(pub_date[:4]) * 12 * 31 + int( pub_date[5:7]) * 31 + int(pub_date[8:10]) # get raw text raw_text = '' txt_file_path = publication_txt_path_prefix + text_file_name with open(txt_file_path) as txt_file: for line in txt_file: stripped_line = line.strip() raw_text += ' ' + stripped_line if len(stripped_line.split()) <= 5: raw_text += '<stop>' # marking for sentence boundary in split_into_sentences() function raw_text.encode('ascii', 'ignore') raw_text = normalize_string(raw_text) # add to formatted_publications dictionary formatted_text_list = [] chopped_raw_text = '' sentences = split_into_sentences(raw_text) for sentence in sentences: words = word_tokenize(sentence) words = [w for w in words if len(w) < 15] if len(words) >= 10 and len(words) <= 30: formatted_text_list.append(words) chopped_raw_text += ' ' + list_to_string(words) formatted_publications[publication_id] = [ formatted_text_list, chopped_raw_text.strip() ] pub_date_dict[publication_id] = pub_date # tag mentions in publication text and write in csv file output_filepath = formatted_txt_path_prefix + output_filename with open(output_filepath, 'w') as csvfile: fieldnames = [ 'publication_id', 'sentence', 'label_sequence', 'labeled' ] writer = csv.DictWriter(csvfile, fieldnames=fieldnames, quoting=csv.QUOTE_ALL) writer.writeheader() print("Tagging pre-found dataset mentions in publications...") output = extract_formatted_data_test(formatted_publications, data_set_mention_info, pub_date_dict) print("Writing on new csv file...", end='') writer.writerows(output) print("DONE")
def save_xml_data(file_path, filename): root = ET.parse(file_path + filename) # 文书情况 condition = "正常" # 案由 cause_of_faction = "" # 是否被证实 is_identified = "-1" # 事实 facts = "" # 查明事实 fact_identify = "" # 证据 evidence = "" # 事实分词列表 divide_fact_list = [] # 关键词列表 key_words_list = [] # SQL语句和游标声明 sql_fact_evidence = "insert into fact_evidence (filename, fact, factidentify, isidentified, causeoffaction, evidence)" \ " values(%s, %s, %s, %s, %s, %s)" sql_fact_info = "insert into fact_info (filename, divideresult, keyword, fact, match_rate)" \ " values(%s, %s, %s, %s, %s)" sql_doc_condition = "insert into doc_condition (filename, con)" \ " values(%s, %s)" cur = conn.cursor() # 案由 ays = root.getiterator('AY') for ay in ays: if ay.attrib.has_key('value'): cause_of_faction = ay.attrib['value'] # print "案由:", cause_of_faction if len(ays) == 0: condition = "无案由" cur.execute(sql_doc_condition, (filename, condition)) cur.close() conn.commit() print filename, "无案由" return cause_of_faction = delete_space(cause_of_faction) # 查明事实段 cmssds = root.getiterator('CMSSD') for cmssd in cmssds: if cmssd.attrib.has_key('value'): fact_identify = cmssd.attrib['value'] # print "查明事实段:", fact_identify if len(cmssds) == 0: condition = "无查明事实" cur.execute(sql_doc_condition, (filename, condition)) cur.close() conn.commit() print filename, "无查明事实" return fact_identify = delete_space(fact_identify) # 证据段 zjds = root.getiterator('ZJD') for zjd in zjds: if zjd.attrib.has_key('value'): # 整合原告+被告的证据 evidence += zjd.attrib['value'] # print "证据段:", evidence if len(zjds) == 0: condition = "无证据" cur.execute(sql_doc_condition, (filename, condition)) cur.close() conn.commit() print filename, "无证据" return evidence = delete_space(evidence) # 原告诉称段 ygscds = root.getiterator('YGSCD') for ygscd in ygscds: if ygscd.attrib.has_key('value'): all_fact = ygscd.attrib['value'].strip() # print "原告诉称段:", all_fact # fact_all = ygscd.attrib['value'] # facts = fact_all.split('。') if len(ygscds) == 0: condition = "无原告诉称" cur.execute(sql_doc_condition, (filename, condition)) cur.close() conn.commit() print filename, "无原告诉称" return # 删除空格 all_fact = delete_space(all_fact) # 根据句号划分 facts = all_fact.split('。') # 删除诉求部分 facts = delete_appeal(facts) # 写入数据库 facts_len = len(facts) # i = 0 for fact in facts: # i += 1 # 删去事实中的脏数据。中间有空白项,就跳过 if (fact == "") or (fact == "……") or (len(fact) < 10): continue # 不写入最后的一个空白项 # if i == facts_len: # break # 获得分词结果 divide_fact_result = list_to_string(get_divide_list(fact)) #获得关键词 key_word = list_to_string(get_key_word_list(fact)) # 判断是否被认定 match_rate = get_match_rate(fact, fact_identify) identify = judge_relevance(match_rate) if identify: is_identified = 1 else: is_identified = 0 cur.execute(sql_fact_evidence, (filename, fact, fact_identify, is_identified, cause_of_faction, evidence)) cur.execute(sql_fact_info, (filename, divide_fact_result, key_word, fact, match_rate)) cur.execute(sql_doc_condition, (filename, condition)) cur.close() conn.commit()