def run(self):
program = []
while len(self.__stack):
print("Stack: " + list_to_string(self.__stack))
want = self.__stack.pop(0)
print("Want " + want)
if want == "token_eos" or want == "epsilon":
continue
if want.startswith("token_keyword"):
program.append(want[len("token_keyword_"):])
continue
if want == "token_identifier":
# FIXME: choose from applicable identifiers
program.append("a")
continue
if want == "token_number":
program.append("0")
continue
if want.startswith("token_"):
program.append(Scanner.tokenNameToString(want))
continue
applicable_rules = [r for r in rules if r.left == want]
# In order to not grow the stack, give the simplest production a lot of weight...
if random() > 0.7:
ix = 0
else:
ix = floor(random() * len(applicable_rules))
print(list_to_string(applicable_rules))
print(ix)
print("Rule: " + str(applicable_rules[ix]))
self.__stack = applicable_rules[ix].right + self.__stack
print("Program so far: ")
print(list_to_string(program, "", "", " "))
return program
def get_ticker(self, pairs):
"""Get ticker infomation
Args:
pairs: a string containing asset pairs delimited by ","
Returns:
A dictionary containing ticker information
See https://www.kraken.com/help/api#get-ticker-info for documentation
Raises:
RuntimeError: An error occurs when failed to retrieve server information
"""
if pairs is None:
raise Exception("KrakenAPIHandle.get_ticker() failed: pairs cannot be None")
conn = self.new_connection()
pair_string = util.list_to_string(pairs)
param = urllib.parse.urlencode({'pair': pair_string})
conn.request('POST', self.api_link.ticker, param)
resp = json.loads(self._read_response(conn.getresponse()).decode())
if len(resp["error"]) != 0:
raise RuntimeError("KrakenAPIHandle.get_ticker(): %s" % util.list_to_string(resp["error"]))
return resp["result"]
def host_inspect(target, extensive):
if core.is_valid_ip(target):
msg = "Ip Validation OK"
logger.debug(msg)
msg = "[+] Valid ip"
logger.info(msg)
msg = "[*] Performing hostname conversion"
logger.info(msg)
try:
value = core.get_host_by_ip(target)
util.list_to_string(value)
except:
msg = "[-] ERROR: Cannot resolve hostname"
logger.error(msg)
elif core.is_valid_hostname(target):
msg = "Host Validation OK"
logger.debug(msg)
msg = "[+] Valid host"
logger.info(msg)
msg = "[*] Performing ip conversion"
logger.info(msg)
try:
value = core.get_host_by_name(target)
util.list_to_string(value)
except:
msg = "[-] ERROR: Cannot resolve hostname"
logger.error(msg)
else:
msg = "[-] ERROR: You must provide a valid target. Given: "+ target
showhelp()
logger.error(msg)
sys.exit(1)
db = GEOIPFILE
geo = core.ip_to_country(core.get_ip(target), db)
if geo:
msg = "[+] The host is situated in "+geo
logger.info(msg)
else:
msg = "[-] Cannot geolocalize the host"
logger.warning(msg)
if extensive:
msg = "Extensive probing"
logger.debug(msg)
msg = "[*] Starting extensive information gathering"
logger.info(msg)
whois = core.get_extensive_data(target, 0)
info = core.get_extensive_data(target, 1)
dns = core.get_extensive_data(target, 2)
def host_inspect(target, extensive):
if core.is_valid_ip(target):
msg = "Ip Validation OK"
logger.debug(msg)
msg = "[+] Valid ip"
logger.info(msg)
msg = "[*] Performing hostname conversion"
logger.info(msg)
try:
value = core.get_host_by_ip(target)
util.list_to_string(value)
except:
msg = "[-] ERROR: Cannot resolve hostname"
logger.error(msg)
elif core.is_valid_hostname(target):
msg = "Host Validation OK"
logger.debug(msg)
msg = "[+] Valid host"
logger.info(msg)
msg = "[*] Performing ip conversion"
logger.info(msg)
try:
value = core.get_host_by_name(target)
util.list_to_string(value)
except:
msg = "[-] ERROR: Cannot resolve hostname"
logger.error(msg)
else:
msg = "[-] ERROR: You must provide a valid target. Given: " + target
showhelp()
logger.error(msg)
sys.exit(1)
db = GEOIPFILE
geo = core.ip_to_country(core.get_ip(target), db)
if geo:
msg = "[+] The host is situated in " + geo
logger.info(msg)
else:
msg = "[-] Cannot geolocalize the host"
logger.warning(msg)
if extensive:
msg = "Extensive probing"
logger.debug(msg)
msg = "[*] Starting extensive information gathering"
logger.info(msg)
whois = core.get_extensive_data(target, 0)
info = core.get_extensive_data(target, 1)
dns = core.get_extensive_data(target, 2)
def get_ohlc(self, pair, interval=1, since=None):
"""Get ticker infomation
Args:
pair: an asset pair string
interval: the time interval of ohlc
(Available values: 1 (default), 5, 15, 30, 60, 240, 1440, 10080, 21600)
since: return committed OHLC data since given id
Returns:
A dictionary containing ticker information
See https://www.kraken.com/help/api#get-ticker-info for documentation
Raises:
RuntimeError: An error occurs when failed to retrieve server information
"""
if pair is None:
return
conn = self.new_connection()
if since is None:
param = urllib.parse.urlencode({'pair': pair, 'interval': interval})
else:
param = urllib.parse.urlencode({'pair': pair, 'interval': interval, 'since': since})
conn.request('POST', self.api_link.ohlc, param)
resp = json.loads(self._read_response(conn.getresponse()).decode())
if len(resp["error"]) != 0:
raise RuntimeError("KrakenAPIHandle.get_ohlc(): %s" % util.list_to_string(resp["error"]))
return resp["result"]
def __init__(self, parameter_types, return_type):
assert (isinstance(return_type, Type))
assert (isinstance(parameter_types, TypeList)
or parameter_types is None)
parameter_types = parameter_types or TypeList(None)
name = list_to_string(
parameter_types.items) + " -> " + str(return_type)
super().__init__(name)
self.return_type = return_type
self.parameter_types = parameter_types.items
def get_orderbook(self, pairs, count=10):
if pairs is None:
return
conn = self.new_connection()
pair_string = util.list_to_string(pairs)
param = urllib.parse.urlencode({'pair': pair_string, 'count': count})
conn.request('POST', self.api_link.orderbook, param)
resp = json.loads(self._read_response(conn.getresponse()).decode())
if len(resp["error"]) != 0:
raise RuntimeError("KrakenAPIHandle.get_orderbook(): Unable to retrieve orderbook")
return resp["result"]
def main():
print("Hello! Welcome~")
print("Please enter the kind of function you would like to use.")
print("Currently supported functions are: \n" +
list_to_string(FUNCTIONS_SUPPORTED))
sel = input("Select an option: ")
# Apply logic based on selection
if (sel == "add"):
print("Specify two numbers, a and b")
a = input("a: ")
b = input("b: ")
# Cast numbers to float via function, then add
res = Calculator.add(float(a), float(b))
print("Result: " + str(res))
elif (sel == "sub"):
print("Specify two numbers, a and b")
a = input("a: ")
b = input("b: ")
res = Calculator.sub(float(a), float(b))
print("Result " + str(res))
elif (sel == "mul"):
print("Specify two numbers, a and b")
a = input("a: ")
b = input("b: ")
res = Calculator.mul(float(a), float(b))
print("Result " + str(res))
elif (sel == "div"):
print("Specify two numbers, a and b")
a = input("a: ")
b = input("b: ")
res = Calculator.div(float(a), float(b))
print("Result " + str(res))
elif (sel == "power"):
print("Specify two numbers, a and b")
a = input("a: ")
b = input("b: ")
res = Calculator.power(float(a), float(b))
print("Result " + str(res))
else:
print("Function \"" + sel + "\" not implemented.")
def get_time(self):
"""Get Kraken server time
Returns:
A dictionary containing server time
See https://www.kraken.com/help/api#get-server-time for documentation
Raises:
RuntimeError: An error occurs when failed to retrieve server information
"""
conn = self.new_connection()
conn.request('GET',self.api_link.time)
resp = json.loads(self._read_response(conn.getresponse()).decode())
if len(resp["error"]) != 0:
raise RuntimeError("KrakenAPIHandle.get_time(): %s" % util.list_to_string(resp["error"]))
return resp["result"]
def get_asset_pairs(self):
"""Get information of all asset pairs available on Kraken exchange
Returns:
A dictionary containing all asset pair information
See https://www.kraken.com/help/api#get-asset-info for documentation
Raises:
RuntimeError: An error occurs when failed to retrieve server information
"""
conn = self.new_connection()
conn.request('GET', self.api_link.assetpair)
resp = json.loads(self._read_response(conn.getresponse()).decode())
if len(resp["error"]) != 0:
raise RuntimeError("KrakenAPIHandle.get_asset_pairs(): %s" % util.list_to_string(resp["error"]))
return resp["result"]
def make_next_move(self):
"""
Get the next square that the AI wants to expose on the board
:return: The coordinates of a square that the AI wants to expose
"""
print "finding move. Queue = "
pprint(self.uncover_queue)
print "Groups = " + util.list_to_string(self.groups)
# If first click:
if self.first_click:
self.first_click = False
x = self.minefield_width / 2
y = self.minefield_height / 2
else:
if not self.uncover_queue:
print "No moves can be deduced! Guessing not yet implemented"
# take an educated guess
click = self.uncover_queue.pop(0)
x = click[0]
y = click[1]
# Is a little long to make anonymous
def callback(degree):
square = self.squares[x][y]
self.groups_to_check += square.uncover(degree, False)
mines = square.degree
squares = []
for neighbor in square.neighbors:
if neighbor.covered:
squares += [neighbor]
elif neighbor.mine:
mines -= 1
group = Group(square.coordinates, squares, mines,
self.dissolve_group)
self.groups += [group]
self.groups_to_check += [group]
while self.groups_to_check:
self.groups_to_check += self.groups_to_check.pop(0).check()
self.query(x, y, callback)
def get_xml_data(file_path, filename):
root = ET.parse(file_path + filename)
# 文书情况
condition = "正常"
# 事实
facts = ""
# 证据
evidence = ""
# 是否被证实
is_identified = "-1"
# 证据关键词
evidence_keyword = ""
# 证据分词
evidence_divide = ""
# 事实分词列表
divide_fact_list = []
# 关键词列表
key_words_list = []
# 返回数组
return_list = []
# 证据段
zjds = root.getiterator('ZJD')
for zjd in zjds:
if zjd.attrib.has_key('value'):
# 整合原告+被告的证据
evidence += zjd.attrib['value']
# print "证据段：", evidence
if len(zjds) == 0:
condition = "无证据"
print filename, "无证据"
return
evidence = delete_space(evidence)
evidence_divide = get_divide_result(evidence)
evidence_keyword = get_evidence_keyword(evidence)
# 原告诉称段
ygscds = root.getiterator('YGSCD')
for ygscd in ygscds:
if ygscd.attrib.has_key('value'):
all_fact = ygscd.attrib['value'].strip()
# print "原告诉称段：", all_fact
# fact_all = ygscd.attrib['value']
# facts = fact_all.split('。')
if len(ygscds) == 0:
condition = "无原告诉称"
print filename, "无原告诉称"
return
# 查明事实段
cmssds = root.getiterator('CMSSD')
for cmssd in cmssds:
if cmssd.attrib.has_key('value'):
fact_identify = cmssd.attrib['value']
# print "查明事实段：", fact_identify
if len(cmssds) == 0:
condition = "无查明事实"
print filename, "无查明事实"
return
fact_identify = delete_space(fact_identify)
# 删除空格
all_fact = delete_space(all_fact)
# 根据句号划分
facts = all_fact.split('。')
# 删除诉求部分
facts = delete_appeal(facts)
for fact in facts:
# 删去事实中的脏数据。中间有空白项，就跳过
if (fact == "") or (fact == "……") or (len(fact) < 10):
continue
# 获得分词结果
fact_divide = list_to_string(get_divide_list(fact))
#获得关键词
fact_keyword = list_to_string(get_key_word_list(fact))
# 判断是否被认定
match_rate = get_match_rate(fact, fact_identify)
identify = judge_relevance(match_rate)
if identify:
is_identified = 1
else:
is_identified = 0
# [fact, fact_divide, fact_keyword, evidence, evidence_divide, evidence_keyword, is_identified]
a_set_of_data = [
fact, fact_divide, fact_keyword, evidence, evidence_divide,
evidence_keyword, is_identified
]
return_list.append(a_set_of_data)
return return_list
# FIXME: choose from applicable identifiers
program.append("a")
continue
if want == "token_number":
program.append("0")
continue
if want.startswith("token_"):
program.append(Scanner.tokenNameToString(want))
continue
applicable_rules = [r for r in rules if r.left == want]
# In order to not grow the stack, give the simplest production a lot of weight...
if random() > 0.7:
ix = 0
else:
ix = floor(random() * len(applicable_rules))
print(list_to_string(applicable_rules))
print(ix)
print("Rule: " + str(applicable_rules[ix]))
self.__stack = applicable_rules[ix].right + self.__stack
print("Program so far: ")
print(list_to_string(program, "", "", " "))
return program
if __name__ == "__main__":
f = Fuzzer(rules)
program = f.run()
print("Program:")
print(list_to_string(program, "", "", " "))
def __str__(self):
return self.left + " -> " + list_to_string(self.right)
def get_evidence_keyword(evidence):
key_word_result = list_to_string(get_key_word_list(evidence))
return key_word_result
def get_divide_result(evidence):
divide_result = list_to_string(get_divide_list(evidence))
return divide_result
def test_set_parser(publication_txt_path_prefix, publications_json_path,
data_sets_json_path, output_filename):
data_set_mention_info = []
pub_date_dict = dict()
print("Loading data_sets.json file...")
with open(data_sets_json_path) as json_data_sets:
data_sets = json.load(json_data_sets)
for data_set_info in tqdm(data_sets, total=len(data_sets)):
data_set_id = data_set_info.get("data_set_id", None)
name = data_set_info.get("name", None)
name.encode('ascii', 'ignore')
date = data_set_info.get("date", None)
date.encode('ascii', 'ignore')
date = date[:10]
if 'None' in date:
date = '1800-01-01'
date = int(date[:4]) * 12 * 31 + int(date[5:7]) * 31 + int(
date[8:10])
mention_list = data_set_info.get("mention_list", None)
formatted_mention_list = []
name = normalize_string(name)
name_words = word_tokenize(name)
formatted_mention_list.append(
[name_words, list_to_string((name_words))])
for mention in mention_list:
mention.encode('ascii', 'ignore')
mention = normalize_string(mention).strip()
mention = re.sub("\s\s+", " ", mention)
if all(c.islower()
for c in mention) and len(mention.split()) <= 2:
continue # to avoid pronoun mentions like 'data', 'time'
sentences = split_into_sentences(mention)
words = []
for sentence in sentences:
words += word_tokenize(sentence)
words = [w for w in words if len(w) < 15]
if len(words) > 0:
formatted_mention_list.append(
[words, list_to_string(words)])
data_set_mention_info.append(
[date, data_set_id, formatted_mention_list])
data_set_mention_info.sort(key=lambda x: int(x[0]), reverse=True)
# set prefix to formatted publication txt files
formatted_txt_path_prefix = "./formatted-data/"
# set path to publications.json
formatted_publications = dict()
print("Tokenizing publications.json file...")
# open the publications.json file
with open(publications_json_path) as json_publication_file:
# parse it as JSON
publication_list = json.load(json_publication_file)
# loop over the elements in the list
for publication_info in tqdm(publication_list,
total=len(publication_list)):
# get information on publication:
publication_id = publication_info.get("publication_id", None)
title = publication_info.get("title", None)
title.encode('ascii', 'ignore')
text_file_name = publication_info.get("text_file_name", None)
unique_identifier = publication_info.get(
"unique_identifier", None) # id가 bbk로 시작하면 pub_date은 None임
if 'bbk' not in unique_identifier:
pub_date = publication_info.get("pub_date", None)
else:
pub_date = '2200-01-01'
pub_date.encode('ascii', 'ignore')
pub_date = int(pub_date[:4]) * 12 * 31 + int(
pub_date[5:7]) * 31 + int(pub_date[8:10])
# get raw text
raw_text = ''
txt_file_path = publication_txt_path_prefix + text_file_name
with open(txt_file_path) as txt_file:
for line in txt_file:
stripped_line = line.strip()
raw_text += ' ' + stripped_line
if len(stripped_line.split()) <= 5:
raw_text += '<stop>' # marking for sentence boundary in split_into_sentences() function
raw_text.encode('ascii', 'ignore')
raw_text = normalize_string(raw_text)
# add to formatted_publications dictionary
formatted_text_list = []
chopped_raw_text = ''
sentences = split_into_sentences(raw_text)
for sentence in sentences:
words = word_tokenize(sentence)
words = [w for w in words if len(w) < 15]
if len(words) >= 10 and len(words) <= 30:
formatted_text_list.append(words)
chopped_raw_text += ' ' + list_to_string(words)
formatted_publications[publication_id] = [
formatted_text_list,
chopped_raw_text.strip()
]
pub_date_dict[publication_id] = pub_date
# tag mentions in publication text and write in csv file
output_filepath = formatted_txt_path_prefix + output_filename
with open(output_filepath, 'w') as csvfile:
fieldnames = [
'publication_id', 'sentence', 'label_sequence', 'labeled'
]
writer = csv.DictWriter(csvfile,
fieldnames=fieldnames,
quoting=csv.QUOTE_ALL)
writer.writeheader()
print("Tagging pre-found dataset mentions in publications...")
output = extract_formatted_data_test(formatted_publications,
data_set_mention_info,
pub_date_dict)
print("Writing on new csv file...", end='')
writer.writerows(output)
print("DONE")
def save_xml_data(file_path, filename):
root = ET.parse(file_path + filename)
# 文书情况
condition = "正常"
# 案由
cause_of_faction = ""
# 是否被证实
is_identified = "-1"
# 事实
facts = ""
# 查明事实
fact_identify = ""
# 证据
evidence = ""
# 事实分词列表
divide_fact_list = []
# 关键词列表
key_words_list = []
# SQL语句和游标声明
sql_fact_evidence = "insert into fact_evidence (filename, fact, factidentify, isidentified, causeoffaction, evidence)" \
" values(%s, %s, %s, %s, %s, %s)"
sql_fact_info = "insert into fact_info (filename, divideresult, keyword, fact, match_rate)" \
" values(%s, %s, %s, %s, %s)"
sql_doc_condition = "insert into doc_condition (filename, con)" \
" values(%s, %s)"
cur = conn.cursor()
# 案由
ays = root.getiterator('AY')
for ay in ays:
if ay.attrib.has_key('value'):
cause_of_faction = ay.attrib['value']
# print "案由：", cause_of_faction
if len(ays) == 0:
condition = "无案由"
cur.execute(sql_doc_condition, (filename, condition))
cur.close()
conn.commit()
print filename, "无案由"
return
cause_of_faction = delete_space(cause_of_faction)
# 查明事实段
cmssds = root.getiterator('CMSSD')
for cmssd in cmssds:
if cmssd.attrib.has_key('value'):
fact_identify = cmssd.attrib['value']
# print "查明事实段：", fact_identify
if len(cmssds) == 0:
condition = "无查明事实"
cur.execute(sql_doc_condition, (filename, condition))
cur.close()
conn.commit()
print filename, "无查明事实"
return
fact_identify = delete_space(fact_identify)
# 证据段
zjds = root.getiterator('ZJD')
for zjd in zjds:
if zjd.attrib.has_key('value'):
# 整合原告+被告的证据
evidence += zjd.attrib['value']
# print "证据段：", evidence
if len(zjds) == 0:
condition = "无证据"
cur.execute(sql_doc_condition, (filename, condition))
cur.close()
conn.commit()
print filename, "无证据"
return
evidence = delete_space(evidence)
# 原告诉称段
ygscds = root.getiterator('YGSCD')
for ygscd in ygscds:
if ygscd.attrib.has_key('value'):
all_fact = ygscd.attrib['value'].strip()
# print "原告诉称段：", all_fact
# fact_all = ygscd.attrib['value']
# facts = fact_all.split('。')
if len(ygscds) == 0:
condition = "无原告诉称"
cur.execute(sql_doc_condition, (filename, condition))
cur.close()
conn.commit()
print filename, "无原告诉称"
return
# 删除空格
all_fact = delete_space(all_fact)
# 根据句号划分
facts = all_fact.split('。')
# 删除诉求部分
facts = delete_appeal(facts)
# 写入数据库
facts_len = len(facts)
# i = 0
for fact in facts:
# i += 1
# 删去事实中的脏数据。中间有空白项，就跳过
if (fact == "") or (fact == "……") or (len(fact) < 10):
continue
# 不写入最后的一个空白项
# if i == facts_len:
# break
# 获得分词结果
divide_fact_result = list_to_string(get_divide_list(fact))
#获得关键词
key_word = list_to_string(get_key_word_list(fact))
# 判断是否被认定
match_rate = get_match_rate(fact, fact_identify)
identify = judge_relevance(match_rate)
if identify:
is_identified = 1
else:
is_identified = 0
cur.execute(sql_fact_evidence,
(filename, fact, fact_identify, is_identified,
cause_of_faction, evidence))
cur.execute(sql_fact_info,
(filename, divide_fact_result, key_word, fact, match_rate))
cur.execute(sql_doc_condition, (filename, condition))
cur.close()
conn.commit()
