def routine4short(self,
get_dict=dict(),
xpath_dict=dict(),
sleep_time=0.2) -> dict:
tmp_dict = dict()
for key in xpath_dict.keys():
if key in get_dict.keys():
self.get_2(get_dict[key])
if key in xpath_dict.keys():
self.dialog_block_wait_xpath(xpath_dict[key])
tmp_tag = self.find_element_by_xpath(xpath_dict[key])
tmp_dict[key] = tmp_tag.get_attribute("outerHTML")
tmp_dict["is_successful"] = self.__is_successful(tmp_dict)
return tmp_dict
def get_results_from_table(self,
table_name: str,
func_dict=dict(),
elapsed_time=15.5) -> pd.DataFrame:
tmp_now = datetime.now()
tmp_command = "select * from {} where created > \"{}\"".format(
table_name, tmp_now - timedelta(minutes=elapsed_time))
tmp_df = self.__sql_db.execute(tmp_command)
tmp_df = tmp_df.to_numpy()
tmp_list = list()
for data_line in tmp_df:
tmp_dict = {"url": data_line[0], "created": data_line[1]}
tmp_dict.update(self.str2dict(data_line[2], func_dict=func_dict))
tmp_list.append(tmp_dict)
return pd.DataFrame(tmp_list)
def routine4selenium(self,
order_list=list(),
get_dict=dict(),
click_dict=dict(),
insert_dict=dict(),
select_dict=dict(),
xpath_dict=dict(),
class_dict=dict(),
sleep_time=0.2) -> dict:
"""
get_dict => click_dict => insert_dict => select_dict => xpath_dict
"""
tmp_dict = dict()
for key in order_list:
if key in get_dict.keys():
if not self.get_2(get_dict[key]):
self.log_error()
return None
if key in click_dict.keys():
if not self.scroll_down_xpath(click_dict[key]):
self.log_error()
return None
self.click_button(click_dict[key], sleep_time=sleep_time)
if key in insert_dict.keys():
self.insert_word(insert_dict[key][0], insert_dict[key][1])
if key in select_dict.keys():
if not self.scroll_down_xpath(select_dict[key][0]):
self.log_error()
return None
if not self.input_select_into_dropdown(select_dict[key][0],
select_dict[key][1]):
return None
if key in xpath_dict.keys():
if not self.scroll_down_xpath(xpath_dict[key]):
self.log_error()
return None
self.dialog_block_wait_xpath(xpath_dict[key])
tmp_tag = self.find_element_by_xpath(xpath_dict[key])
tmp_dict[key] = tmp_tag.get_attribute("outerHTML")
if key in class_dict.keys():
if not self.scroll_down_class_name(class_dict[key]):
self.log_error()
return None
self.dialog_block_wait_class_name(class_dict[key])
tmp_tag = self.find_element_by_class_name(class_dict[key])
tmp_dict[key] = tmp_tag.get_attribute("outerHTML")
tmp_dict["is_successful"] = self.__is_successful(tmp_dict)
return tmp_dict
def str2dict(self, dict_str: str, func_dict=dict()) -> dict:
tmp_dict = json.loads(dict_str)
tmp_head_dtype_dict = self.__sql_db.get_head_dtype_dict()
tmp_set = set(key for key in tmp_dict.keys()) - set(
key for key in tmp_head_dtype_dict.keys()) - set(
key for key in func_dict.keys())
self.__sql_db.insert_head_dtypes(list(tmp_set))
tmp_head_dtype_dict = self.__sql_db.get_head_dtype_dict()
if not func_dict:
self.__functions.update(func_dict)
for key in list(tmp_dict.keys()) + list(func_dict.keys()):
if "char" in tmp_head_dtype_dict[key]:
tmp_dict[key] = self.__functions["str"](tmp_dict[key])
else:
if key in list(tmp_head_dtype_dict.keys()):
tmp_dict[key] = self.__functions[tmp_head_dtype_dict[key]](
tmp_dict[key])
else:
tmp_dict.update(self.__functions[key](tmp_dict[key]))
del tmp_dict[key]
return tmp_dict
import dict
#main program
word = input("Enter word for meaning: ")
print(dict.dict(word))
# print('Type two attribs and then type enter key.')
print("Please input the first word:")
list.append(input())
print("Please intpu the second word")
list.append(input())
else:
list[0]=argv[0]
list[1]=argv[1]
print('[searching on website]: '+list[0]+' '+list[1])
a = list[0]
b = list[1]
webAdd = 'https://en.wikipedia.org/wiki/'
webPost = ''
# webAdd='https://cn.bing.com/search?q='
# webPost='&qs=n&form=QBLH&sp=-1&pq=furude&sc=0-6&sk=&cvid=A07B1D0226E14E5D8585A15510974222'
obj = dict.dict()
lis = obj.data()
wa = r.get(webAdd+a+webPost).text
wb = r.get(webAdd+b+webPost).text
print('[pairing infomation]')
# fil = re.compile('<[^>]*>')
# wa = fil.sub('', wa)
# wb = fil.sub('', wb)
# wa=webGet().filter_tags(wa)
# wb=webGet().filter_tags(wb)
wa=webGet.webGet().filter_tags(wa)
wb=webGet.webGet().filter_tags(wb)
na = []
nb = []
for i in lis:
def privmsg(self, user, channel, msg): # What happens when bot gets a message
self.count = self.count + 1
swag = user.split('!')
user = user.split('!', 1)[0] # Turn user!ident@host into nickname
print("%s:<%s> %s" % (channel, user, msg)) # Show the message in console
capsIntact = msg
if channel == self.nickname: # Check if it's a private message
pm = 1
else:
pm = 0
# append(channel, user, msg)
msg = msg.lower()
if swag[1].lower() in self.opersh or channel.lower() == "#dino": # if the user is in the oper list
admin = 1
else:
admin = 0
if channel == "#thegrammarnaziboners":
tstchan = 1
else:
tstchan = 0
insult_ = True
fibo_ = True
choose_ = True
dict_ = True
wiki_ = True
tz_ = True
wz_ = True
rule_ = True
convert_ = True
flist = {'insult': insult_, 'fibo': fibo_, 'choose': choose_, 'dict': dict_, 'wiki': wiki_, 'wz': wz_, 'tz': tz_, 'rule': rule_, 'convert': convert_}
if msg.startswith("~die"):
try:
if admin == 1:
if user.lower() in self.azi:
print "Killed by %s" % user
reactor.stop()
else:
self.notice(user, "Not amused")
print("%s attempted to kill me!")
except:
self.notice(user, "You are not authorized to perform this command.")
print("%s attempted to kill me!" % user)
elif msg.startswith("~set ") and swag.lower() == "*****@*****.**":
# self.sendLine("MODE %s +B %s" % (self.nickname, self.nickname))
self.mode(self.nickname, True, 'B', limit=None, user=self.nickname, mask=None)
elif msg.startswith("~stop") and admin == 1:
self.quit()
elif msg.startswith("~id") and (admin == 1 or user.lower == "Kojiro"):
self.msg("Nickserv", "identify hamsterbot %s" % self.password)
elif msg.startswith("~ghost ") and admin == 1:
msg = msg.split()
self.msg("Nickserv", "ghost %s swagonball" % msg[1])
elif msg.startswith("~donut") and admin == 1:
msg = msg.split(' ')
try:
if msg[1].isdigit(): #hopefully if i use a number it'll join that chan
i = int(msg[1])
chan = self.chanlist[i]
self.join(chan) #why doesnt this work nvm it does
elif msg[1] == "joinall" or msg[1] == "all":
for item in self.chanlist:
self.join(item)
else:
self.join(msg[1])
except:
if admin != 1:
self.notice(user, "You are not authorized to perform this command.")
else:
self.say(channel, "-_-")
elif msg.startswith("~bye"):
try:
if admin == 1:
self.part(channel)
except:
self.notice(user, "You are not authorized to perform this command.")
print("%s attempted to use command ~bye." % user)
elif msg.startswith("~nick"):
try:
if admin == 1:
msg = msg.split(' ')
self.setNick(msg[1])
except:
self.notice(user, "You are not authorized to perform this command.")
print("%s tried to change my nick" % user)
elif msg.startswith("~toggle ") and admin == 1:
msg = msg.split()
if len(msg) == 3:
if flist.has_key(msg[1]):
funct = msg[1].strip()
if msg[2].strip() == "on" or msg[2] == '1':
flist[funct] = True
elif msg[2].strip() == 'off' or msg[2] == '0':
flist[funct] = False
elif msg[2] == 'not':
flist[funct] = not flist[funct]
else:
self.notice(user, "Values accepted: on/off or 1/0")
else:
self.notice(user, "No function found with that name or function may not be switched on/off")
elif len(msg) == 2:
if msg[1] == 'help':
self.notice(user, "The following functions may be switched on/off: insult, choose, convert, dict, fibo, rule, tz, wz, wiki")
elif msg[1] == 'is_on':
is_on = []
for key, value in flist.iteritems():
if value == True:
is_on.append(key)
self.notice(user, "The following functions are on: %r" % is_on)
else:
self.notice(user, "Toggle takes 2 arguments. Do ~toggle help for functions that may be toggled on/off")
elif msg.startswith("~say ") and admin == 1:
msg = msg[5:]
self.say(channel, msg)
elif msg.startswith("~note ") and (swag[1].lower()=="*****@*****.**"):
msg = msg[6:]
# bs.notes(str(msg))
self.notice(user, notes(str(msg)))
elif msg.startswith(":prefixes "):
msg = msg[10:]
self.say(channel, whattodo(msg, user))
elif msg.startswith(".insult ") and insult_ == True:
msg = msg.split(' ')
#nick = msg[1]
try:
if msg[1].lower() in self.azi:
self.say(channel, random.choice(azi_insults))
elif msg[1].lower() == "?roulette":
self.say(channel, "%s, %s" % (user, random.choice(insults)))
else:
insult = "%s, %s" % (msg[1], random.choice(insults))
self.say(channel, insult)
except IndexError:
self.say(channel, random.choice(insults))
elif msg.startswith(".inslut") and flist['insult'] == True:
msg = msg.split(' ')
insult = "%s, %s" % (user, random.choice(insults))
self.say(channel, insult)
elif msg.startswith(".halp") or msg.startswith(".help"):
try:
if admin == 1:
self.notice(user, "Commands: .help, .insult <user>, .fibo, .convert <conversion> <numbers>, .gay, .rule <1-47>, .choose <option1, option2,...>, .dict <query>")
self.notice(user, "Restricted commands: ~die, ~bye, ~donut <channel>, ~nick <newnick>")
except:
self.notice(user, "Commands: .help, .about, .insult <user>, .fibo, .convert <conversion> <numbers>")
# elif msg.startswith("~~"):
# msg = msg.strip("~~")
# exec(msg)
elif msg.startswith(".choose ") and flist['choose'] == True:
msg = msg[8:]
# re.sub("or*", '', msg)
msg = msg.split(",")
msg = random.choice(msg)
words = msg.split(' ')
if words[0] == 'or' and len(words) > 1:
msg = msg.replace('or ','')
self.msg(channel, "%s: %s" % (user, msg.strip()))
# elif msg.startswith(".last"):
# msg = msg[5:]
# list = last(channel, msg)
# for item in list:
# self.msg(user, item)
elif msg.startswith(".wiki ") and flist['wiki'] == True: #broken wikipedia feature
msg = msg[6:]
msg = msg.replace(' ', '_')
self.msg(channel, str(wiki(str(msg))))
elif msg.startswith(".dict ") and flist['dict'] == True: #dictionary feature
# msg = capsIntact
msg = msg[6:]
self.msg(channel, dict(msg))
elif msg.startswith(".tz ") and flist['tz'] == True: #timezone feature
msg = msg[4:]
self.msg(channel, tz(msg))
elif msg.startswith(".wz ") and flist['wz'] == True: #weatherbot feature
msg = msg[4:]
self.msg(channel, wz(msg))
elif msg.startswith(".gay"):
self.msg(channel, "%s: http://blog.okcupid.com/index.php/gay-sex-vs-straight-sex" % user)
elif msg.startswith(".rule ") and flist['rule'] == True:
msg = msg.split()
self.msg(channel, rules((msg[1].strip())))
elif msg.startswith(".fibo") and flist['fibo'] == True:
msg = msg.split(' ')
num = msg[1]
i = fib2(int(num))
self.msg(channel, str(i))
elif msg.startswith(".convert") and flist['convert'] == True:
msg = msg.split(' ')
if convert(msg).startswith('S') or convert(msg).startswith('N'):
if '::' in convert(msg):
resp = convert(msg).split('::')
self.notice(user, resp[0])
self.notice(user, resp[1])
else:
self.notice(user, convert(msg))
elif pm == 1:
self.notice(user, convert(msg))
else:
self.msg(channel, convert(msg))
for s in self.cmap ]
self.vowel_pat = re.compile("[aeiouy]*")
self.vowel_pat_plus = re.compile("[aeiouy]+")
self.d = dictionary
def set_dictionary(self, d):
self.d = d
# A sequence of numbers each 0-15
def pat_for(self, *n):
pats = [ self.vowel_pat ]
for num in n:
pats += [ self.pats[num], self.vowel_pat_plus ]
pats.pop()
pats.append(self.vowel_pat)
pat = re.compile("^" + "".join([p.pattern for p in pats]) + "$")
print(pat.pattern)
return pat
def words_for(self, *n):
pat = self.pat_for(*n)
words = self.d.select(pat)
yield from [ (w, len(n)) for w in words ]
if __name__ == '__main__':
s = selector(dictionary=dict("/usr/share/dict/words"))
w = s.words_for(0b1111, 0b1010, 0b0101)
# w = s.words_for(0b1111, 0b1010)
for str in w: print(str)
def crawl_site(self,
queue_table_name: str,
scanned_url_table=None,
func=None,
length=1000,
sleep_time=0.3) -> None:
tmp_command: str = "select * from {0} limit {1}".format(
queue_table_name, str(length))
if self.__sql_db is None:
print("No sql_db exists")
else:
tmp_df = self.__sql_db.execute(tmp_command)
if len(tmp_df) != 0:
self.__sql_db.execute(
"lock tables {} write;".format(queue_table_name))
tmp_command = "select * from {0} limit {1}".format(
queue_table_name, length)
tmp_df = self.__sql_db.execute(tmp_command)
tmp_command = "delete from {0} limit {1}".format(
queue_table_name, str(length))
self.__sql_db.execute(tmp_command)
self.__sql_db.execute("unlock tables;")
tmp_df = tmp_df.to_numpy().tolist()
tmp_heads_list, tmp_heads_dtype = self.__sql_db.get_heads_dtype(
queue_table_name)
tmp_sqllike = PsuedoSQLFromCSV("")
tmp_sqllike.dtype = dict(table_name='str',
url='str',
created='datetime',
dict='str')
tmp_sqllike.header = ('table_name', 'url', 'created', 'dict')
tmp_sqllike.data = list()
if scanned_url_table:
tmp_sqllike_urls = PsuedoSQLFromCSV("")
tmp_sqllike_urls.dtype = dict(url='str')
tmp_sqllike_urls.header = ['url']
tmp_sqllike_urls.data = list()
for data_line in tqdm(tmp_df):
tmp_table_list = self.__sql_db.get_tables()
if data_line[1] not in tmp_table_list:
tmp_command = "create table " + data_line[
1] + "(table_name varchar(50), url varchar(120), created datetime, dict " \
"mediumtext); "
print(tmp_command)
self.__sql_db.execute(tmp_command)
tmp_order_list = list()
tmp_get_dict = dict()
tmp_click_dict = dict()
tmp_insert_dict = dict()
tmp_select_dict = dict()
tmp_xpath_dict = dict()
tmp_class_name_dict = dict()
tmp_datetime = datetime.now()
tmp_table_name = data_line[tmp_heads_list.index(
"table_name")]
if data_line[tmp_heads_list.index(
"order_list")] is not None:
tmp_order_list = json.loads(
data_line[tmp_heads_list.index("order_list")])
if data_line[tmp_heads_list.index("get_dict")] is not None:
tmp_get_dict = json.loads(
data_line[tmp_heads_list.index("get_dict")])
if data_line[tmp_heads_list.index(
"click_dict")] is not None:
tmp_click_dict = json.loads(
data_line[tmp_heads_list.index("click_dict")])
if data_line[tmp_heads_list.index(
"insert_dict")] is not None:
tmp_insert_dict = json.loads(
data_line[tmp_heads_list.index("insert_dict")])
if data_line[tmp_heads_list.index(
"select_dict")] is not None:
tmp_select_dict = json.loads(
data_line[tmp_heads_list.index("select_dict")])
if data_line[tmp_heads_list.index(
"xpath_dict")] is not None:
tmp_xpath_dict = json.loads(
data_line[tmp_heads_list.index("xpath_dict")])
if data_line[tmp_heads_list.index(
"class_dict")] is not None:
tmp_class_name_dict = json.loads(
data_line[tmp_heads_list.index("class_dict")])
if not self.get_2(data_line[0]):
self.log_error()
continue
tmp_result = self.routine4selenium(
order_list=tmp_order_list,
get_dict=tmp_get_dict,
click_dict=tmp_click_dict,
insert_dict=tmp_insert_dict,
select_dict=tmp_select_dict,
xpath_dict=tmp_xpath_dict,
class_dict=tmp_class_name_dict,
sleep_time=sleep_time)
if tmp_result:
if not tmp_result["is_successful"]:
self.log_error()
continue
if func:
tmp_result["url"] = data_line[0]
func(tmp_result)
tmp_sqllike.data.append(list(tmp_result.values()))
else:
tmp_result = json.dumps(tmp_result)
tmp_sqllike.data.append([
tmp_table_name, data_line[0], tmp_datetime,
tmp_result
])
if scanned_url_table:
tmp_sqllike_urls.data.append(data_line[0])
try:
if func:
tmp_sqllike.header = list(tmp_result.keys())
self.__sql_db.insert_data(data_line[1], tmp_sqllike)
except Exception:
tmp_sqllike.save_data_to_csv("test.csv")
if scanned_url_table:
self.__sql_db.insert_data(scanned_url_table,
tmp_sqllike_urls)
data = build_data(f)
#Load data raster
print "Loading Data Raster..."
#Load data raster
raster_list = glob.glob(path + '*.TIF')
dataRaster = []
for i in raster_list:
band = gdal.Open(i)
dataRaster.append(band.GetRasterBand(1).ReadAsArray().astype(float))
filename = []
for a in [os.path.basename(x) for x in glob.glob(path + '*.TIF')]:
p = os.path.splitext(a)[0]
filename.append(p)
my_dict = dict(zip(filename, dataRaster))
#Load data raster aspect & slope
pathname = 'Folder name' # folder consist of aspect and slope data
raster_list_dem = glob.glob(pathname + '/*.TIF')
dataTopo = []
for d in raster_list_dem:
band2 = gdal.Open(d)
dataTopo.append(band2.GetRasterBand(1).ReadAsArray())
#Load data raster sample area
pathsample = 'Folder name' # consist of sample area based on visual digitization
sampledata = gdal.Open(pathsample)
sample = np.array(sampledata.GetRasterBand(1).ReadAsArray())