def routine4short(self, get_dict=dict(), xpath_dict=dict(), sleep_time=0.2) -> dict: tmp_dict = dict() for key in xpath_dict.keys(): if key in get_dict.keys(): self.get_2(get_dict[key]) if key in xpath_dict.keys(): self.dialog_block_wait_xpath(xpath_dict[key]) tmp_tag = self.find_element_by_xpath(xpath_dict[key]) tmp_dict[key] = tmp_tag.get_attribute("outerHTML") tmp_dict["is_successful"] = self.__is_successful(tmp_dict) return tmp_dict
def get_results_from_table(self, table_name: str, func_dict=dict(), elapsed_time=15.5) -> pd.DataFrame: tmp_now = datetime.now() tmp_command = "select * from {} where created > \"{}\"".format( table_name, tmp_now - timedelta(minutes=elapsed_time)) tmp_df = self.__sql_db.execute(tmp_command) tmp_df = tmp_df.to_numpy() tmp_list = list() for data_line in tmp_df: tmp_dict = {"url": data_line[0], "created": data_line[1]} tmp_dict.update(self.str2dict(data_line[2], func_dict=func_dict)) tmp_list.append(tmp_dict) return pd.DataFrame(tmp_list)
def routine4selenium(self, order_list=list(), get_dict=dict(), click_dict=dict(), insert_dict=dict(), select_dict=dict(), xpath_dict=dict(), class_dict=dict(), sleep_time=0.2) -> dict: """ get_dict => click_dict => insert_dict => select_dict => xpath_dict """ tmp_dict = dict() for key in order_list: if key in get_dict.keys(): if not self.get_2(get_dict[key]): self.log_error() return None if key in click_dict.keys(): if not self.scroll_down_xpath(click_dict[key]): self.log_error() return None self.click_button(click_dict[key], sleep_time=sleep_time) if key in insert_dict.keys(): self.insert_word(insert_dict[key][0], insert_dict[key][1]) if key in select_dict.keys(): if not self.scroll_down_xpath(select_dict[key][0]): self.log_error() return None if not self.input_select_into_dropdown(select_dict[key][0], select_dict[key][1]): return None if key in xpath_dict.keys(): if not self.scroll_down_xpath(xpath_dict[key]): self.log_error() return None self.dialog_block_wait_xpath(xpath_dict[key]) tmp_tag = self.find_element_by_xpath(xpath_dict[key]) tmp_dict[key] = tmp_tag.get_attribute("outerHTML") if key in class_dict.keys(): if not self.scroll_down_class_name(class_dict[key]): self.log_error() return None self.dialog_block_wait_class_name(class_dict[key]) tmp_tag = self.find_element_by_class_name(class_dict[key]) tmp_dict[key] = tmp_tag.get_attribute("outerHTML") tmp_dict["is_successful"] = self.__is_successful(tmp_dict) return tmp_dict
def str2dict(self, dict_str: str, func_dict=dict()) -> dict: tmp_dict = json.loads(dict_str) tmp_head_dtype_dict = self.__sql_db.get_head_dtype_dict() tmp_set = set(key for key in tmp_dict.keys()) - set( key for key in tmp_head_dtype_dict.keys()) - set( key for key in func_dict.keys()) self.__sql_db.insert_head_dtypes(list(tmp_set)) tmp_head_dtype_dict = self.__sql_db.get_head_dtype_dict() if not func_dict: self.__functions.update(func_dict) for key in list(tmp_dict.keys()) + list(func_dict.keys()): if "char" in tmp_head_dtype_dict[key]: tmp_dict[key] = self.__functions["str"](tmp_dict[key]) else: if key in list(tmp_head_dtype_dict.keys()): tmp_dict[key] = self.__functions[tmp_head_dtype_dict[key]]( tmp_dict[key]) else: tmp_dict.update(self.__functions[key](tmp_dict[key])) del tmp_dict[key] return tmp_dict
import dict #main program word = input("Enter word for meaning: ") print(dict.dict(word))
# print('Type two attribs and then type enter key.') print("Please input the first word:") list.append(input()) print("Please intpu the second word") list.append(input()) else: list[0]=argv[0] list[1]=argv[1] print('[searching on website]: '+list[0]+' '+list[1]) a = list[0] b = list[1] webAdd = 'https://en.wikipedia.org/wiki/' webPost = '' # webAdd='https://cn.bing.com/search?q=' # webPost='&qs=n&form=QBLH&sp=-1&pq=furude&sc=0-6&sk=&cvid=A07B1D0226E14E5D8585A15510974222' obj = dict.dict() lis = obj.data() wa = r.get(webAdd+a+webPost).text wb = r.get(webAdd+b+webPost).text print('[pairing infomation]') # fil = re.compile('<[^>]*>') # wa = fil.sub('', wa) # wb = fil.sub('', wb) # wa=webGet().filter_tags(wa) # wb=webGet().filter_tags(wb) wa=webGet.webGet().filter_tags(wa) wb=webGet.webGet().filter_tags(wb) na = [] nb = [] for i in lis:
def privmsg(self, user, channel, msg): # What happens when bot gets a message self.count = self.count + 1 swag = user.split('!') user = user.split('!', 1)[0] # Turn user!ident@host into nickname print("%s:<%s> %s" % (channel, user, msg)) # Show the message in console capsIntact = msg if channel == self.nickname: # Check if it's a private message pm = 1 else: pm = 0 # append(channel, user, msg) msg = msg.lower() if swag[1].lower() in self.opersh or channel.lower() == "#dino": # if the user is in the oper list admin = 1 else: admin = 0 if channel == "#thegrammarnaziboners": tstchan = 1 else: tstchan = 0 insult_ = True fibo_ = True choose_ = True dict_ = True wiki_ = True tz_ = True wz_ = True rule_ = True convert_ = True flist = {'insult': insult_, 'fibo': fibo_, 'choose': choose_, 'dict': dict_, 'wiki': wiki_, 'wz': wz_, 'tz': tz_, 'rule': rule_, 'convert': convert_} if msg.startswith("~die"): try: if admin == 1: if user.lower() in self.azi: print "Killed by %s" % user reactor.stop() else: self.notice(user, "Not amused") print("%s attempted to kill me!") except: self.notice(user, "You are not authorized to perform this command.") print("%s attempted to kill me!" % user) elif msg.startswith("~set ") and swag.lower() == "*****@*****.**": # self.sendLine("MODE %s +B %s" % (self.nickname, self.nickname)) self.mode(self.nickname, True, 'B', limit=None, user=self.nickname, mask=None) elif msg.startswith("~stop") and admin == 1: self.quit() elif msg.startswith("~id") and (admin == 1 or user.lower == "Kojiro"): self.msg("Nickserv", "identify hamsterbot %s" % self.password) elif msg.startswith("~ghost ") and admin == 1: msg = msg.split() self.msg("Nickserv", "ghost %s swagonball" % msg[1]) elif msg.startswith("~donut") and admin == 1: msg = msg.split(' ') try: if msg[1].isdigit(): #hopefully if i use a number it'll join that chan i = int(msg[1]) chan = self.chanlist[i] self.join(chan) #why doesnt this work nvm it does elif msg[1] == "joinall" or msg[1] == "all": for item in self.chanlist: self.join(item) else: self.join(msg[1]) except: if admin != 1: self.notice(user, "You are not authorized to perform this command.") else: self.say(channel, "-_-") elif msg.startswith("~bye"): try: if admin == 1: self.part(channel) except: self.notice(user, "You are not authorized to perform this command.") print("%s attempted to use command ~bye." % user) elif msg.startswith("~nick"): try: if admin == 1: msg = msg.split(' ') self.setNick(msg[1]) except: self.notice(user, "You are not authorized to perform this command.") print("%s tried to change my nick" % user) elif msg.startswith("~toggle ") and admin == 1: msg = msg.split() if len(msg) == 3: if flist.has_key(msg[1]): funct = msg[1].strip() if msg[2].strip() == "on" or msg[2] == '1': flist[funct] = True elif msg[2].strip() == 'off' or msg[2] == '0': flist[funct] = False elif msg[2] == 'not': flist[funct] = not flist[funct] else: self.notice(user, "Values accepted: on/off or 1/0") else: self.notice(user, "No function found with that name or function may not be switched on/off") elif len(msg) == 2: if msg[1] == 'help': self.notice(user, "The following functions may be switched on/off: insult, choose, convert, dict, fibo, rule, tz, wz, wiki") elif msg[1] == 'is_on': is_on = [] for key, value in flist.iteritems(): if value == True: is_on.append(key) self.notice(user, "The following functions are on: %r" % is_on) else: self.notice(user, "Toggle takes 2 arguments. Do ~toggle help for functions that may be toggled on/off") elif msg.startswith("~say ") and admin == 1: msg = msg[5:] self.say(channel, msg) elif msg.startswith("~note ") and (swag[1].lower()=="*****@*****.**"): msg = msg[6:] # bs.notes(str(msg)) self.notice(user, notes(str(msg))) elif msg.startswith(":prefixes "): msg = msg[10:] self.say(channel, whattodo(msg, user)) elif msg.startswith(".insult ") and insult_ == True: msg = msg.split(' ') #nick = msg[1] try: if msg[1].lower() in self.azi: self.say(channel, random.choice(azi_insults)) elif msg[1].lower() == "?roulette": self.say(channel, "%s, %s" % (user, random.choice(insults))) else: insult = "%s, %s" % (msg[1], random.choice(insults)) self.say(channel, insult) except IndexError: self.say(channel, random.choice(insults)) elif msg.startswith(".inslut") and flist['insult'] == True: msg = msg.split(' ') insult = "%s, %s" % (user, random.choice(insults)) self.say(channel, insult) elif msg.startswith(".halp") or msg.startswith(".help"): try: if admin == 1: self.notice(user, "Commands: .help, .insult <user>, .fibo, .convert <conversion> <numbers>, .gay, .rule <1-47>, .choose <option1, option2,...>, .dict <query>") self.notice(user, "Restricted commands: ~die, ~bye, ~donut <channel>, ~nick <newnick>") except: self.notice(user, "Commands: .help, .about, .insult <user>, .fibo, .convert <conversion> <numbers>") # elif msg.startswith("~~"): # msg = msg.strip("~~") # exec(msg) elif msg.startswith(".choose ") and flist['choose'] == True: msg = msg[8:] # re.sub("or*", '', msg) msg = msg.split(",") msg = random.choice(msg) words = msg.split(' ') if words[0] == 'or' and len(words) > 1: msg = msg.replace('or ','') self.msg(channel, "%s: %s" % (user, msg.strip())) # elif msg.startswith(".last"): # msg = msg[5:] # list = last(channel, msg) # for item in list: # self.msg(user, item) elif msg.startswith(".wiki ") and flist['wiki'] == True: #broken wikipedia feature msg = msg[6:] msg = msg.replace(' ', '_') self.msg(channel, str(wiki(str(msg)))) elif msg.startswith(".dict ") and flist['dict'] == True: #dictionary feature # msg = capsIntact msg = msg[6:] self.msg(channel, dict(msg)) elif msg.startswith(".tz ") and flist['tz'] == True: #timezone feature msg = msg[4:] self.msg(channel, tz(msg)) elif msg.startswith(".wz ") and flist['wz'] == True: #weatherbot feature msg = msg[4:] self.msg(channel, wz(msg)) elif msg.startswith(".gay"): self.msg(channel, "%s: http://blog.okcupid.com/index.php/gay-sex-vs-straight-sex" % user) elif msg.startswith(".rule ") and flist['rule'] == True: msg = msg.split() self.msg(channel, rules((msg[1].strip()))) elif msg.startswith(".fibo") and flist['fibo'] == True: msg = msg.split(' ') num = msg[1] i = fib2(int(num)) self.msg(channel, str(i)) elif msg.startswith(".convert") and flist['convert'] == True: msg = msg.split(' ') if convert(msg).startswith('S') or convert(msg).startswith('N'): if '::' in convert(msg): resp = convert(msg).split('::') self.notice(user, resp[0]) self.notice(user, resp[1]) else: self.notice(user, convert(msg)) elif pm == 1: self.notice(user, convert(msg)) else: self.msg(channel, convert(msg))
for s in self.cmap ] self.vowel_pat = re.compile("[aeiouy]*") self.vowel_pat_plus = re.compile("[aeiouy]+") self.d = dictionary def set_dictionary(self, d): self.d = d # A sequence of numbers each 0-15 def pat_for(self, *n): pats = [ self.vowel_pat ] for num in n: pats += [ self.pats[num], self.vowel_pat_plus ] pats.pop() pats.append(self.vowel_pat) pat = re.compile("^" + "".join([p.pattern for p in pats]) + "$") print(pat.pattern) return pat def words_for(self, *n): pat = self.pat_for(*n) words = self.d.select(pat) yield from [ (w, len(n)) for w in words ] if __name__ == '__main__': s = selector(dictionary=dict("/usr/share/dict/words")) w = s.words_for(0b1111, 0b1010, 0b0101) # w = s.words_for(0b1111, 0b1010) for str in w: print(str)
def crawl_site(self, queue_table_name: str, scanned_url_table=None, func=None, length=1000, sleep_time=0.3) -> None: tmp_command: str = "select * from {0} limit {1}".format( queue_table_name, str(length)) if self.__sql_db is None: print("No sql_db exists") else: tmp_df = self.__sql_db.execute(tmp_command) if len(tmp_df) != 0: self.__sql_db.execute( "lock tables {} write;".format(queue_table_name)) tmp_command = "select * from {0} limit {1}".format( queue_table_name, length) tmp_df = self.__sql_db.execute(tmp_command) tmp_command = "delete from {0} limit {1}".format( queue_table_name, str(length)) self.__sql_db.execute(tmp_command) self.__sql_db.execute("unlock tables;") tmp_df = tmp_df.to_numpy().tolist() tmp_heads_list, tmp_heads_dtype = self.__sql_db.get_heads_dtype( queue_table_name) tmp_sqllike = PsuedoSQLFromCSV("") tmp_sqllike.dtype = dict(table_name='str', url='str', created='datetime', dict='str') tmp_sqllike.header = ('table_name', 'url', 'created', 'dict') tmp_sqllike.data = list() if scanned_url_table: tmp_sqllike_urls = PsuedoSQLFromCSV("") tmp_sqllike_urls.dtype = dict(url='str') tmp_sqllike_urls.header = ['url'] tmp_sqllike_urls.data = list() for data_line in tqdm(tmp_df): tmp_table_list = self.__sql_db.get_tables() if data_line[1] not in tmp_table_list: tmp_command = "create table " + data_line[ 1] + "(table_name varchar(50), url varchar(120), created datetime, dict " \ "mediumtext); " print(tmp_command) self.__sql_db.execute(tmp_command) tmp_order_list = list() tmp_get_dict = dict() tmp_click_dict = dict() tmp_insert_dict = dict() tmp_select_dict = dict() tmp_xpath_dict = dict() tmp_class_name_dict = dict() tmp_datetime = datetime.now() tmp_table_name = data_line[tmp_heads_list.index( "table_name")] if data_line[tmp_heads_list.index( "order_list")] is not None: tmp_order_list = json.loads( data_line[tmp_heads_list.index("order_list")]) if data_line[tmp_heads_list.index("get_dict")] is not None: tmp_get_dict = json.loads( data_line[tmp_heads_list.index("get_dict")]) if data_line[tmp_heads_list.index( "click_dict")] is not None: tmp_click_dict = json.loads( data_line[tmp_heads_list.index("click_dict")]) if data_line[tmp_heads_list.index( "insert_dict")] is not None: tmp_insert_dict = json.loads( data_line[tmp_heads_list.index("insert_dict")]) if data_line[tmp_heads_list.index( "select_dict")] is not None: tmp_select_dict = json.loads( data_line[tmp_heads_list.index("select_dict")]) if data_line[tmp_heads_list.index( "xpath_dict")] is not None: tmp_xpath_dict = json.loads( data_line[tmp_heads_list.index("xpath_dict")]) if data_line[tmp_heads_list.index( "class_dict")] is not None: tmp_class_name_dict = json.loads( data_line[tmp_heads_list.index("class_dict")]) if not self.get_2(data_line[0]): self.log_error() continue tmp_result = self.routine4selenium( order_list=tmp_order_list, get_dict=tmp_get_dict, click_dict=tmp_click_dict, insert_dict=tmp_insert_dict, select_dict=tmp_select_dict, xpath_dict=tmp_xpath_dict, class_dict=tmp_class_name_dict, sleep_time=sleep_time) if tmp_result: if not tmp_result["is_successful"]: self.log_error() continue if func: tmp_result["url"] = data_line[0] func(tmp_result) tmp_sqllike.data.append(list(tmp_result.values())) else: tmp_result = json.dumps(tmp_result) tmp_sqllike.data.append([ tmp_table_name, data_line[0], tmp_datetime, tmp_result ]) if scanned_url_table: tmp_sqllike_urls.data.append(data_line[0]) try: if func: tmp_sqllike.header = list(tmp_result.keys()) self.__sql_db.insert_data(data_line[1], tmp_sqllike) except Exception: tmp_sqllike.save_data_to_csv("test.csv") if scanned_url_table: self.__sql_db.insert_data(scanned_url_table, tmp_sqllike_urls)
data = build_data(f) #Load data raster print "Loading Data Raster..." #Load data raster raster_list = glob.glob(path + '*.TIF') dataRaster = [] for i in raster_list: band = gdal.Open(i) dataRaster.append(band.GetRasterBand(1).ReadAsArray().astype(float)) filename = [] for a in [os.path.basename(x) for x in glob.glob(path + '*.TIF')]: p = os.path.splitext(a)[0] filename.append(p) my_dict = dict(zip(filename, dataRaster)) #Load data raster aspect & slope pathname = 'Folder name' # folder consist of aspect and slope data raster_list_dem = glob.glob(pathname + '/*.TIF') dataTopo = [] for d in raster_list_dem: band2 = gdal.Open(d) dataTopo.append(band2.GetRasterBand(1).ReadAsArray()) #Load data raster sample area pathsample = 'Folder name' # consist of sample area based on visual digitization sampledata = gdal.Open(pathsample) sample = np.array(sampledata.GetRasterBand(1).ReadAsArray())