def test_load_dump(self): g = Grab(transport=GRAB_TRANSPORT) cookies = {'foo': 'bar', 'spam': 'ham'} g.setup(cookies=cookies) g.go(SERVER.BASE_URL) g.dump_cookies(TMP_FILE) self.assertEqual(set(cookies.items()), set((x['name'], x['value']) for x in json.load(open(TMP_FILE)))) # Test non-ascii g = Grab(transport=GRAB_TRANSPORT) cookies = {'foo': 'bar', 'spam': u'бегемот'} g.setup(cookies=cookies) g.go(SERVER.BASE_URL) g.dump_cookies(TMP_FILE) self.assertEqual(set(cookies.items()), set((x['name'], x['value']) for x in json.load(open(TMP_FILE)))) # Test load cookies g = Grab(transport=GRAB_TRANSPORT) cookies = [{'name': 'foo', 'value': 'bar'}, {'name': 'spam', 'value': u'бегемот'}] json.dump(cookies, open(TMP_FILE, 'w')) g.load_cookies(TMP_FILE) self.assertEqual(set(g.cookies.items()), set((x['name'], x['value']) for x in cookies))
def test_load_dump(self): g = Grab(transport=GRAB_TRANSPORT) cookies = {'foo': 'bar', 'spam': 'ham'} g.setup(cookies=cookies) g.dump_cookies(TMP_FILE) self.assertEqual(set(cookies.items()), set(json.load(open(TMP_FILE)).items())) # Test non-ascii g = Grab(transport=GRAB_TRANSPORT) cookies = {'foo': 'bar', 'spam': u'бегемот'} g.setup(cookies=cookies) g.dump_cookies(TMP_FILE) self.assertEqual(set(cookies.items()), set(json.load(open(TMP_FILE)).items())) # Test load cookies g = Grab(transport=GRAB_TRANSPORT) cookies = {'foo': 'bar', 'spam': u'бегемот'} json.dump(cookies, open(TMP_FILE, 'w')) g.load_cookies(TMP_FILE) self.assertEqual(set(g.config['cookies'].items()), set(cookies.items()))
class SpiderGUI(object): def __init__(self): self.sawe_search = 1 self.sawe_country = '' self.sawe_state = '' self.sent = 0 self.total = 0 self.time = 0 self.minChars = 200 self.run = False self._run = False self.cookies_file = 'cookies.json' letter = ur'letter.txt' self.table_name = 'default' self.engine = create_engine('sqlite:///database.db', echo=False) Session = sessionmaker(bind=self.engine) Session.configure(bind=self.engine) self.metadata = MetaData() self.session = Session() self.table = Table(self.table_name, self.metadata, Column('id', Integer, primary_key=True), Column('name', String), Column('fullname', String), Column('userid', String), Column('key', String)) self.metadata.create_all(self.engine) mapper(User, self.table) self.g = Grab() self.tk = Tk() self.note = ttk.Notebook(self.tk) self.tk.geometry('650x306') #self.tk.iconbitmap(default='icon.ico') self.tk.title("Spyder search engine") ### ВЕРХНЯЯ ПАНЕЛЬ ### self.Frame = Frame(self.tk, width=460, height=62) self.Frame.pack(side='top', fill='x') # строка статуса рассылки self.lb = ttk.Label(self.Frame, text='Welcome to the afa sender ...') self.lb.place(x=3, y=3) self.lb8 = ttk.Label(self.Frame, text='00:00:00', font="Arial 25 bold") self.lb8.place(x=25, y=20) # строка прогресс бара self.lb1 = ttk.Label(self.Frame, text='waiting ...') self.lb1.place(x=400, y=0, width=250) # прогресс бар self.Bar = ttk.Progressbar(self.Frame, orient=HORIZONTAL, mode='determinate', length=250) self.Bar.place(x=400, y=17) # кнопка старт/стоп self.runBtn = ttk.Button(self.Frame, text='Start', command=self.spider_run) self.runBtn.place(x=495, y=38) # кнопка выход self.quitBtn = ttk.Button(self.Frame, text='Quit', command=self.spider_quit) self.quitBtn.place(x=575, y=38) # вкладки self.tab1 = Frame(self.note) self.tab3 = Frame(self.note) self.note.add(self.tab1, text="Main") self.note.add(self.tab3, text="Letter") self.note.pack(fill="both", expand="yes", side='top') # testMode = True self.mode = BooleanVar() self.modeBtn = ttk.Checkbutton(self.tab1, text="Test mode", variable=self.mode, onvalue=True, offvalue=False).place(x=3, y=3) # enable logging self.log = BooleanVar() #self.log.set(True) self.logBtn = ttk.Checkbutton(self.tab1, text='Enable logging', variable=self.log, onvalue=True, offvalue=False).place(x=3, y=23) # load_cookies self.cookies = BooleanVar() self.cookies.set(True) self.cookiesBtn = ttk.Checkbutton(self.tab1, text="load cookies", variable=self.cookies, command=self.lform, onvalue=True, offvalue=False).place(x=3, y=43) # fromAge lb3 = ttk.Label(self.tab1, text="Age from search").place(x=3, y=75) self.from_age = IntVar() self.from_age.set(18) self.from_ageBox = ttk.Combobox(self.tab1, width=2, textvariable=self.from_age) self.from_ageBox['values'] = range(18, 100) self.from_ageBox.place(x=150, y=75) # toAge lb4 = ttk.Label(self.tab1, text="to").place(x=200, y=75) self.to_age = IntVar() self.to_age.set(99) self.to_ageBox = ttk.Combobox(self.tab1, width=2, textvariable=self.to_age) self.to_ageBox['values'] = range(18, 100) self.to_ageBox.place(x=225, y=75) # del database self.delbaseBtn = ttk.Button(self.tab1, text='del database', command=self.delbase).place(x=450, y=175) # state self.lb7 = ttk.Label(self.tab1, text="State for search") self.state = StringVar() self.stateBox = ttk.Combobox(self.tab1, width=25, textvariable=self.state) self.lb7.place(x=3, y=125) self.state.set('__all__') self.stateBox['values'] = usa_state_list self.stateBox.place(x=153, y=125) # country lb5 = ttk.Label(self.tab1, text="Country for search").place(x=3, y=100) self.country = StringVar() self.country.set('United States') self.countryBox = ttk.Combobox(self.tab1, width=25, textvariable=self.country) self.countryBox['values'] = country_list self.countryBox.bind('<<ComboboxSelected>>', self.set_state) self.countryBox.place(x=153, y=100) # run search self.searchBtn = ttk.Button(self.tab1, text='run search', command=self.search_run) self.searchBtn.place(x=550, y=175) # text box txtpanelFrame = Frame(self.tab3, bg='white') txtpanelFrame.pack(side='bottom', fill='x') self.textbox = Text(self.tab3, font='Verdana 10', wrap='word') self.textbox.insert( '1.0', 'Message header must contain the variable "$name" instead of the recipient!' ) scrollbar = Scrollbar(self.tab3) scrollbar['command'] = self.textbox.yview self.textbox['yscrollcommand'] = scrollbar.set self.textbox.pack(side='left', fill='both', expand=1) scrollbar.pack(side='right', fill='y') tloadBtn = ttk.Button(txtpanelFrame, text='Load', command=self.load_letter) tloadBtn.pack(side='right') tloadBtn = ttk.Button(txtpanelFrame, text='Save', command=self.save_letter) tloadBtn.pack(side='right') tloadBtn = ttk.Button(txtpanelFrame, text='Apply', command=self.apply_letter) tloadBtn.pack(side='right') # lady id lb9 = ttk.Label(self.tab1, text='ladyId:').place(x=450, y=3) self.ladyId = ttk.Entry(self.tab1) #self.ladyId.insert(0,'100820') self.ladyId.place(x=500, y=3) ### ПАНЕЛЬ СТАТУСА ### self.lb6 = ttk.Label(self.tab1, text="Waiting for user activity ...") self.lb6.pack(side='bottom', fill='x') def ladyId_set(self): if self.ladyId.get() != '': return self.ladyId.get() else: print 'ENTER LADY ID, EMPTY !!!' self.run = False self._run = False def delbase(self): try: print 'Deleting database ...' self.lb6["text"] = 'Deleting database ...' c = self.session.query(User).count() if c != 0: self.table.drop(self.engine) self.session.commit() self.metadata.create_all(self.engine) print 'OK ...' else: print 'Database is empty...' self.session.commit() except Exception: print 'Database acess error' def set_state(self, *args): print self.country.get() if self.country.get() == 'United States': self.lb7.place(x=3, y=125) self.state.set('__all__') self.stateBox['values'] = usa_state_list self.stateBox.place(x=153, y=125) else: self.lb7.place_forget() self.stateBox.place_forget() def country_set(self): return country_dict[self.country.get()] def state_set(self): if self.country.get( ) == 'United States' and self.state.get() != '__all__': string = '--state-' + str(usa_state_dict[self.state.get()]) return string else: return '' def start(self): self.tk.mainloop() def test(self): try: self.g.go('http://office.loveme.com') except Exception: print >> sys.__stderr__, '** Could not connect to the network **\n** Check you internet connection **' sys.exit(1) self.ladyId_set() tmp = self.textbox.get('1.0', 'end') if len(tmp) < self.minChars: self.run = False print 'LOAD A VALID LETTER !!!' print '****' * 7 def spider_run(self): if self.run == False: self.run = True self.test() self.runBtn["text"] = "Stop" if self.run: threading.Thread(target=self.spider).start() else: self.runBtn["text"] = "Start" else: print '\nKilling spider ...' self.lb6["text"] = 'Killing spider ...' self.runBtn["text"] = "Start" self.run = False self.sent = 0 def spider_quit(self): print('Quit (%d) ' % self.mode.get()) self.tk.destroy() sys.exit(1) def check_base(self): try: c = self.session.query(User).filter_by(key='0').count() if c != 0: self.total = c print 'Database contains %s records ...' % self.total else: print '** Database is empty, Run search !!! **' self.lb["text"] = 'Database is empty, run search !!!' self.run = False self.session.commit() except Exception: print 'Database acess error' def spider(self): self.set_mode() self.set_log() self.set_timeout() self.load_cookies() self.check_base() runtime = 0 _iter = 0 self.letter = self.textbox.get('1.0', 'end') mes = self.letter.split('\n') s = mes[0] if s.find('$name') != -1: self._subject = s else: self._subject = default_subject print 'Please enter a valid email subject!' self.lb6["text"] = 'Please enter a valid email subject!' del mes[0] self.body = '\n'.join(mes) for instance in self.session.query(User).filter_by(key='0'): if self.run: _tmp = time.clock() _iter += 1 self.name = instance.name self.menId = instance.userid self.key = instance.key s = Template(self._subject) self.subject = s.substitute(name=self.name) try: self.send_letter() except Exception: print 'Sender function crashed, try again, maby ...' sys.exit(1) instance.key = True self.session.commit() run = time.clock() - _tmp runtime += run fulltime = (runtime / _iter) * self.total remaining = (fulltime - runtime) self.Bar["maximum"] = fulltime self.Bar["value"] = runtime self.lb[ "text"] = 'Sent letters to %s of %s men %s men missing' % ( _iter, self.total, (_iter - self.sent)) self.lb8["text"] = self.seconds_to(remaining) self.lb1["text"] = 'Need %s / Last %s / Run %s' % ( self.seconds_to(fulltime), self.seconds_to(remaining), self.seconds_to(runtime)) print 'Time need %s, remaining %s, process run %s cpu %s' % ( self.seconds_to(fulltime), self.seconds_to(remaining), self.seconds_to(runtime), self.seconds_to(run)) else: print ' *** ' * 16 print 'Session aborted, exit ...' break print '\nSending letters completed ...' print ' *** ' * 16 self.lb6["text"] = 'Sending letters completed ...' self.runBtn["text"] = "Start" self.lb8["text"] = self.seconds_to(0) self.run = False self.sent = 0 self.session.commit() def self_basetes(self): for instance in self.session.query(User).all(): print instance.userid, repr(instance.name), instance.key def seconds_to(self, seconds): return time.strftime('%H:%M:%S', time.gmtime(seconds)) def send_letter(self): self.session.commit() print '\n================================ RESTART ================================\n' self.lb6["text"] = '** Sending letter to %s %s ... **' % (repr( self.name), self.menId) uri = 'http://office.loveme.com/send?mid=' + str( self.menId ) + '&wid=' + str( self.ladyId_set() ) + '&_RETURN=http%3A%2F%2Foffice.loveme.com%2Fsearch_men_results%3Fq%3D' + str( self.from_age.get()) + '--age_to-' + str( self.to_age.get()) + '--country-' + str( self.country_set()) + '%26women_id%3D' + str( self.ladyId_set()) self.g.go(uri) if self.g.search(u'User does not want to receive intro letters!' ) or self.g.search( u'Already sent an intro letter to this man.'): print('** BAD user %s id %s **' % (repr(self.name), self.menId)) self.lb6["text"] = '** BAD user %s id %s **' % (repr( self.name), self.menId) return False self.g.set_input('mbox_subject', self.subject) self.g.set_input('mbox_body', self.body) if self.mode.get(): print '** Sending letter to %s %s (TEST MODE) ... **' % (repr( self.name), self.menId) self.lb6[ "text"] = '** Sending letter to %s %s (TEST MODE) ... **' % ( repr(self.name), self.menId) self.sent += 1 else: self.g.submit() print '** Sending letter to %s %s ... **' % (repr( self.name), self.menId) self.lb6["text"] = '** Sending letter to %s %s ... **' % (repr( self.name), self.menId) self.sent += 1 def load_cookies(self): cookies = self.cookies_file try: self.g.load_cookies(cookies) except Exception: print '** Could not load cookie **\n** Click login check botton**' self.lb["text"] = 'Need login !!!' sys.exit(1) print('Loading cookie from file (%s) ...' % cookies) self.lb6["text"] = 'Loading cookie from file (%s) ...' % cookies def lform(self): self.form = Tk() self.form.geometry('200x100') self.form.title("Login") label = ttk.Label(self.form, text='login:'******'Arial 10 bold').place(x=0, y=5) label2 = ttk.Label(self.form, text='pass:'******'Arial 10 bold').place(x=0, y=25) self.entry = ttk.Entry(self.form) self.entry2 = ttk.Entry(self.form, show='*') self.entry.insert(0, 'Luga8') self.entry2.insert(0, 'jNGRtxr5') self.entry.place(x=50, y=5) self.entry2.place(x=50, y=30) Btn = ttk.Button(self.form, text='login', command=self.login).place(x=75, y=65) self.form.mainloop() def login(self): print 'Login ...' print('current value is %s' % self.entry.get()) print('current value is %s' % self.entry2.get()) self.g.go('http://office.loveme.com/assign_women') self.g.set_input('logins_ident', self.entry.get()) self.g.set_input('logins_password', self.entry2.get()) self.g.set_input('remember_login', '1') self.g.submit() cookies = self.cookies_file self.g.dump_cookies(cookies) self.cookies.set(True) self.form.destroy() def set_timeout(self): print 'Setting timeout to magic random random choice ...' self.g.setup(hammer_mode=True, hammer_timeouts=((2, 5), (10, 15), (20, 30), (60, 100))) def set_log(self): if self.log.get() == True: print 'Loging enabled ...' self.lb6["text"] = 'Loging enabled ...' logger = logging.getLogger('grab') logger.addHandler(logging.StreamHandler()) logger.setLevel(logging.DEBUG) self.g.setup(log_dir='log') self.g.setup(debug_post='True') self.log.set(False) try: os.mkdir('log') except Exception: shutil.rmtree('log') os.mkdir('log') print 'Dir "log" already exsit or maby ome error ...' def set_mode(self): if self.mode.get(): print 'Testing mode enabled ...' self.lb6["text"] = 'Testing mode enabled ...' def search_run(self): if self._run == False: self._run = True self.test() self.searchBtn["text"] = "Stop" if self._run: threading.Thread(target=self.search).start() else: self.searchBtn["text"] = "run search" else: self._run = False def set_age(self): if self.from_age.get() <= self.to_age.get(): print('Age from search set to %d ...' % self.from_age.get()) print('Age to search set to %d ...' % self.to_age.get()) self.lb6[ "text"] = 'Age from search set to %d ...' % self.from_age.get( ) self.lb6["text"] = 'Age to search set to %d ...' % self.to_age.get( ) else: print 'Specify the correct settings for search !!!' self.from_age.set(18) def search_restore(self): print 'RESTORE SEARCH (page (%s), country (%s), state (%s))' % ( self.sawe_search, self.sawe_country, self.sawe_state) if self.sawe_search != 1 and self.country_set() == self.sawe_country: if self.state_set() == self.sawe_state: print 'RESTORE SEARCH OK, page (%s) state_set (%s) sawe_state (%s)' % ( self.sawe_search, self.state_set(), self.sawe_state) else: print 'RESTORE SEARCH (New search begin ...)' self.sawe_search = 1 else: print 'RESTORE SEARCH (New search begin ...)' self.sawe_search = 1 def search_backup(self, page): if self.sawe_search == page: print 'SEARCH BACKUP (Search is done, current page (%s))' % self.sawe_search self.sawe_search = 1 self.sawe_country = '' self.sawe_state = '' else: print 'SEARCH BACKUP (Sawe search result, sawe_search != page (%s))' % self.sawe_search self.sawe_country = self.country_set() self.sawe_state = self.state_set() print 'SEARCH BACKUP (sawe_country (%s), sawe_state (%s))' % ( self.sawe_country, self.sawe_state) self.session.commit() def search(self): print '\n================================ RESTART ================================\n' print 'Running search... ' self.lb6["text"] = "Running search... " self.lb["text"] = "Running search... " self.set_log() self.set_timeout() self.load_cookies() self.set_age() runtime = 0 _iter = 0 uri = 'http://office.loveme.com/search_men_results~pg1?q=age_from-' + str( self.from_age.get()) + '--age_to-' + str(self.to_age.get( )) + '--country-' + str(self.country_set()) + str( self.state_set()) + '&women_id=' + str(self.ladyId_set()) self.g.go(uri) try: q = self.g.doc.select('//div[@class="f_left"]/b') found = q[2].text() found = int(found) page = (found / 20) + 1 except Exception: print 'Exception in parse page total' page = 1 self.search_restore() for x in range(self.sawe_search, page + 1): if self._run: xtmp = x _tmp = time.clock() _iter += 1 try: self.men_search(x) except Exception: print 'Search function crashed, try again NEWER!!! ...' sys.exit(1) run = time.clock() - _tmp runtime += run fulltime = (runtime / _iter) * (page - self.sawe_search) remaining = (fulltime - runtime) self.Bar["maximum"] = fulltime self.Bar["value"] = runtime self.lb["text"] = 'Parsing page ' + str( self.sawe_search + _iter - 1) + ' from ' + str(page) + ' ...' self.lb8["text"] = self.seconds_to(remaining) self.lb1["text"] = 'Need %s / Last %s / Run %s' % ( self.seconds_to(fulltime), self.seconds_to(remaining), self.seconds_to(runtime)) print 'Time need %s, remaining %s, process run %s' % ( self.seconds_to(fulltime), self.seconds_to(remaining), self.seconds_to(runtime)) else: print "\nStop searching ..." self.sawe_search = x break self.sawe_search = xtmp print ' *** ' * 16 self.lb6["text"] = 'Search over, found (%d) men ...' % self.total self.lb["text"] = 'Search over, found (%d) men ...' % self.total self.searchBtn["text"] = "Search" self.lb8["text"] = self.seconds_to(0) self.lb1["text"] = 'Need %s / Last %s / Run %s' % ( self.seconds_to(0), self.seconds_to(0), self.seconds_to(0)) self._run = False self.search_backup(page) def men_search(self, x): uri = 'http://office.loveme.com/search_men_results~pg' + str( x) + '?q=age_from-' + str(self.from_age.get()) + '--age_to-' + str( self.to_age.get()) + '--country-' + str( self.country_set()) + str( self.state_set()) + '&women_id=' + str( self.ladyId_set()) self.g.go(uri) if self.g.search( u'Your search returned no results. Please try again using different criteria' ): print('** Search over, found (%d) men **' % self.total) self.session.commit() self._run = False self.searchBtn["text"] = "Search" else: h = self.g.doc.select('//p[@class="bold"]/a') h = filter(lambda x: x != '', h) self.total += len(h) - 1 for x in range(1, len(h)): ur = h[x].attr('href') self.men_id = re.sub('[\D]', '', ur) self.name = h[x].attr('title').split(' ')[0] self.key = False self.session.add(User(self.name, self.men_id, self.key)) print('Adding a user %s (%s) to the database ...' % (repr(self.name), self.men_id)) self.lb6[ "text"] = 'Adding a user %s (%s) to the database ...' % ( repr(self.name), self.men_id) self.session.commit() def load_letter(self): fn = tkFileDialog.Open(self.tk, filetypes=[('*.txt files', '.txt')]).show() if fn == '': return self.textbox.delete('1.0', 'end') self.textbox.insert('1.0', open(fn, 'rt').read()) self.lb6["text"] = "Loading letter complete... " def save_letter(self): fn = tkFileDialog.SaveAs(self.tk, filetypes=[('*.txt files', '.txt')]).show() if fn == '': return if not fn.endswith(".txt"): fn += ".txt" open(fn, 'wt').write(self.textbox.get('1.0', 'end')) self.lb6["text"] = "Saving letter complete... " def apply_letter(self): print "Apply changes ..." self.letter = self.textbox.get('1.0', 'end') print self.letter self.lb6["text"] = "Applying changes complete... "
def main(): global _run _run = True count = 0 url = 'http://www.natashaclub.com/' url_s = 'https://www.natashaclub.com/search_result.php?p_per_page=1000&photos_only=on&online_only=on&Sex=female&LookingFor=male&DateOfBirth_start=18&DateOfBirth_end=75&Region[]=3&Region[]=4&Region[]=7&Country[]=179&CityST=0&City=&&page=' g = Grab() g.setup(hammer_mode=True, hammer_timeouts=((20, 30), (60, 90), (150, 200))) g.setup(follow_refresh=True) if log.get() == True: print 'makin log dir' lb["text"] = 'makin log dir' try: os.mkdir('log') g.setup(log_dir='log') logger = logging.getLogger('grab') logger.addHandler(logging.StreamHandler()) logger.setLevel(logging.DEBUG) g.setup(debug_post='True') except Exception: shutil.rmtree('log') os.mkdir('log') g.setup(log_dir='log') #sys.exit('[!] unable to delete log directory') g.go(url) g.set_input('ID', ladyId.get()) g.set_input('Password', ladyPas.get()) g.set_input('rememberme', '1') g.submit() g.dump_cookies('cookies') g.go(url) if g.search(u'Member Menu'): print 'Authorization complete ...' lb["text"] = 'Authorization complete ...' else: print "\033[31m\t[!]Invalid login\033[0m" lb['text'] = '[!]Invalid login' _run = False sys.exit() g.go(url_s) list = g.doc.select('//a[contains(@href, "&&page=")]/@href')[13].text() page = re.findall('(\d+)', list)[8] page = int(page) for i in range(page + 1): if _run: g.go(url_s + repr(i)) list2 = g.doc.select( '//a[contains(@href, "vkiss.php?sendto=")]/@href').text_list() for item in list2: if _run: try: g.go(url + item) if g.search(u'Virtual smile NOT sent'): print 'Virtual smile NOT sent' lb["text"] = 'Virtual smile NOT sent' if g.search( u"Sorry, but you've reached your limit for today." ): print "\033[31m\t[+]Sorry, but you've reached your limit for today.\033[0m" lb['text'] = "Sorry, but you've reached your limit for today." _run = False sys.exit() else: text = '\033[32m[+] Виртуальная улыбка была отправлена : ', re.findall( '(\d+)', item), '\033[0m' print text lb["text"] = text count() except Exception: print _run