Esempio n. 1
0
 def __init__(self, url, keyword, casperJSScript):
     self.url = url
     self.keyword = keyword
     self.casperJSScript = casperJSScript
     self.urls = []
     rua = Random_user_agent()
     self.ua = rua.rand()
     self.time = rua.randsleep()
     self.result = []
 def __init__(self,url,casperJSScript='pastebintext.js'):
     threading.Thread.__init__(self)
     self.url=url
     self.casperJSScript=casperJSScript
     self.content=Content()
     self.data=[]
     rua=Random_user_agent()
     self.ua=rua.rand()
     self.result=None
Esempio n. 3
0
 def __init__(self, url, casperJSScript='pastebintext.js'):
     threading.Thread.__init__(self)
     self.url = url
     self.casperJSScript = casperJSScript
     self.content = Content()
     self.data = []
     rua = Random_user_agent()
     self.ua = rua.rand()
     self.result = None
Esempio n. 4
0
 def __init__(self, url, keyword, casperJSScript):
     self.url = url
     self.keyword = keyword
     self.casperJSScript = casperJSScript
     self.urls = []
     rua = Random_user_agent()
     self.ua = rua.rand()
     self.time = rua.randsleep()
     self.result = []
Esempio n. 5
0
 def __init__(self,limit,criteria,scriptjs,db,url_pattern='((https?|ftp|gopher|telnet|file|notes|ms-help):((//)|(\\\\))+([\w\d:#@%/;$()~_?\+-=\\\.&])*)'):
     threading.Thread.__init__(self)
     self.result=[]
     self.limit=limit    
     self.criteria=criteria
     self.scriptjs=scriptjs
     self.connection=Connection('localhost',27017)
     self.db=self.connection[db]
     self.whitelist=white_list(db)
     self.regex_url=re.compile(url_pattern)
     rua=Random_user_agent()
     self.ua=rua.rand()
     self.urls_by_domaine={}
Esempio n. 6
0
 def __init__(self,limit,criteria,scriptjs,db,url_pattern='((https?|ftp|gopher|telnet|file|notes|ms-help):((//)|(\\\\))+([\w\d:#@%/;$()~_?\+-=\\\.&])*)'):
     threading.Thread.__init__(self)
     self.result=[]
     self.limit=limit
     self.criteria=criteria
     self.scriptjs=scriptjs
     self.connection= MongoClient(host='localhost', port=27017)
     self.db=self.connection[db]
     self.whitelist=white_list(db)
     self.regex_url=re.compile(url_pattern)
     rua=Random_user_agent()
     self.ua=rua.rand()
     self.urls_by_domaine={}