def siteAnnounce(self, address): from Site.Site import Site from Site import SiteManager SiteManager.site_manager.load() logging.info("Opening a simple connection server") global file_server from File import FileServer file_server = FileServer("127.0.0.1", 1234) file_server.start() logging.info("Announcing site %s to tracker..." % address) site = Site(address) s = time.time() site.announce() print "Response time: %.3fs" % (time.time() - s) print site.peers
def need(self, address, all_file=True): from Site import Site site = self.get(address) if not site: # Site not exist yet if not self.isAddress(address): return False # Not address: %s % address logging.debug("Added new site: %s" % address) site = Site(address) self.sites[address] = site if not site.settings["serving"]: # Maybe it was deleted before site.settings["serving"] = True site.saveSettings() if all_file: # Also download user files on first sync site.download(blind_includes=True) else: if all_file: site.download() return site
def load(self): from Site import Site if not self.sites: self.sites = {} address_found = [] added = 0 # Load new adresses for address in json.load(open("data/sites.json")): if address not in self.sites and os.path.isfile("data/%s/content.json" % address): self.sites[address] = Site(address) added += 1 address_found.append(address) # Remove deleted adresses for address in self.sites.keys(): if address not in address_found: del(self.sites[address]) logging.debug("Removed site: %s" % address) if added: logging.debug("SiteManager added %s sites" % added)
def siteSign(self, address, privatekey=None, passphrase=False, inner_path="content.json", publish=False, remove_missing_optional=False): from Site import Site from Site import SiteManager from Debug import Debug SiteManager.site_manager.load() address = getPublicKey(address) logging.info("Signing site: %s..." % address) site = Site(address, allow_create=False) if passphrase: import getpass passphrase = getpass.getpass( "Type your passphrase (input hidden): ") privatekey = hashlib.sha256( passphrase if isinstance(passphrase, bytes) else passphrase. encode("utf-8")).hexdigest() elif not privatekey: # If no privatekey defined from User import UserManager user = UserManager.user_manager.get() if user: site_data = user.getSiteData(address) privatekey = site_data.get("privatekey") else: privatekey = None if not privatekey: # Not found in users.json, ask from console import getpass privatekey = getpass.getpass("Private key (input hidden):") try: succ = site.content_manager.sign( inner_path=inner_path, privatekey=privatekey, update_changed_files=True, remove_missing_optional=remove_missing_optional) except Exception, err: logging.error("Sign error: %s" % Debug.formatException(err)) succ = False
def siteNeedFile(self, address, inner_path): from Site import Site def checker(): while 1: s = time.time() time.sleep(1) print "Switch time:", time.time() - s gevent.spawn(checker) logging.info("Opening a simple connection server") global file_server from Connection import ConnectionServer file_server = ConnectionServer("127.0.0.1", 1234) site = Site(address) site.announce() print site.needFile(inner_path, update=True)
def siteCreate(self): logging.info("Generating new privatekey...") from Crypt import CryptBitcoin privatekey = CryptBitcoin.newPrivatekey() logging.info( "----------------------------------------------------------------------" ) logging.info("Site private key: %s" % privatekey) logging.info( " !!! ^ Save it now, required to modify the site ^ !!!" ) address = CryptBitcoin.privatekeyToAddress(privatekey) logging.info("Site address: %s" % address) logging.info( "----------------------------------------------------------------------" ) while True and not config.batch: if raw_input("? Have you secured your private key? (yes, no) > " ).lower() == "yes": break else: logging.info( "Please, secure it now, you going to need it to modify your site!" ) logging.info("Creating directory structure...") from Site import Site from Site import SiteManager SiteManager.site_manager.load() os.mkdir("%s/%s" % (config.data_dir, address)) open("%s/%s/index.html" % (config.data_dir, address), "w").write("Hello %s!" % address) logging.info("Creating content.json...") site = Site(address) site.content_manager.sign(privatekey=privatekey, extend={"postmessage_nonce_security": True}) site.settings["own"] = True site.saveSettings() logging.info("Site created!")
def sign(address, content, zeronet_directory): privatekey = None with open(zeronet_directory + "data/users.json") as f: users = json.loads(f.read()) try: user = users[users.keys()[0]] privatekey = user["certs"]["zeroid.bit"]["auth_privatekey"] except KeyError: raise TypeError( "Private key for zeroid.bit not found in users.json") from Site import Site site = Site(address, allow_create=False) site.content_manager.sign(inner_path=content, privatekey=privatekey, update_changed_files=True, remove_missing_optional=False)
def parseQafqazInfo(query): query = querize(query) request = Request('https://qafqazinfo.az/news/search?keyword=' + query, None, HEADERS) source = urlopen(request) file = bs.BeautifulSoup(source, 'lxml') news_tags = file.find_all('a', class_='https://qafqazinfo.aznews/search', href=True) site = Site('QafqazInfo') for i in range(len(news_tags)): tag = news_tags[i] site.results.append(News(tag['href'], tag.get_text().strip(), '00.00.0000', '00:00')) for r in site.results: print(r.source + '|' + r.headline + '|' + r.date) return site
def sitePublish(self, address, peer_ip=None, peer_port=15441, inner_path="content.json", diffs={}): global file_server from Site import Site from Site import SiteManager from File import FileServer # We need fileserver to handle incoming file requests from Peer import Peer SiteManager.site_manager.load() logging.info("Loading site...") site = Site(address, allow_create=False) site.settings["serving"] = True # Serving the site even if its disabled logging.info("Creating FileServer....") file_server = FileServer() site.connection_server = file_server file_server_thread = gevent.spawn(file_server.start, check_sites=False) # Dont check every site integrity time.sleep(0.001) if not file_server_thread.ready(): # Started fileserver file_server.openport() if peer_ip: # Announce ip specificed site.addPeer(peer_ip, peer_port) else: # Just ask the tracker logging.info("Gathering peers from tracker") site.announce() # Gather peers published = site.publish(5, inner_path, diffs=diffs) # Push to peers if published > 0: time.sleep(3) logging.info("Serving files (max 60s)...") gevent.joinall([file_server_thread], timeout=60) logging.info("Done.") else: logging.info("No peers found, sitePublish command only works if you already have visitors serving your site") else: # Already running, notify local client on new content logging.info("Sending siteReload") my_peer = Peer("127.0.0.1", config.fileserver_port) logging.info(my_peer.request("siteReload", {"site": site.address, "inner_path": inner_path})) logging.info("Sending sitePublish") logging.info(my_peer.request("sitePublish", {"site": site.address, "inner_path": inner_path, "diffs": diffs})) logging.info("Done.")
def site(request): con = pymysql.connect(host='localhost', user='******', password='', db='dan_py', charset='utf8mb4', autocommit=True, cursorclass=pymysql.cursors.DictCursor) SITE = Site() SITE.db = con.cursor() path = request.match_info.get('url', '') SITE.path = path SITE.p = path.split('/') i = len(SITE.p) while i < 7: SITE.p.append('') i += 1 SITE.request = request return SITE
def parseEastNews(query): query = querize(query) request = Request('https://eastnews.org/search?query=' + query, None, HEADERS) source = urlopen(request) file = bs.BeautifulSoup(source, 'lxml') parent = file.find('div', class_='col-1') news_tags = parent.find_all('a', {'style' : 'color:#000'}, href=True) site = Site("Eastnews.org") for i in range(len(news_tags)): tag = news_tags[i] site.results.append(News(tag['href'], tag.get_text().strip(), ' ', '00:00')) for r in site.results: print(r.source + '|' + r.headline + '|' + r.date) return site
def siteSign(self, address, privatekey=None, inner_path="content.json", publish=False): from Site import Site logging.info("Signing site: %s..." % address) site = Site(address, allow_create=False) if not privatekey: # If no privatekey definied from User import UserManager user = UserManager.user_manager.get() if user: site_data = user.getSiteData(address) privatekey = site_data.get("privatekey") else: privatekey = None if not privatekey: # Not found in users.json, ask from console import getpass privatekey = getpass.getpass("Private key (input hidden):") succ = site.content_manager.sign(inner_path=inner_path, privatekey=privatekey, update_changed_files=True) if succ and publish: self.sitePublish(address, inner_path=inner_path)
def parseAzadliq(query): query = querize(query) request = Request('https://www.azadliq.org/s?k=' + query, None, HEADERS) source = urlopen(request) file = bs.BeautifulSoup(source, 'lxml') source_tags = file.find_all('a', class_='img-wrap img-wrap--t-spac img-wrap--size-3 img-wrap--float img-wrap--xs', href=True) headline_tags = file.find_all('h4', class_='media-block__title media-block__title--size-3') date_tags = file.find_all('span', class_='date date--mb date--size-3') site = Site('Azadliq Radiosu') for i in range(len(source_tags)): source = source_tags[i] headline = source['title'] date = date_tags[i] site.results.append(News('https://www.azadliq.org' + source['href'], headline, date.string, '00:00')) return site
def parseAzToday(query): query = querize(query) request = Request('https://www.aztoday.az/?s=' + query, None, HEADERS) source = urlopen(request) file = bs.BeautifulSoup(source, 'lxml') parent = file.find('ul', class_='penci-wrapper-data penci-grid') news_tags = parent.find_all('a', class_='penci-image-holder penci-lazy', href=True) date_tags = parent.find_all('time', class_='entry-date published') site = Site('Aztoday.az') for i in range(len(news_tags)): tag = news_tags[i] date = date_tags[i] site.results.append(News(tag['href'], tag['title'], date.get_text().strip(), '00:00')) return site
def parseRealTV(query): query = querize(query) request = Request('https://www.realtv.az/search/?text=' + query, None, HEADERS) source = urlopen(request) file = bs.BeautifulSoup(source, 'lxml') parent_tags = file.find_all('h4', class_='card-title gel-pica-bold') news_tags = list() site = Site("Real TV") for tag in parent_tags: news_tags.append(tag.find('a', href=True)) for i in range(len(news_tags)): tag = news_tags[i] site.results.append(News('https://www.lent.az' + tag['href'], tag.get_text().strip(), ' ', '00:00')) return site
def get_random_site(problem): num_operations = problem.num_operations random_site = Site(num_operations) #Create a list of size equal to number of operation, and every operation is marked by a number which equals the job job_operation_list = [] for i in range(problem.num_jobs): for j in range(problem.num_machines): job_operation_list.append(i) for i in range(len(random_site.location)): random_operation_nr = random.randint(0, len(job_operation_list) - 1) # print("random_operation_nr: ", random_operation_nr) # print("len(job_operation_list): ", len(job_operation_list)) random_site.location[i] = job_operation_list[random_operation_nr] del job_operation_list[random_operation_nr] return random_site
def siteVerify(address): from Site import Site logging.info("Verifing site: %s..." % address) site = Site(address) logging.info("Verifing content.json signature...") if site.verifyFile("content.json", open(site.getPath("content.json"), "rb"), force=True) != False: # Force check the sign logging.info("[OK] content.json signed by address %s!" % address) else: logging.error("[ERROR] Content.json not signed by address %s!" % address) logging.info("Verifying site files...") bad_files = site.verifyFiles() if not bad_files: logging.info("[OK] All file sha512sum matches!") else: logging.error("[ERROR] Error during verifying site files!")
def parseOrduAz(query): query = querize(query) request = Request('https://ordu.az/index.php?search=' + query, None, HEADERS) source = urlopen(request) file = bs.BeautifulSoup(source, 'lxml') super_parent = file.find('div', class_='col-md-9 col-sm-12') parent = super_parent.find('div', class_='row') source_tags = parent.find_all('a', href=True) headline_tags = parent.find_all('div', class_='news-title') site = Site('Ordu.az') for i in range(len(source_tags)): source = source_tags[i] headline = headline_tags[i] site.results.append(News(source['href'], headline.get_text().strip(), '00.00.0000', '00:00')) return site
def siteNeedFile(self, address, inner_path): from Site import Site from Site import SiteManager SiteManager.site_manager.load() def checker(): while 1: s = time.time() time.sleep(1) print "Switch time:", time.time() - s gevent.spawn(checker) logging.info("Opening a simple connection server") global file_server from File import FileServer file_server = FileServer("127.0.0.1", 1234) file_server_thread = gevent.spawn(file_server.start, check_sites=False) site = Site(address) site.announce() print site.needFile(inner_path, update=True)
def need(self, address, all_file=True): from Site import Site site = self.get(address) if not site: # Site not exist yet # Try to find site with differect case for recover_address, recover_site in self.sites.items(): if recover_address.lower() == address.lower(): return recover_site if not self.isAddress(address): return False # Not address: %s % address self.log.debug("Added new site: %s" % address) site = Site(address) self.sites[address] = site if not site.settings["serving"]: # Maybe it was deleted before site.settings["serving"] = True site.saveSettings() if all_file: # Also download user files on first sync site.download(check_size=True, blind_includes=True) return site
def parseFemidaAz(query): query = querize(query) request = Request('http://femida.az/?search=' + query, None, HEADERS) source = urlopen(request) file = bs.BeautifulSoup(source, 'lxml') parent = file.find('ul', class_='list') news_tags = parent.find_all('a', href=True) site = Site('Femida.az') for i in range(len(news_tags)): tag = news_tags[i] site.results.append(News(tag['href'], tag.get_text().strip(), '00.00.0000', '00:00')) for r in site.results: print(r.source + '|' + r.headline + '|' + r.date) return site
def parseDemokratAz(query): query = querize(query) request = Request('https://demokrat.az/search?q=' + query, None, HEADERS) source = urlopen(request) file = bs.BeautifulSoup(source, 'lxml') parent = file.find('div', {'id' : 'wrapper'}) news_tags = parent.find_all('a', href=True) site = Site('Demokrat.az') for i in range(len(news_tags)): tag = news_tags[i] site.results.append(News('https://demokrat.az/' + tag['href'], tag.get_text().strip(), '00.00.0000', '00:00')) for r in site.results: print(r.source + '|' + r.headline + '|' + r.date) return site
def parseAzVision(query): query = querize(query) request = Request('https://azvision.az/search.php?search=' + query, None, HEADERS) source = urlopen(request) file = bs.BeautifulSoup(source, 'lxml') parent = file.find('ul', 'contents') news_tags = file.find_all('a', {'itemprop' : 'name'}) site = Site('AzVision.az') for i in range(len(news_tags)): tag = news_tags[i] site.results.append(News('https://azvision.az' + tag['href'], tag.get_text().strip(), '00.00.0000', '00:00')) for r in site.results: print(r.source + '|' + r.headline + '|' + r.date) return site
def getSite(self,siteId = None, dateReq = NOW.strftime("%Y-%m-%d %H:00:00"), end = NOW.strftime("%Y-%m-%d %H:00:00"),db=None,locked=False): #to get site description specified by time dateReq = str(dateReq) end = str(end) site = None dbStatus = False if siteId == None: return None if datetime.strptime(dateReq, "%Y-%m-%d %H:00:00") - datetime.strptime(end, "%Y-%m-%d %H:00:00") > timedelta(hours=0) : end = dateReq if db == None: db = Database() if db.connect(): dbStatus = True else: dbStatus = True if dbStatus : try: if not locked : db.lock({'site':'READ','schedule':'READ','reservation':'READ','site_reserved':'READ'}) db.execute('SELECT * FROM `site` WHERE `site_id` = "'+str(siteId)+'";') data = db.getCursor().fetchone() site = Site(site=data,db=db) res = site.getResources() for i in range(0,len(res)): res[i].setAvailableAmount(db=db,begin=dateReq,end=end) site.setRunningAmount(db=db,aTime=dateReq) db.unlock() finally: if not locked : db.close() return site
def parsePublikaAz(query): query = querize(query) request = Request('https://publika.az/search.php?query=' + query, None, HEADERS) source = urlopen(request) file = bs.BeautifulSoup(source, 'lxml') parent = file.find('div', class_='page_layout clearfix') news_tags = parent.find_all('a', href=True) date_tags = parent.find_all('li', class_='date') print(len(news_tags)) print(len(date_tags)) length = len(news_tags) del news_tags[length - 1] del news_tags[length - 2] i = 0 while (i < len(news_tags)): news_tags[i] = date_tags[0] i = i + 2 while date_tags[0] in news_tags: news_tags.remove(date_tags[0]) site = Site('Publika.az') for i in range(len(news_tags)): tag = news_tags[i] date = date_tags[i] site.results.append(News(tag['href'], tag.get_text().strip(), date.get_text().strip(), '00:00')) for r in site.results: print(r.source + '|' + r.headline + '|' + r.date) return site
def load(self, cleanup=True): self.log.debug("Loading sites...") self.loaded = False from Site import Site if self.sites is None: self.sites = {} address_found = [] added = 0 # Load new adresses for address, settings in json.load(open("%s/sites.json" % config.data_dir)).iteritems(): if address not in self.sites and os.path.isfile("%s/%s/content.json" % (config.data_dir, address)): s = time.time() try: site = Site(address, settings=settings) site.content_manager.contents.get("content.json") except Exception, err: self.log.debug("Error loading site %s: %s" % (address, err)) continue self.sites[address] = site self.log.debug("Loaded site %s in %.3fs" % (address, time.time() - s)) added += 1 address_found.append(address)
def parseQaynarInfo(query): query = querize(query) request = Request('https://qaynarinfo.az/az/search/?query=' + query, None, HEADERS) source = urlopen(request) file = bs.BeautifulSoup(source, 'lxml') parent = file.find('div', class_='posts-wrap') source_tags = parent.find_all('a', href=True) headline_tags = parent.find_all('h2', 'post-title') date_tags = parent.find_all('div', class_='post-date') site = Site('Qaynarinfo.az') for i in range(len(source_tags)): source = source_tags[i] headline = headline_tags[i] date = date_tags[i] site.results.append(News(source['href'], headline.get_text().strip(), date.get_text().strip(), '00:00')) return site
def parseReportAz(query): query = querize(query) request = Request('https://www.report.az/search?query=' + query, None, HEADERS) source = urlopen(request) file = bs.BeautifulSoup(source, 'lxml') news_tags = file.find_all('a', class_='title', href=True) date_tags = file.find_all('div', class_='news-date') site = Site('Report.az') for i in range(len(news_tags)): tag = news_tags[i] date = date_tags[i] site.results.append(News("https://report.az/" + tag['href'], tag.get_text().strip(), date.get_text().strip(), '00:00')) # to print for debugging for r in site.results: print(r.source + '|' + r.headline + '|' + r.date) return site
def load(self, cleanup=True): self.log.debug("Loading sites...") self.loaded = False from Site import Site if self.sites is None: self.sites = {} address_found = [] added = 0 # Load new adresses for address, settings in json.load( open("%s/sites.json" % config.data_dir)).iteritems(): if address not in self.sites and os.path.isfile( "%s/%s/content.json" % (config.data_dir, address)): s = time.time() self.sites[address] = Site(address, settings=settings) self.log.debug("Loaded site %s in %.3fs" % (address, time.time() - s)) added += 1 address_found.append(address) # Remove deleted adresses if cleanup: for address in self.sites.keys(): if address not in address_found: del (self.sites[address]) self.log.debug("Removed site: %s" % address) # Remove orpan sites from contentdb for row in ContentDb.getContentDb().execute("SELECT * FROM site"): if row["address"] not in self.sites: self.log.info("Deleting orphan site from content.db: %s" % row["address"]) ContentDb.getContentDb().execute( "DELETE FROM site WHERE ?", {"address": row["address"]}) if added: self.log.debug("SiteManager added %s sites" % added) self.loaded = True
def testVerify(): from Site import Site site = Site("12Hw8rTgzrNo4DSh2AkqwPRqDyTticwJyH") content_manager = ContentManager(site) print "Loaded contents:", content_manager.contents.keys() file = open( site.storage.getPath( "data/users/1KRxE1s3oDyNDawuYWpzbLUwNm8oDbeEp6/content.json")) print "content.json valid:", content_manager.verifyFile( "data/users/1KRxE1s3oDyNDawuYWpzbLUwNm8oDbeEp6/content.json", file, ignore_same=False) file = open( site.storage.getPath( "data/users/1KRxE1s3oDyNDawuYWpzbLUwNm8oDbeEp6/messages.json")) print "messages.json valid:", content_manager.verifyFile( "data/users/1KRxE1s3oDyNDawuYWpzbLUwNm8oDbeEp6/messages.json", file, ignore_same=False)