def getallsubsforurl(url, langs): result = [] content = geturl(url) soup = BeautifulSoup(content) for versions in soup.findAll("div", {"class": "ssdiv", "id": re.compile("version[0-9]+")}): version = versions.find("p", {"class": "title-sub"}) # log(__name__, version) if version is None: continue version = version.text.split(",")[0] # log(__name__, version) for subtitulo in versions.findAll("ul", {"class": "sslist"}): idioma = subtitulo.findNext("li", {"class": "li-idioma"}).findNext("b").text # log(__name__, idioma) enlace = subtitulo.findNext("li", {"class": "rng download green"}) if enlace is None: continue enlace = enlace.findNext("a", {"class": None})["href"] # log(__name__, enlace) _idioma = languages.get(idioma, ("Unknown", "-", "???", 3)) if _idioma[1] not in langs: continue filename = "{0}".format(version.encode("utf-8")) result.append( { "filename": filename, "flag": _idioma[2], "link": "http://www.tusubtitulo.com/" + enlace, "lang": _idioma[0], "order": 1 + _idioma[3], "referer": url, } ) return result
def getcode(tvshow): #log(__name__, "CODE=%s" %(tvshow)) reg_code = '<a href="/show/([0-9]+)">' + tvshow + '</a>' content = geturl("http://www.tusubtitulo.com/series.php") result = re.findall(reg_code, content, re.IGNORECASE | re.DOTALL | re.MULTILINE | re.UNICODE) return result[0] if len(result) > 0 else None
def _getimdbmeta(movie): imdbmain = utils.geturl('http://akas.imdb.com/title/' + movie.imdbid) if not imdbmain: return False soup = bs4.BeautifulSoup(imdbmain) movie.plot = soup.find('div', itemprop='description').p.contents[0].strip() movie.outline = soup.find('p', itemprop='description').string.strip() movie.rating = soup.find('span', itemprop='ratingValue').string.strip() movie.votes = soup.find('span', itemprop='ratingCount').string.strip() # for entry in soup.find_all('div', class_='txt-block'): # if entry.find(text='Taglines:') is not None: # movie.tagline = list(entry.stripped_strings)[1] # if entry.find(text='Runtime:') is not None: # movie.runtime = str(int(entry.find('time')['datetime'][2:-1]) * 60) # break match = soup.find_all('a', href=re.compile('\/chart\/top\?tt')) if match: movie.top250 = match[0].find('strong').string.split('#')[1] # restudio = '"/company/[^>]+>[^<]*<[^>]*>([^<]+)</span>' # ismatch = getregex(str(imdbmain, encoding='utf-8'), restudio) # if ismatch: # moviestudio = ismatch # recountries = '"/country/[^>]+>([^<]+)</a>' # ismatch = getregex(str(imdbmain, encoding='utf-8'), recountries, re.IGNORECASE, True) # if ismatch: # for country in ismatch: # moviecountries.append(country) return True
def main(): logger.info('프로그램 시작') #메인 URL 로드 겸 변경 체크 utils.geturl() #웹툰리스트 로드 list = utils.getList() logger.info(list) # selenium 초기화 option = webdriver.ChromeOptions() option.add_argument('headless') option.add_argument('disable-gpu') driver = webdriver.Chrome('chromedriver') # ,chrome_options=option) #웹툰페이지 로스 스타트 for page in list: search(driver, page.pageType, page.url) driver.quit() logger.info('프로그램 종료') exit()
def search_tvshow(tvshow, season, episode, languages): #log(__name__,"TVSHOW = %s" % (tvshow)) code = None level = 0 while code is None and level < 4: _tvshow, code = parsetvshow(tvshow, level) if code is None: level += 1 #log(__name__, tvshow) #log(__name__, season) #log(__name__, episode) #log(__name__, languages) ajax_url = main_url + _tvshow + '&season=' + season #log( __name__ ,"URL = %s" % (url)) ajax_content = geturl(ajax_url) soup = BeautifulSoup(ajax_content) links = soup.findAll("td", {"class": "NewsTitle"}) return getallsubsforurl("http:" + links[int(episode)].a['href'], languages)
def getallsubsforurl(url, langs): result = [] content = geturl(url) soup = BeautifulSoup(content) for versions in soup.findAll("div", { "class": "ssdiv", "id": re.compile('version[0-9]+') }): version = versions.find("p", {"class": "title-sub"}) #log(__name__, version) if version is None: continue version = version.text.split(',')[0] #log(__name__, version) for subtitulo in versions.findAll("ul", {"class": "sslist"}): idioma = subtitulo.findNext("li", { "class": "li-idioma" }).findNext("b").text #log(__name__, idioma) enlace = subtitulo.findNext("li", {"class": "rng download green"}) if enlace is None: continue enlace = enlace.findNext("a", {"class": None})["href"] #log(__name__, enlace) _idioma = languages.get(idioma, ("Unknown", "-", "???", 3)) if _idioma[1] not in langs: continue filename = "{0}".format(version.encode('utf-8')) result.append({ 'filename': filename, 'flag': _idioma[2], 'link': "http://www.tusubtitulo.com/" + enlace, 'lang': _idioma[0], 'order': 1 + _idioma[3], 'referer': url }) return result
def getcode(tvshow): # log(__name__, "CODE=%s" %(tvshow)) reg_code = '<a href="/show/([0-9]+)">' + tvshow + "</a>" content = geturl("http://www.tusubtitulo.com/series.php") result = re.findall(reg_code, content, re.IGNORECASE | re.DOTALL | re.MULTILINE | re.UNICODE) return result[0] if len(result) > 0 else None
def down_install_blas(self): print """ The netlib BLAS library is being installed. Don't expect high performance from this netlib library! If you want performance, you need to use an optimized BLAS library and, to avoid unnecessary complications, if you need to compile this optimized BLAS library, use the same compiler you're using here.""" sys.stdout.flush() savecwd = os.getcwd() # creating the build,lib and log dirs if don't exist if not os.path.isdir(os.path.join(self.config.prefix, 'lib')): os.mkdir(os.path.join(self.config.prefix, 'lib')) if not os.path.isdir(os.path.join(os.getcwd(), 'log')): os.mkdir(os.path.join(os.getcwd(), 'log')) # Check if blas.tgz is already present in the working dir # otherwise download it if not os.path.isfile( os.path.join(self.config.prefix, 'lib/librefblas.a')): # if not os.path.isfile(os.path.join(os.getcwd(),getURLName(self.blasurl))): # if not os.path.isfile(os.path.join(os.getcwd(),'BLAS')): print "Downloading BLAS...", # downloader(self.blasurl,self.downcmd) # urllib.urlretrieve(self.blasurl, "blas.tgz") geturl(self.blasurl, "blas.tgz") print "Download is done" else: print "Netlib Blas library is already installed at " + os.path.join( self.config.prefix, 'lib/librefblas.a') self.config.blaslib = '-L' + os.path.join(self.config.prefix, 'lib') + ' -lrefblas ' return 0 # unzip and untar os.chdir('download') print 'Unzip and untar netlib BLAS...', # comm = 'gunzip -f blas.tgz' comm = 'mkdir BLAS; tar zx --strip-components=1 -C BLAS -f blas.tgz ' (output, error, retz) = shellcmd(comm) if retz: print '\n\nBLAS: cannot unzip blas.tgz' print 'stderr:\n', '*' * 50, '\n', comm, '\n', error, '\n', '*' * 50 sys.exit() # comm = 'mkdir BLAS && tar x --strip-components=1 -C BLAS -f blas.tar' # (output, error, retz) = shellcmd(comm) # if retz: # print '\n\nBLAS: cannot untar blas.tgz' # print 'stderr:\n','*'*50,'\n',comm,'\n',error,'\n','*'*50 # sys.exit() # os.remove('blas.tar') print 'done' # change to BLAS dir os.chdir(os.path.join(os.getcwd(), 'BLAS')) # compile and generate library print 'Compile and generate netlib BLAS...', sys.stdout.flush() comm = self.config.fc + ' ' + self.config.fflags + " -c *.f" (output, error, retz) = shellcmd(comm) if retz: print "\n\nBLAS: cannot compile blas" print "stderr:\n", "*" * 50, "\n", comm, '\n', error, "\n", "*" * 50 sys.exit() log = output + error comm = "ar cr librefblas.a *.o" (output, error, retz) = shellcmd(comm) if retz: print "\n\nBLAS: cannot create blas library" print "stderr:\n", "*" * 50, "\n", comm, '\n', error, "\n", "*" * 50 sys.exit() print "done" log = log + output + error comm = self.config.ranlib + " librefblas.a" (output, error, retz) = shellcmd(comm) if retz: print "\n\nBLAS: cannot create table of contents for blas library" print "stderr:\n", "*" * 50, "\n", comm, '\n', error, "\n", "*" * 50 sys.exit() print "done" # write the log on a file log = log + output + error fulllog = os.path.join(savecwd, 'log/log.blas') writefile(fulllog, log) print 'Installation of netlib BLAS successful.' print '(log is in ', fulllog, ')' # move librefblas.a to the lib directory shutil.copy('librefblas.a', os.path.join(self.config.prefix, 'lib/librefblas.a')) # set framework variables to point to the freshly installed BLAS library self.config.blaslib = '-L' + os.path.join(self.config.prefix, 'lib') + ' -lrefblas ' os.chdir(savecwd)
def down_install_lapack(self): print """ The LAPACK library is being installed. """ sys.stdout.flush() savecwd = os.getcwd() # creating the build,lib and log dirs if don't exist if not os.path.isdir(os.path.join(self.config.prefix, 'lib')): os.mkdir(os.path.join(self.config.prefix, 'lib')) if not os.path.isdir(os.path.join(os.getcwd(), 'log')): os.mkdir(os.path.join(os.getcwd(), 'log')) # Check if lapack.tgz is already present in the working dir # otherwise download it if not os.path.isfile( os.path.join(self.config.prefix, 'lib/liblapack.a')): # if not os.path.isfile(os.path.join(os.getcwd(),getURLName(self.lapackurl))): print "Downloading LAPACK...", # downloader(self.lapackurl,self.downcmd) # urllib.urlretrieve(self.lapackurl, "lapack.tgz") geturl(self.lapackurl, "lapack.tgz") print "Download is done" else: print "Netlib Lapack library is already installed at " + os.path.join( self.config.prefix, 'lib/liblapack.a') self.config.lapacklib = '-L' + os.path.join( self.config.prefix, 'lib') + ' -ltmg -llapack' return 0 # unzip and untar os.chdir('download') print 'Unzip and untar Lapack...', #comm = 'gunzip -f '+self.lapversion+'.tgz' comm = 'tar zxf lapack.tgz' (output, error, retz) = shellcmd(comm) if retz: print '\n\nLAPACK: cannot unzip ' + self.lapversion + '.tgz' print 'stderr:\n', '*' * 50, '\n', comm, '\n', error, '\n', '*' * 50 sys.exit() #comm = 'tar xf '+self.lapversion+'.tar' #(output, error, retz) = shellcmd(comm) #if retz: # print '\n\nLAPACK: cannot untar '+self.lapversion+'.tar' # print 'stderr:\n','*'*50,'\n',comm,'\n',error,'\n','*'*50 # sys.exit() #os.remove(self.lapversion+'.tar') print 'done' ##Apply the patch to correct [sd]lantr #print 'Apply patch on lapacke...', #comm = '(cd '+self.lapversion+' && patch -p 0 < '+(os.path.join(savecwd,'../script/patch_lantr'))+')' #(output, error, retz) = shellcmd(comm) #print 'done' # # Overwrite [sd]lamch.f # shutil.copy(os.path.join(self.dmft.installerdir,'src/dlamch.f'), # os.path.join(os.getcwd(),'lapack-3.3.1/INSTALL')) # shutil.copy(os.path.join(self.dmft.installerdir,'src/slamch.f'), # os.path.join(os.getcwd(),'lapack-3.3.1/INSTALL')) # change to BLAS dir os.chdir(os.path.join(os.getcwd(), self.lapversion)) # Write Makefile.in writefile( 'make.inc', """ # -*- Makefile generated by DMFT installer -*- #################################################################### # LAPACK make include file. # # LAPACK, Version """ + self.lapversion + """" # # April 2012 # #################################################################### # SHELL = /bin/sh # # Modify the FORTRAN and OPTS definitions to refer to the # compiler and desired compiler options for your machine. NOOPT # refers to the compiler options desired when NO OPTIMIZATION is # selected. Define LOADER and LOADOPTS to refer to the loader and # desired load options for your machine. # FORTRAN = """ + self.config.fc + """ OPTS = """ + self.config.fflags + """ DRVOPTS = $(OPTS) NOOPT = -O0 LOADER = """ + self.config.fc + """ LOADOPTS = MAKE = make -j 8 # # Timer for the SECOND and DSECND routines # # Default : SECOND and DSECND will use a call to the EXTERNAL FUNCTION ETIME # TIMER = EXT_ETIME # For RS6K : SECOND and DSECND will use a call to the EXTERNAL FUNCTION ETIME_ # TIMER = EXT_ETIME_ # For gfortran compiler: SECOND and DSECND will use a call to the INTERNAL FUNCTION ETIME # TIMER = INT_ETIME # If your Fortran compiler does not provide etime (like Nag Fortran Compiler, etc...) # SECOND and DSECND will use a call to the INTERNAL FUNCTION CPU_TIME TIMER = INT_CPU_TIME # If neither of this works...you can use the NONE value... In that case, SECOND and DSECND will always return 0 # TIMER = NONE # # Configuration LAPACKE: Native C interface to LAPACK # To generate LAPACKE library: type 'make lapackelib' # Configuration file: turned off (default) # Complex types: C99 (default) # Name pattern: mixed case (default) # (64-bit) Data model: LP64 (default) # # CC is the C compiler, normally invoked with options CFLAGS. # CC = """ + self.config.cc + """ CFLAGS = """ + self.config.cflags + """ # # The archiver and the flag(s) to use when building archive (library) # If you system has no ranlib, set RANLIB = echo. # ARCH = ar ARCHFLAGS= """ + self.config.arflags + """ RANLIB = """ + self.config.ranlib + """ # # The location of BLAS library for linking the testing programs. # The target's machine-specific, optimized BLAS library should be # used whenever possible. # BLASLIB = """ + self.config.blaslib + """ # # Location of the extended-precision BLAS (XBLAS) Fortran library # used for building and testing extended-precision routines. The # relevant routines will be compiled and XBLAS will be linked only if # USEXBLAS is defined. # # USEXBLAS = Yes XBLASLIB = # XBLASLIB = -lxblas # # Names of generated libraries. # LAPACKLIB = liblapack.a TMGLIB = libtmg.a EIGSRCLIB = libeigsrc.a LINSRCLIB = liblinsrc.a LAPACKELIB = liblapacke.a """) # compile and generate library print 'Compile and generate LAPACK...', sys.stdout.flush() comm = self.make + ' lapacklib tmglib' (output, error, retz) = shellcmd(comm) if retz: print "\n\nLAPACK: cannot compile LAPACK" print "stderr:\n", "*" * 50, "\n", comm, '\n', error, "\n", "*" * 50 sys.exit() log = output + error # write the log on a file log = log + output + error fulllog = os.path.join(savecwd, 'log/log.lapack') writefile(fulllog, log) print 'Installation of liblapack.a successful.' print '(log is in ', fulllog, ')' # move libcblas.a to the lib directory shutil.copy('liblapack.a', os.path.join(self.config.prefix, 'lib/liblapack.a')) shutil.copy('libtmg.a', os.path.join(self.config.prefix, 'lib/libtmg.a')) # set framework variables to point to the freshly installed BLAS library self.config.lapacklib = '-L' + os.path.join(self.config.prefix, 'lib') + ' -ltmg -llapack' os.chdir(savecwd) self.config.lapinstalled = 1 # Check if the installation is successful self.dmft.verbose = 1 ret = self.check_lapack() self.dmft.verbose = 0 if ret != 0: sys.exit()
def down_install_gsl(self): print "The GSL library is being installed." sys.stdout.flush() savecwd = os.getcwd() # creating the build,lib and log dirs if don't exist if not os.path.isdir(os.path.join(self.config.prefix, 'gsl')): os.mkdir(os.path.join(self.config.prefix, 'gsl')) if not os.path.isdir(os.path.join(os.getcwd(), 'log')): os.mkdir(os.path.join(os.getcwd(), 'log')) # Check if gsl.tgz is already present in the working dir # otherwise download it if not os.path.isfile( os.path.join(os.getcwd(), getURLName(self.gslurl))): print "Downloading GSL ...", #downloader(self.lapackurl,self.downcmd) #urllib.urlretrieve(self.gslurl, "gsl.tgz") geturl(self.gslurl, "gsl.tgz") print "done" # unzip and untar os.chdir('download') print 'Unzip and untar GSL...', comm = 'tar zxf gsl.tgz ' (output, error, retz) = shellcmd(comm) if retz: print '\n\nlibgsl: cannot unzip ' + self.gslversion + '.tgz' print 'stderr:\n', '*' * 50, '\n', comm, '\n', error, '\n', '*' * 50 sys.exit() print 'done' # change to GSL dir os.chdir(os.path.join(os.getcwd(), self.gslversion)) # compile and generate library print 'Configure GSL...', sys.stdout.flush() comm = './configure CC=' + self.config.cc + ' --prefix=' + os.path.join( self.config.prefix, 'gsl') (output, error, retz) = shellcmd(comm) if retz: print "\n\nlingsl: cannot configure GSL" print "stderr:\n", "*" * 50, "\n", comm, '\n', error, "\n", "*" * 50 sys.exit() log = output + error # write log on a file log = log + output + error fulllog = os.path.join(savecwd, 'log/log.gsl') writefile(fulllog, log) print 'Configuration of GSL successful.' print '(log is in ', fulllog, ')' # compile and generate library print 'Compile and generate GSL...', sys.stdout.flush() comm = self.make + ' -j4; ' + self.make + ' install' (output, error, retz) = shellcmd(comm) if retz: print "\n\nlingsl: cannot compile GSL" print "stderr:\n", "*" * 50, "\n", comm, '\n', error, "\n", "*" * 50 sys.exit() log = output + error # write the log on a file log = log + output + error fulllog = os.path.join(savecwd, 'log/log.gsl') writefile(fulllog, log) print 'Installation of GSL successful.' print '(log is in ', fulllog, ')' # move libcblas.a to the lib directory #shutil.copy('libtmg.a',os.path.join(self.config.prefix,'gsl/libtmg.a')) # set framework variables to point to the freshly installed GSL library self.config.gsl = '-L' + os.path.join(self.config.prefix, 'gsl') + ' -lgsl ' os.chdir(savecwd) # Check if the installation is successful self.dmft.verbose = 1 # self.check_gsl() self.dmft.verbose = 0
from time import sleep import sys import socket import argparse import logging # by kzl from settings import mylogger,NOT_EXIST,ACCESS_DENIED,ALREADY_EXIST,SUCCESS,PORT,SHARED_FOLDER,SERVER_START_TIME,SECRET_LENGTH,IPS_FILE,URL_PREFIX,UPDATE_INTERVAL from utils import random_string,get_lan_ip,geturl from p2p_server import Node from threads import UpdateGUIListTimer # gui from PyQt4 import QtGui,QtCore from settings import WIN_WIDTH,WIN_HEIGHT,ICON_APP,ICON_FETCH,ICON_QUIT IP_LAN = get_lan_ip() SERVER_URL = geturl(URL_PREFIX,IP_LAN,PORT) class NodeServerThread(Thread): """ thread for starting and stopping node server """ def __init__(self,name,url,dirname,secret,ipsfile,event_running): mylogger.info('[__init__]: {0}'.format(name)) super(NodeServerThread,self).__init__() self.name = name self.daemon = True self.url = url self.dirname = dirname self.secret = secret self.ipsfile = ipsfile self.event_running = event_running
def down_install_fftw(self): print """ The FFTW library is being installed. """ sys.stdout.flush() savecwd = os.getcwd() # creating the build,lib and log dirs if don't exist if not os.path.isdir(os.path.join(self.config.prefix, 'fftw')): os.mkdir(os.path.join(self.config.prefix, 'fftw')) if not os.path.isdir(os.path.join(os.getcwd(), 'log')): os.mkdir(os.path.join(os.getcwd(), 'log')) # Check if fftw.tgz is already present in the working dir # otherwise download it if not os.path.isfile(os.path.join(os.getcwd(), "fftw.tgz")): print "Downloading FFTW ...", #downloader(self.lapackurl,self.downcmd) #urllib.urlretrieve(self.fftwurl, "fftw.tgz") geturl(self.fftwurl, "fftw.tgz") print "done" # unzip and untar os.chdir('download') print 'Unzip and untar FFTW...', comm = 'tar zxf fftw.tgz ' (output, error, retz) = shellcmd(comm) if retz: print '\n\nlibfftw: cannot unzip ' + self.fftwversion + '.tgz' print 'stderr:\n', '*' * 50, '\n', comm, '\n', error, '\n', '*' * 50 sys.exit() print 'done' # change to FFTW dir os.chdir(os.path.join(os.getcwd(), self.fftwversion)) # compile and generate library print 'Configure FFTW...', sys.stdout.flush() if (self.config.ompflag == ""): comm = './configure MPICC='+self.config.pcc+' CC='+self.config.cc+' F77='+self.config.fc+\ ' --enable-mpi --enable-threads --enable-shared --prefix='+os.path.join(self.config.prefix,'fftw') else: comm = './configure MPICC='+self.config.pcc+' CC='+self.config.cc+' F77='+self.config.fc+\ ' --enable-mpi --enable-threads --enable-shared -enable-openmp --prefix='+os.path.join(self.config.prefix,'fftw') (output, error, retz) = shellcmd(comm) if retz: print "\n\nlinfftw: cannot configure FFTW" print "stderr:\n", "*" * 50, "\n", comm, '\n', error, "\n", "*" * 50 sys.exit() log = output + error # write log on a file log = log + output + error fulllog = os.path.join(savecwd, 'log/log.fftw') writefile(fulllog, log) print 'Configuration of FFTW successful.' print '(log is in ', fulllog, ')' # compile and generate library print 'Compile and generate FFTW...', sys.stdout.flush() comm = self.make + ' -j4; ' + self.make + ' install' (output, error, retz) = shellcmd(comm) if retz: print "\n\nlinfftw: cannot compile FFTW" print "stderr:\n", "*" * 50, "\n", comm, '\n', error, "\n", "*" * 50 sys.exit() log = output + error # write the log on a file log = log + output + error fulllog = os.path.join(savecwd, 'log/log.fftw') writefile(fulllog, log) print 'Installation of FFTW successful.' print '(log is in ', fulllog, ')' # move libcblas.a to the lib directory #shutil.copy('libtmg.a',os.path.join(self.config.prefix,'fftw/libtmg.a')) self.set_fftlib() os.chdir(savecwd) # Check if the installation is successful self.dmft.verbose = 1 self.check_fftw() self.dmft.verbose = 0