def getallsubsforurl(url, langs):
result = []
content = geturl(url)
soup = BeautifulSoup(content)
for versions in soup.findAll("div", {"class": "ssdiv", "id": re.compile("version[0-9]+")}):
version = versions.find("p", {"class": "title-sub"})
# log(__name__, version)
if version is None:
continue
version = version.text.split(",")[0]
# log(__name__, version)
for subtitulo in versions.findAll("ul", {"class": "sslist"}):
idioma = subtitulo.findNext("li", {"class": "li-idioma"}).findNext("b").text
# log(__name__, idioma)
enlace = subtitulo.findNext("li", {"class": "rng download green"})
if enlace is None:
continue
enlace = enlace.findNext("a", {"class": None})["href"]
# log(__name__, enlace)
_idioma = languages.get(idioma, ("Unknown", "-", "???", 3))
if _idioma[1] not in langs:
continue
filename = "{0}".format(version.encode("utf-8"))
result.append(
{
"filename": filename,
"flag": _idioma[2],
"link": "http://www.tusubtitulo.com/" + enlace,
"lang": _idioma[0],
"order": 1 + _idioma[3],
"referer": url,
}
)
return result
def getcode(tvshow):
#log(__name__, "CODE=%s" %(tvshow))
reg_code = '<a href="/show/([0-9]+)">' + tvshow + '</a>'
content = geturl("http://www.tusubtitulo.com/series.php")
result = re.findall(reg_code, content,
re.IGNORECASE | re.DOTALL | re.MULTILINE | re.UNICODE)
return result[0] if len(result) > 0 else None
def _getimdbmeta(movie):
imdbmain = utils.geturl('http://akas.imdb.com/title/' + movie.imdbid)
if not imdbmain:
return False
soup = bs4.BeautifulSoup(imdbmain)
movie.plot = soup.find('div', itemprop='description').p.contents[0].strip()
movie.outline = soup.find('p', itemprop='description').string.strip()
movie.rating = soup.find('span', itemprop='ratingValue').string.strip()
movie.votes = soup.find('span', itemprop='ratingCount').string.strip()
# for entry in soup.find_all('div', class_='txt-block'):
# if entry.find(text='Taglines:') is not None:
# movie.tagline = list(entry.stripped_strings)[1]
# if entry.find(text='Runtime:') is not None:
# movie.runtime = str(int(entry.find('time')['datetime'][2:-1]) * 60)
# break
match = soup.find_all('a', href=re.compile('\/chart\/top\?tt'))
if match:
movie.top250 = match[0].find('strong').string.split('#')[1]
# restudio = '"/company/[^>]+>[^<]*<[^>]*>([^<]+)</span>'
# ismatch = getregex(str(imdbmain, encoding='utf-8'), restudio)
# if ismatch:
# moviestudio = ismatch
# recountries = '"/country/[^>]+>([^<]+)</a>'
# ismatch = getregex(str(imdbmain, encoding='utf-8'), recountries, re.IGNORECASE, True)
# if ismatch:
# for country in ismatch:
# moviecountries.append(country)
return True
def main():
logger.info('프로그램 시작')
#메인 URL 로드 겸 변경 체크
utils.geturl()
#웹툰리스트 로드
list = utils.getList()
logger.info(list)
# selenium 초기화
option = webdriver.ChromeOptions()
option.add_argument('headless')
option.add_argument('disable-gpu')
driver = webdriver.Chrome('chromedriver') # ,chrome_options=option)
#웹툰페이지 로스 스타트
for page in list:
search(driver, page.pageType, page.url)
driver.quit()
logger.info('프로그램 종료')
exit()
def search_tvshow(tvshow, season, episode, languages):
#log(__name__,"TVSHOW = %s" % (tvshow))
code = None
level = 0
while code is None and level < 4:
_tvshow, code = parsetvshow(tvshow, level)
if code is None:
level += 1
#log(__name__, tvshow)
#log(__name__, season)
#log(__name__, episode)
#log(__name__, languages)
ajax_url = main_url + _tvshow + '&season=' + season
#log( __name__ ,"URL = %s" % (url))
ajax_content = geturl(ajax_url)
soup = BeautifulSoup(ajax_content)
links = soup.findAll("td", {"class": "NewsTitle"})
return getallsubsforurl("http:" + links[int(episode)].a['href'], languages)
def getallsubsforurl(url, langs):
result = []
content = geturl(url)
soup = BeautifulSoup(content)
for versions in soup.findAll("div", {
"class": "ssdiv",
"id": re.compile('version[0-9]+')
}):
version = versions.find("p", {"class": "title-sub"})
#log(__name__, version)
if version is None:
continue
version = version.text.split(',')[0]
#log(__name__, version)
for subtitulo in versions.findAll("ul", {"class": "sslist"}):
idioma = subtitulo.findNext("li", {
"class": "li-idioma"
}).findNext("b").text
#log(__name__, idioma)
enlace = subtitulo.findNext("li", {"class": "rng download green"})
if enlace is None:
continue
enlace = enlace.findNext("a", {"class": None})["href"]
#log(__name__, enlace)
_idioma = languages.get(idioma, ("Unknown", "-", "???", 3))
if _idioma[1] not in langs:
continue
filename = "{0}".format(version.encode('utf-8'))
result.append({
'filename': filename,
'flag': _idioma[2],
'link': "http://www.tusubtitulo.com/" + enlace,
'lang': _idioma[0],
'order': 1 + _idioma[3],
'referer': url
})
return result
def getcode(tvshow):
# log(__name__, "CODE=%s" %(tvshow))
reg_code = '<a href="/show/([0-9]+)">' + tvshow + "</a>"
content = geturl("http://www.tusubtitulo.com/series.php")
result = re.findall(reg_code, content, re.IGNORECASE | re.DOTALL | re.MULTILINE | re.UNICODE)
return result[0] if len(result) > 0 else None
def down_install_blas(self):
print """
The netlib BLAS library is being installed.
Don't expect high performance from this netlib library!
If you want performance, you need to use an optimized BLAS library and,
to avoid unnecessary complications, if you need to compile this optimized BLAS
library, use the same compiler you're using here."""
sys.stdout.flush()
savecwd = os.getcwd()
# creating the build,lib and log dirs if don't exist
if not os.path.isdir(os.path.join(self.config.prefix, 'lib')):
os.mkdir(os.path.join(self.config.prefix, 'lib'))
if not os.path.isdir(os.path.join(os.getcwd(), 'log')):
os.mkdir(os.path.join(os.getcwd(), 'log'))
# Check if blas.tgz is already present in the working dir
# otherwise download it
if not os.path.isfile(
os.path.join(self.config.prefix, 'lib/librefblas.a')):
# if not os.path.isfile(os.path.join(os.getcwd(),getURLName(self.blasurl))):
# if not os.path.isfile(os.path.join(os.getcwd(),'BLAS')):
print "Downloading BLAS...",
# downloader(self.blasurl,self.downcmd)
# urllib.urlretrieve(self.blasurl, "blas.tgz")
geturl(self.blasurl, "blas.tgz")
print "Download is done"
else:
print "Netlib Blas library is already installed at " + os.path.join(
self.config.prefix, 'lib/librefblas.a')
self.config.blaslib = '-L' + os.path.join(self.config.prefix,
'lib') + ' -lrefblas '
return 0
# unzip and untar
os.chdir('download')
print 'Unzip and untar netlib BLAS...',
# comm = 'gunzip -f blas.tgz'
comm = 'mkdir BLAS; tar zx --strip-components=1 -C BLAS -f blas.tgz '
(output, error, retz) = shellcmd(comm)
if retz:
print '\n\nBLAS: cannot unzip blas.tgz'
print 'stderr:\n', '*' * 50, '\n', comm, '\n', error, '\n', '*' * 50
sys.exit()
# comm = 'mkdir BLAS && tar x --strip-components=1 -C BLAS -f blas.tar'
# (output, error, retz) = shellcmd(comm)
# if retz:
# print '\n\nBLAS: cannot untar blas.tgz'
# print 'stderr:\n','*'*50,'\n',comm,'\n',error,'\n','*'*50
# sys.exit()
# os.remove('blas.tar')
print 'done'
# change to BLAS dir
os.chdir(os.path.join(os.getcwd(), 'BLAS'))
# compile and generate library
print 'Compile and generate netlib BLAS...',
sys.stdout.flush()
comm = self.config.fc + ' ' + self.config.fflags + " -c *.f"
(output, error, retz) = shellcmd(comm)
if retz:
print "\n\nBLAS: cannot compile blas"
print "stderr:\n", "*" * 50, "\n", comm, '\n', error, "\n", "*" * 50
sys.exit()
log = output + error
comm = "ar cr librefblas.a *.o"
(output, error, retz) = shellcmd(comm)
if retz:
print "\n\nBLAS: cannot create blas library"
print "stderr:\n", "*" * 50, "\n", comm, '\n', error, "\n", "*" * 50
sys.exit()
print "done"
log = log + output + error
comm = self.config.ranlib + " librefblas.a"
(output, error, retz) = shellcmd(comm)
if retz:
print "\n\nBLAS: cannot create table of contents for blas library"
print "stderr:\n", "*" * 50, "\n", comm, '\n', error, "\n", "*" * 50
sys.exit()
print "done"
# write the log on a file
log = log + output + error
fulllog = os.path.join(savecwd, 'log/log.blas')
writefile(fulllog, log)
print 'Installation of netlib BLAS successful.'
print '(log is in ', fulllog, ')'
# move librefblas.a to the lib directory
shutil.copy('librefblas.a',
os.path.join(self.config.prefix, 'lib/librefblas.a'))
# set framework variables to point to the freshly installed BLAS library
self.config.blaslib = '-L' + os.path.join(self.config.prefix,
'lib') + ' -lrefblas '
os.chdir(savecwd)
def down_install_lapack(self):
print """
The LAPACK library is being installed.
"""
sys.stdout.flush()
savecwd = os.getcwd()
# creating the build,lib and log dirs if don't exist
if not os.path.isdir(os.path.join(self.config.prefix, 'lib')):
os.mkdir(os.path.join(self.config.prefix, 'lib'))
if not os.path.isdir(os.path.join(os.getcwd(), 'log')):
os.mkdir(os.path.join(os.getcwd(), 'log'))
# Check if lapack.tgz is already present in the working dir
# otherwise download it
if not os.path.isfile(
os.path.join(self.config.prefix, 'lib/liblapack.a')):
# if not os.path.isfile(os.path.join(os.getcwd(),getURLName(self.lapackurl))):
print "Downloading LAPACK...",
# downloader(self.lapackurl,self.downcmd)
# urllib.urlretrieve(self.lapackurl, "lapack.tgz")
geturl(self.lapackurl, "lapack.tgz")
print "Download is done"
else:
print "Netlib Lapack library is already installed at " + os.path.join(
self.config.prefix, 'lib/liblapack.a')
self.config.lapacklib = '-L' + os.path.join(
self.config.prefix, 'lib') + ' -ltmg -llapack'
return 0
# unzip and untar
os.chdir('download')
print 'Unzip and untar Lapack...',
#comm = 'gunzip -f '+self.lapversion+'.tgz'
comm = 'tar zxf lapack.tgz'
(output, error, retz) = shellcmd(comm)
if retz:
print '\n\nLAPACK: cannot unzip ' + self.lapversion + '.tgz'
print 'stderr:\n', '*' * 50, '\n', comm, '\n', error, '\n', '*' * 50
sys.exit()
#comm = 'tar xf '+self.lapversion+'.tar'
#(output, error, retz) = shellcmd(comm)
#if retz:
# print '\n\nLAPACK: cannot untar '+self.lapversion+'.tar'
# print 'stderr:\n','*'*50,'\n',comm,'\n',error,'\n','*'*50
# sys.exit()
#os.remove(self.lapversion+'.tar')
print 'done'
##Apply the patch to correct [sd]lantr
#print 'Apply patch on lapacke...',
#comm = '(cd '+self.lapversion+' && patch -p 0 < '+(os.path.join(savecwd,'../script/patch_lantr'))+')'
#(output, error, retz) = shellcmd(comm)
#print 'done'
# # Overwrite [sd]lamch.f
# shutil.copy(os.path.join(self.dmft.installerdir,'src/dlamch.f'),
# os.path.join(os.getcwd(),'lapack-3.3.1/INSTALL'))
# shutil.copy(os.path.join(self.dmft.installerdir,'src/slamch.f'),
# os.path.join(os.getcwd(),'lapack-3.3.1/INSTALL'))
# change to BLAS dir
os.chdir(os.path.join(os.getcwd(), self.lapversion))
# Write Makefile.in
writefile(
'make.inc', """
# -*- Makefile generated by DMFT installer -*-
####################################################################
# LAPACK make include file. #
# LAPACK, Version """ + self.lapversion +
"""" #
# April 2012 #
####################################################################
#
SHELL = /bin/sh
#
# Modify the FORTRAN and OPTS definitions to refer to the
# compiler and desired compiler options for your machine. NOOPT
# refers to the compiler options desired when NO OPTIMIZATION is
# selected. Define LOADER and LOADOPTS to refer to the loader and
# desired load options for your machine.
#
FORTRAN = """ + self.config.fc + """
OPTS = """ + self.config.fflags + """
DRVOPTS = $(OPTS)
NOOPT = -O0
LOADER = """ + self.config.fc + """
LOADOPTS =
MAKE = make -j 8
#
# Timer for the SECOND and DSECND routines
#
# Default : SECOND and DSECND will use a call to the EXTERNAL FUNCTION ETIME
# TIMER = EXT_ETIME
# For RS6K : SECOND and DSECND will use a call to the EXTERNAL FUNCTION ETIME_
# TIMER = EXT_ETIME_
# For gfortran compiler: SECOND and DSECND will use a call to the INTERNAL FUNCTION ETIME
# TIMER = INT_ETIME
# If your Fortran compiler does not provide etime (like Nag Fortran Compiler, etc...)
# SECOND and DSECND will use a call to the INTERNAL FUNCTION CPU_TIME
TIMER = INT_CPU_TIME
# If neither of this works...you can use the NONE value... In that case, SECOND and DSECND will always return 0
# TIMER = NONE
#
# Configuration LAPACKE: Native C interface to LAPACK
# To generate LAPACKE library: type 'make lapackelib'
# Configuration file: turned off (default)
# Complex types: C99 (default)
# Name pattern: mixed case (default)
# (64-bit) Data model: LP64 (default)
#
# CC is the C compiler, normally invoked with options CFLAGS.
#
CC = """ + self.config.cc + """
CFLAGS = """ + self.config.cflags + """
#
# The archiver and the flag(s) to use when building archive (library)
# If you system has no ranlib, set RANLIB = echo.
#
ARCH = ar
ARCHFLAGS= """ + self.config.arflags + """
RANLIB = """ + self.config.ranlib + """
#
# The location of BLAS library for linking the testing programs.
# The target's machine-specific, optimized BLAS library should be
# used whenever possible.
#
BLASLIB = """ + self.config.blaslib + """
#
# Location of the extended-precision BLAS (XBLAS) Fortran library
# used for building and testing extended-precision routines. The
# relevant routines will be compiled and XBLAS will be linked only if
# USEXBLAS is defined.
#
# USEXBLAS = Yes
XBLASLIB =
# XBLASLIB = -lxblas
#
# Names of generated libraries.
#
LAPACKLIB = liblapack.a
TMGLIB = libtmg.a
EIGSRCLIB = libeigsrc.a
LINSRCLIB = liblinsrc.a
LAPACKELIB = liblapacke.a
""")
# compile and generate library
print 'Compile and generate LAPACK...',
sys.stdout.flush()
comm = self.make + ' lapacklib tmglib'
(output, error, retz) = shellcmd(comm)
if retz:
print "\n\nLAPACK: cannot compile LAPACK"
print "stderr:\n", "*" * 50, "\n", comm, '\n', error, "\n", "*" * 50
sys.exit()
log = output + error
# write the log on a file
log = log + output + error
fulllog = os.path.join(savecwd, 'log/log.lapack')
writefile(fulllog, log)
print 'Installation of liblapack.a successful.'
print '(log is in ', fulllog, ')'
# move libcblas.a to the lib directory
shutil.copy('liblapack.a',
os.path.join(self.config.prefix, 'lib/liblapack.a'))
shutil.copy('libtmg.a', os.path.join(self.config.prefix,
'lib/libtmg.a'))
# set framework variables to point to the freshly installed BLAS library
self.config.lapacklib = '-L' + os.path.join(self.config.prefix,
'lib') + ' -ltmg -llapack'
os.chdir(savecwd)
self.config.lapinstalled = 1
# Check if the installation is successful
self.dmft.verbose = 1
ret = self.check_lapack()
self.dmft.verbose = 0
if ret != 0:
sys.exit()
def down_install_gsl(self):
print "The GSL library is being installed."
sys.stdout.flush()
savecwd = os.getcwd()
# creating the build,lib and log dirs if don't exist
if not os.path.isdir(os.path.join(self.config.prefix, 'gsl')):
os.mkdir(os.path.join(self.config.prefix, 'gsl'))
if not os.path.isdir(os.path.join(os.getcwd(), 'log')):
os.mkdir(os.path.join(os.getcwd(), 'log'))
# Check if gsl.tgz is already present in the working dir
# otherwise download it
if not os.path.isfile(
os.path.join(os.getcwd(), getURLName(self.gslurl))):
print "Downloading GSL ...",
#downloader(self.lapackurl,self.downcmd)
#urllib.urlretrieve(self.gslurl, "gsl.tgz")
geturl(self.gslurl, "gsl.tgz")
print "done"
# unzip and untar
os.chdir('download')
print 'Unzip and untar GSL...',
comm = 'tar zxf gsl.tgz '
(output, error, retz) = shellcmd(comm)
if retz:
print '\n\nlibgsl: cannot unzip ' + self.gslversion + '.tgz'
print 'stderr:\n', '*' * 50, '\n', comm, '\n', error, '\n', '*' * 50
sys.exit()
print 'done'
# change to GSL dir
os.chdir(os.path.join(os.getcwd(), self.gslversion))
# compile and generate library
print 'Configure GSL...',
sys.stdout.flush()
comm = './configure CC=' + self.config.cc + ' --prefix=' + os.path.join(
self.config.prefix, 'gsl')
(output, error, retz) = shellcmd(comm)
if retz:
print "\n\nlingsl: cannot configure GSL"
print "stderr:\n", "*" * 50, "\n", comm, '\n', error, "\n", "*" * 50
sys.exit()
log = output + error
# write log on a file
log = log + output + error
fulllog = os.path.join(savecwd, 'log/log.gsl')
writefile(fulllog, log)
print 'Configuration of GSL successful.'
print '(log is in ', fulllog, ')'
# compile and generate library
print 'Compile and generate GSL...',
sys.stdout.flush()
comm = self.make + ' -j4; ' + self.make + ' install'
(output, error, retz) = shellcmd(comm)
if retz:
print "\n\nlingsl: cannot compile GSL"
print "stderr:\n", "*" * 50, "\n", comm, '\n', error, "\n", "*" * 50
sys.exit()
log = output + error
# write the log on a file
log = log + output + error
fulllog = os.path.join(savecwd, 'log/log.gsl')
writefile(fulllog, log)
print 'Installation of GSL successful.'
print '(log is in ', fulllog, ')'
# move libcblas.a to the lib directory
#shutil.copy('libtmg.a',os.path.join(self.config.prefix,'gsl/libtmg.a'))
# set framework variables to point to the freshly installed GSL library
self.config.gsl = '-L' + os.path.join(self.config.prefix,
'gsl') + ' -lgsl '
os.chdir(savecwd)
# Check if the installation is successful
self.dmft.verbose = 1
# self.check_gsl()
self.dmft.verbose = 0
from time import sleep
import sys
import socket
import argparse
import logging
# by kzl
from settings import mylogger,NOT_EXIST,ACCESS_DENIED,ALREADY_EXIST,SUCCESS,PORT,SHARED_FOLDER,SERVER_START_TIME,SECRET_LENGTH,IPS_FILE,URL_PREFIX,UPDATE_INTERVAL
from utils import random_string,get_lan_ip,geturl
from p2p_server import Node
from threads import UpdateGUIListTimer
# gui
from PyQt4 import QtGui,QtCore
from settings import WIN_WIDTH,WIN_HEIGHT,ICON_APP,ICON_FETCH,ICON_QUIT
IP_LAN = get_lan_ip()
SERVER_URL = geturl(URL_PREFIX,IP_LAN,PORT)
class NodeServerThread(Thread):
"""
thread for starting and stopping node server
"""
def __init__(self,name,url,dirname,secret,ipsfile,event_running):
mylogger.info('[__init__]: {0}'.format(name))
super(NodeServerThread,self).__init__()
self.name = name
self.daemon = True
self.url = url
self.dirname = dirname
self.secret = secret
self.ipsfile = ipsfile
self.event_running = event_running
def down_install_fftw(self):
print """
The FFTW library is being installed.
"""
sys.stdout.flush()
savecwd = os.getcwd()
# creating the build,lib and log dirs if don't exist
if not os.path.isdir(os.path.join(self.config.prefix, 'fftw')):
os.mkdir(os.path.join(self.config.prefix, 'fftw'))
if not os.path.isdir(os.path.join(os.getcwd(), 'log')):
os.mkdir(os.path.join(os.getcwd(), 'log'))
# Check if fftw.tgz is already present in the working dir
# otherwise download it
if not os.path.isfile(os.path.join(os.getcwd(), "fftw.tgz")):
print "Downloading FFTW ...",
#downloader(self.lapackurl,self.downcmd)
#urllib.urlretrieve(self.fftwurl, "fftw.tgz")
geturl(self.fftwurl, "fftw.tgz")
print "done"
# unzip and untar
os.chdir('download')
print 'Unzip and untar FFTW...',
comm = 'tar zxf fftw.tgz '
(output, error, retz) = shellcmd(comm)
if retz:
print '\n\nlibfftw: cannot unzip ' + self.fftwversion + '.tgz'
print 'stderr:\n', '*' * 50, '\n', comm, '\n', error, '\n', '*' * 50
sys.exit()
print 'done'
# change to FFTW dir
os.chdir(os.path.join(os.getcwd(), self.fftwversion))
# compile and generate library
print 'Configure FFTW...',
sys.stdout.flush()
if (self.config.ompflag == ""):
comm = './configure MPICC='+self.config.pcc+' CC='+self.config.cc+' F77='+self.config.fc+\
' --enable-mpi --enable-threads --enable-shared --prefix='+os.path.join(self.config.prefix,'fftw')
else:
comm = './configure MPICC='+self.config.pcc+' CC='+self.config.cc+' F77='+self.config.fc+\
' --enable-mpi --enable-threads --enable-shared -enable-openmp --prefix='+os.path.join(self.config.prefix,'fftw')
(output, error, retz) = shellcmd(comm)
if retz:
print "\n\nlinfftw: cannot configure FFTW"
print "stderr:\n", "*" * 50, "\n", comm, '\n', error, "\n", "*" * 50
sys.exit()
log = output + error
# write log on a file
log = log + output + error
fulllog = os.path.join(savecwd, 'log/log.fftw')
writefile(fulllog, log)
print 'Configuration of FFTW successful.'
print '(log is in ', fulllog, ')'
# compile and generate library
print 'Compile and generate FFTW...',
sys.stdout.flush()
comm = self.make + ' -j4; ' + self.make + ' install'
(output, error, retz) = shellcmd(comm)
if retz:
print "\n\nlinfftw: cannot compile FFTW"
print "stderr:\n", "*" * 50, "\n", comm, '\n', error, "\n", "*" * 50
sys.exit()
log = output + error
# write the log on a file
log = log + output + error
fulllog = os.path.join(savecwd, 'log/log.fftw')
writefile(fulllog, log)
print 'Installation of FFTW successful.'
print '(log is in ', fulllog, ')'
# move libcblas.a to the lib directory
#shutil.copy('libtmg.a',os.path.join(self.config.prefix,'fftw/libtmg.a'))
self.set_fftlib()
os.chdir(savecwd)
# Check if the installation is successful
self.dmft.verbose = 1
self.check_fftw()
self.dmft.verbose = 0
