def __crawljobs(self, keyword, city=u'全国'):
page_num = 1
url = u'http://www.lagou.com/jobs/positionAjax.json?px=default&city=%s&needAddtionalResult=false' % (city,)
page_size = sys.maxint
errors = []
while page_num < page_size:
data = u'first=false&pn=%d&kd=%s' % (page_num, keyword)
cache, content = Downloader.get_html(url, data, city)
if content:
try:
result = json.loads(content)
page_no = result['content']['pageNo']
if page_no > 0:
print '{kv}: {page_num}'.format(kv=keyword, page_num=page_num)
page_num += 1
if not cache:
# 每个网络请求间隔一秒
sleep(0.1)
else:
# 为空,说明最后一页
break
except Exception, e:
page_num += 1
# 发生异常, 删除缓存文件
Downloader.remove_file(url, data, city)
errors.append((1, url, data, city, e))
else:
# 请求结果为空
errors.append((2, url, data, city, e))
def applyGloablBlackList(cfg_params):
removeBList = cfg_params.get("GRID.remove_default_blacklist", 0 )
blackAnaOps = None
if int(removeBList) == 0:
try:
blacklist = Downloader("http://cmsdoc.cern.ch/cms/LCG/crab/config/")
result = blacklist.config("site_black_list.conf")
except:
msg = "ERROR talking to cmsdoc\n%s"%str(sys.exc_info()[1])
common.logger.info(msg)
common.logger.info("WARNING: Could not download default site black list")
result = []
if result :
blackAnaOps = result.strip()
common.logger.debug("Enforced black list: %s "%blackAnaOps)
else:
common.logger.info("WARNING: Skipping default black list!")
if blackAnaOps:
blackUser = cfg_params.get("GRID.se_black_list", '' )
if blackUser:
cfg_params['GRID.se_black_list'] = blackUser + ',' + blackAnaOps
else:
cfg_params['GRID.se_black_list'] = blackAnaOps
msg = "PSN black list: %s" % cfg_params['GRID.se_black_list']
common.logger.info(msg)
def configure(self, cfg_params):
self.cfg_params = cfg_params
self.jobtypeName = cfg_params.get('CRAB.jobtype','')
self.schedulerName = cfg_params.get('CRAB.scheduler','')
Scheduler.configure(self,cfg_params)
#self.proxyValid=0
#self.dontCheckProxy=int(cfg_params.get("GRID.dont_check_proxy",0))
self.space_token = cfg_params.get("USER.space_token",None)
self.proxyServer= 'myproxy.cern.ch'
#self.group = cfg_params.get("GRID.group", None)
#self.role = cfg_params.get("GRID.role", None)
removeBList = cfg_params.get("GRID.remove_default_blacklist", 0 )
blackAnaOps = None
if int(removeBList) == 0:
blacklist = Downloader("http://cmsdoc.cern.ch/cms/LCG/crab/config/")
result = blacklist.config("site_black_list.conf")
if result != None:
blackAnaOps = result
common.logger.debug("Enforced black list: %s "%blackAnaOps)
else:
common.logger.info("WARNING: Skipping default black list!")
self.EDG_ce_black_list = None
if cfg_params.has_key('GRID.ce_black_list') and cfg_params['GRID.ce_black_list']:
self.EDG_ce_black_list = cfg_params.get('GRID.ce_black_list')
if int(removeBList) == 0 and blackAnaOps:
self.EDG_ce_black_list += ",%s"%blackAnaOps
elif int(removeBList) == 0 and blackAnaOps:
self.EDG_ce_black_list = blackAnaOps
if self.EDG_ce_black_list:
self.EDG_ce_black_list = str(self.EDG_ce_black_list).split(',')
self.EDG_ce_white_list = cfg_params.get('GRID.ce_white_list',None)
if (self.EDG_ce_white_list): self.EDG_ce_white_list = str(self.EDG_ce_white_list).split(',')
self.VO = cfg_params.get('GRID.virtual_organization','cms')
self.EDG_clock_time = cfg_params.get('GRID.max_wall_clock_time',None)
# Default minimum CPU time to >= 130 minutes
self.EDG_cpu_time = cfg_params.get('GRID.max_cpu_time', '130')
## Add EDG_WL_LOCATION to the python path
#if not self.CRAB_useServer and not self.CRAB_serverName:
# if not os.environ.has_key('EDG_WL_LOCATION'):
# msg = "Error: the EDG_WL_LOCATION variable is not set."
# raise CrabException(msg)
# path = os.environ['EDG_WL_LOCATION']
# libPath=os.path.join(path, "lib")
# sys.path.append(libPath)
# libPath=os.path.join(path, "lib", "python")
# sys.path.append(libPath)
self.checkProxy()
return
def runProgram(youtube_search_query, text_to_find):
dl = Downloader()
celeb_videos = Search.search(youtube_search_query)
links = [Caption.YOUTUBE_LINK + x for x in celeb_videos]
print links
count = 0
for videoid in celeb_videos:
cap = Caption(videoid)
results = cap.find(text_to_find)
for result in results:
dl.download(result, text_to_find + str(count))
count += 1
def manageCredential(self):
"""
Prepare configuration and Call credential API
"""
common.logger.info("Registering credential to the server : %s"%self.server_name)
try:
myproxyserver = Downloader("http://cmsdoc.cern.ch/cms/LCG/crab/config/").config("myproxy_server.conf")
myproxyserver = myproxyserver.strip()
if myproxyserver is None:
raise CrabException("myproxy_server.conf retrieved but empty")
except Exception, e:
common.logger.info("Problem setting myproxy server endpoint: using myproxy.cern.ch")
common.logger.debug(e)
myproxyserver = 'myproxy.cern.ch'
def insert_jobs(self):
errors = []
types = self.get_types()
for keyword in types:
page_num = 1
city = u'全国'
url = u'http://www.lagou.com/jobs/positionAjax.json?px=default&city=%s&needAddtionalResult=false' % (city,)
page_size = sys.maxint
while page_num < page_size:
data = u'first=false&pn=%d&kd=%s' % (page_num, keyword)
cache, content = Downloader.get_html(url, data, city)
if content:
try:
result = json.loads(content)
pos_result = result['content']['positionResult']
jobs = pos_result['result']
if len(jobs) > 0:
self.db.insert_jobs(jobs)
print data
else:
# 为空说明是最后一页
errors.append('result empty {0} {1}'.format(url, data, content))
break
page_num += 1
except Exception, e:
page_num += 1
# 发生异常
errors.append((1, url, data, city, e))
else:
# 请求结果为空
errors.append((2, url, data, city, e))
def craw(self, root_url):
# todo: 层数设置？
page_count = self.page_count
UrlManager.add_new_url(root_url)
try:
while UrlManager.has_new_url():
todo_url = UrlManager.get_new_url()
try:
print u"\nNO: %d 正在检测链接:%s" % (page_count, todo_url)
except:
print u"出错", todo_url
# todo: 换成 contiune 是不是更好？
# page_status, bad_links = Downloader.test(todo_url)
# print u"\n结果：%s" % page_status
# todo: 加入状态码的说明
state, content = Downloader.download(todo_url)
if state:
prase_state, new_urls = Parser.parse(todo_url, content)
if prase_state:
UrlManager.add_new_urls(new_urls)
else:
Outputer.collect_data(new_urls)
else:
Outputer.collect_data(content)
page_count += 1
# except:
# print u"页面爬取失败"
# UrlManager.add_wrong_url(todo_url)
# todo:测试代码
if page_count == 5000:
self.page_count = page_count
break
print UrlManager.num_new_url()
finally:
Outputer.output_txt(self.page_count)
def infomation_crew(self):
# 一条一条取
total_count = 20000
i = 0
while i < total_count:
i += 1
# 去除诗词的详情页url
info = self.db.select_unanalyzed_infomation()
if info is not None:
# 下载分析
url = info['url']
html = Downloader.get_html(url, 'infomation')
if html:
content = Analyzer.get_info_detail(html)
if content:
self.db.update_infomationurl(url, content)
print '%d/%d %s %s' % (i, total_count, info['title'], url)
else:
self.db.insert_error('analyze_info_detail_error', 7, 'reason', url)
else:
self.db.insert_error('download_info_detail_error', 6, 'reason', url)
else:
# 没有了
return
def _createXMLcommand(
self,
taskUName,
cmdSpec="status",
rng="all",
newTaskAddIns=False,
flavour="analysis",
type="fullySpecified",
jobs="-1",
):
xmlString = ""
cfile = minidom.Document()
ver = common.prog_version_str
node = cfile.createElement("TaskCommand")
node.setAttribute("Task", str(taskUName))
node.setAttribute("Subject", str(self.userSubj))
node.setAttribute("Command", str(cmdSpec))
node.setAttribute("Range", str(rng))
node.setAttribute("TotJob", str(jobs))
node.setAttribute("Scheduler", str(self.cfg_params["CRAB.scheduler"]))
node.setAttribute("Flavour", str(flavour))
node.setAttribute("Type", str(type))
node.setAttribute("ClientVersion", str(ver))
## Only Temporary. it should be at Server level
removeBList = self.cfg_params.get("GRID.remove_default_blacklist", 0)
blackAnaOps = "None.Nowhere"
if int(removeBList) == 0:
blacklist = Downloader("http://cmsdoc.cern.ch/cms/LCG/crab/config/")
result = blacklist.config("site_black_list.conf")
if result != None:
blackAnaOps = result
common.logger.debug("Enforced black list: %s " % str(blacklist))
else:
common.logger.info("WARNING: Skipping default black list!")
try:
myproxyserver = Downloader("http://cmsdoc.cern.ch/cms/LCG/crab/config/").config("myproxy_server.conf")
myproxyserver = myproxyserver.strip()
if myproxyserver is None:
raise CrabException("myproxy_server.conf retrieved but empty")
except Exception, e:
common.logger.info("Problem setting myproxy server endpoint: using myproxy.cern.ch")
common.logger.debug(e)
myproxyserver = "myproxy.cern.ch"
def __init__(self, serverName):
import string
self.serverName = string.lower(serverName)
common.logger.debug('Calling ServerConfig '+self.serverName)
url ='http://cmsdoc.cern.ch/cms/LCG/crab/config/'
self.downloader = Downloader(url)
def rb_configure(self, RB):
url ='http://cmsdoc.cern.ch/cms/LCG/crab/config/'
from Downloader import Downloader
import httplib
common.logger.debug('Downloading config files for WMS: '+url)
## 25-Jun-2009 SL: patch to use Cream enabled WMS
if ( self.cfg_params.get('GRID.use_cream',None) ):
RB='CREAM'
if not RB: return None
rb_param_file = None
configFileName = 'glite_wms_'+str(RB)+'.conf'
results = Downloader(url)
try:
gliteConfig = results.filePath(configFileName)
except httplib.HTTPException, ex:
raise CrabException( "Problem getting RB config file: %s, reason:"%(configFileName, ex) )
def poem_list_crew(self):
for i in range(1, 6):
url = 'http://www.haoshiwen.org/type.php?x=%d' % i
content = Downloader.get_html(url, 'poemlist')
if content:
page_count = Analyzer.get_page_count(content)
# 分析
for j in range(1, page_count + 1):
page_url = 'http://www.haoshiwen.org/type.php?x=%d&page=%d' % (i, j)
# 入库
self.db.insert_url(page_url, 1)
# 判断是否分析过
if self.db.url_analyzed(page_url):
pass
else:
content = Downloader.get_html(page_url, 'poemlist')
if content:
# 分析诗的列表
poems = Analyzer.get_poems_from_list_page(content)
if poems:
# 入库
self.db.insert_urls(poems, 2)
self.db.update_url(page_url)
print '%d %d/%d: %s' % (i, j, page_count, page_url)
else:
if Analyzer.check_poem_list_last_page(content):
# 最后一页
break
else:
print u'分析失败'
self.db.insert_error('analyze_poem_list_error', 3, 'reason', page_url)
# 错误入库：analyze_poem_list_error
else:
print u'获取页面诗词列表错误'
self.db.insert_error('get_poem_list_error', 2, 'reason', page_url)
# 错误入库：get_poem_list_error
else:
print u'分析首页失败'
self.db.insert_error('analyze_poem_list_first_page_error', 1, 'reason', page_url)
def parseAnswerServer(self, updateInfo):
updateDict = parse_qs(updateInfo, strict_parsing=True)
# updateDict has: mustUpdate, urlUser, newVersionUser, urlAdmin,
# newVersionAdmin
updates = dict() # updates has: mustUpdate, userFile, adminFile
mustUpdate = int(updateDict['mustUpdate'][0])
updates['mustUpdate'] = mustUpdate
downloader = Downloader()
location = "../../persistent/downloads"
if mustUpdate == 0:
logger.info('No update available')
elif (mustUpdate & UPDATE_ADMIN_MODE):
adminURL = updateDict['urlAdmin'][0]
logger.info('Downloading Admin update: %s' % adminURL)
adminFile = downloader.downloadUpdate(location, adminURL)
updates['adminFile'] = adminFile
logger.info('Administrator update is available called: %s' %
adminFile)
if DISPLAY_GUI_MESSAGES and not(checkFiles.checkIfAdmin()):
root = Tk()
root.title('HiSPARC')
Message(root, anchor='s', text="Update is available requiring "
"administrator rights!\nPlease ask your administrator "
"to reboot and install it!").pack(padx=150, pady=100)
root.mainloop()
elif (mustUpdate & UPDATE_USER_MODE):
userURL = updateDict['urlUser'][0]
logger.info('Downloading User update: %s' % userURL)
userFile = downloader.downloadUpdate(location, userURL)
updates['userFile'] = userFile
logger.info('User update is available called: %s' % userFile)
# Run the update to install it.
# First call a batch file so that Python can be closed.
os.system(".\\runUserUpdate.bat %s" % userFile)
return updates
class Dispatcher(object):
""" Dispatcher class to control process of working. """
def __init__(self):
create_or_check_path("logs/")
fileConfig('config/logger.conf')
self.database = Database()
self.parser = Parser()
self.event_processor = EventProcessor()
self.downloader = Downloader()
self.new_events_file = None
self.new_data = None
self.new_event = None
def processor(self):
""" Process of downloading, parsing and saving information. """
download_status = True
while download_status:
getting_event_status = self.get_event()
if getting_event_status:
processing_event_status = self.process_event()
if processing_event_status:
self.process_data()
else:
download_status = self.download()
def get_event(self):
""" Asks Parser for a new event.
:return: False if getting event failed else True
"""
self.new_event = self.parser.get_event(self.new_events_file)
return False if self.new_event is None else True
def download(self):
""" Sends a request to Downloader.
:return: False if downloading failed else True
"""
self.new_events_file = self.downloader.download_archive()
return False if self.new_events_file is None else True
def process_event(self):
""" Asks EventProcessor to get data from event.
:return: False if processing event failed else True
"""
self.new_data = self.event_processor.process_event(self.new_event)
return False if self.new_data is None else True
def process_data(self):
""" Sends data to Database. """
self.database.process_data(self.new_data)
def poem_type_crew(self):
url = 'http://www.haoshiwen.org/type.php'
content = Downloader.get_html(url, 'poemlist')
if content:
result = Analyzer.get_poem_types(content)
k = 1
for i in result:
# 只处理类型
if i[0] == u'类型':
for j in i[1]:
self.db.insert_poem_types(k, i[0], j[0], j[1])
print '\t' + j[0]
k += 1
def __init__(self):
create_or_check_path("logs/")
fileConfig('config/logger.conf')
self.database = Database()
self.parser = Parser()
self.event_processor = EventProcessor()
self.downloader = Downloader()
self.new_events_file = None
self.new_data = None
self.new_event = None
def main():
log = setup_logging("sentinal.log")
parser = argparse.ArgumentParser(description='Grabs sentinal images from the ESA stite',
epilog='-h for help',
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('-c', action="store_true", help='create db table only', dest='create_db')
parser.add_argument('-C', type=str, help='path to the configuration file', dest='configuration_file')
parser.add_argument('-f', action="store_true", help='force', dest='force')
parser.add_argument('-v', action="store_true", help='run verbosely', dest='verbose')
parser.add_argument('-o', action="store_true", help='overwrite data .zip file even if it exists', dest='overwrite')
args = parser.parse_args()
db = Database(args)
if args.create_db:
db.createDb(args.verbose)
sys.exit(0)
config = ConfigManager(args.configuration_file,log)
auth = config.getAuth()
if not len(auth):
log.error('Missing ESA SCIHUB authentication information')
sys.exit(7)
types = config.getTypes()
polygons = config.getPolygons()
if len(types) != len(polygons):
log.error( 'Incorrect number of polygons and types in configuration file' )
sys.exit(6)
downloader = Downloader(args, config, db, log)
downloader.getProducts()
downloader.downloadProducts()
def configure(self, cfg_params):
self.cfg_params = cfg_params
self.jobtypeName = cfg_params.get('CRAB.jobtype','')
self.schedulerName = cfg_params.get('CRAB.scheduler','')
Scheduler.configure(self,cfg_params)
self.proxyValid=0
self.dontCheckProxy=int(cfg_params.get("GRID.dont_check_proxy",0))
self.space_token = cfg_params.get("USER.space_token",None)
try:
self.proxyServer = Downloader("http://cmsdoc.cern.ch/cms/LCG/crab/config/").config("myproxy_server.conf")
self.proxyServer = self.proxyServer.strip()
if self.proxyServer is None:
raise CrabException("myproxy_server.conf retrieved but empty")
except Exception, e:
common.logger.info("Problem setting myproxy server endpoint: using myproxy.cern.ch")
common.logger.debug(e)
self.proxyServer= 'myproxy.cern.ch'
def poem_detail_crew(self):
# update urls set analyzed = 1 where url = 'http://www.haoshiwen.org/view.php?id=9510'
# 一条一条取
total_count = 10
c = 1
while c < total_count:
c += 1
continue
# 去除诗词的详情页url
url = self.db.select_unanalyzed_url(2)
if url is not None:
url = url['url']
# 判断是否抓取过
if not self.db.poem_exists(url):
content = Downloader.get_html(url, 'poem')
if content:
try:
poem = Analyzer.get_poem_detail(content, url)
if poem:
# 分析成功,入库
poem_content = poem[0]
poem_info = poem[1]
self.db.insert_poem(poem_content)
self.db.insert_infomations(url, 1, poem_info)
self.db.update_url(url)
print "%d/%d %s %s" % (c, total_count, poem_content['title'], poem_content['url'])
else:
self.db.insert_error('analyze_poem_detail_error', 5, 'reason', url)
except Exception, e:
continue
else:
self.db.insert_error('get_poem_detail_error', 4, 'reason', url)
else:
self.db.update_url(url)
else:
count = total_count
def poem_author_crew(self):
# 一条一条取
total_count = 20000
i = 0
while i < total_count:
i += 1
# 去除诗词的详情页url
authors = self.db.select_authors()
cou = len(authors)
i = 1
for author in authors:
url = author[0]
if url is not None and url is not '':
# 判断是否存在
if not self.db.author_exists(url):
html = Downloader.get_html(url, 'author')
if html:
try:
author_info = Analyzer.get_author_detail(html, url)
if author_info:
author_content = author_info[0]
author_infos = author_info[1]
self.db.insert_author(author_content)
self.db.insert_infomations(url, 2, author_infos)
print '%d/%d %s %s' % (i, cou, author_content['name'], url)
i += 1
else:
self.db.insert_error('analyze_author_detail_error', 9, 'reason', url)
except Exception, e:
print 'error %s' % (url,)
else:
self.db.insert_error('download_author_detail_error', 8, 'reason', url)
else:
# 没有了
return
def poem_type_poem_list_craw(self):
types = self.db.get_poem_types()
for i in types:
j = 1
while True:
page_url = i['url'] + "&page=" + str(j)
# 入库
self.db.insert_url(page_url, 3)
# 判断是否分析过
if self.db.url_analyzed(page_url):
j += 1
continue
else:
content = Downloader.get_html(page_url, 'poemlist')
if content:
# 分析诗的列表
poems = Analyzer.get_poems_from_list_page(content)
if poems:
# 入库
self.db.insert_type_poems(i['id'], poems)
self.db.update_url(page_url)
print '%d: %s' % (j, page_url)
j += 1
else:
if Analyzer.check_poem_list_last_page(content):
# 最后一页
break
else:
print u'分析失败'
self.db.insert_error('analyze_poem_list_error', 3, 'reason', page_url)
# 错误入库：analyze_poem_list_error
else:
print u'获取页面诗词列表错误'
self.db.insert_error('get_poem_list_error', 2, 'reason', page_url)
def MainProcess(logger, uperList, saveRootPath, concurrency=3):
pp = None
try:
# --------------------------------------------------------------
# 进行每个 UP 主视频页数的获取
pp = PreProcess(logger=logger, uperList=uperList)
pp.ScanLoclInfo(saveRootPath)
pp.Process()
# --------------------------------------------------------------
# 爬取要下载视频的 url
for uper in pp.uperList:
logger.info(uper.UserName + " Spider Start···")
OneSpiderRetryTimes = 0
# 打算下载的数量，要去网络动态获取的数量进行对比
while ((uper.NeedDownloadFilmCount > len(
uper.VideoInfoDic_NetFileName)
or len(uper.ErrorUrl_Dic) > 0)
and OneSpiderRetryTimes <= 10):
# dd = BiliSpider()
# GithubDeveloperSpider
BiliSpider.start(logger=logger,
uper=uper,
saveRootPath=saveRootPath,
concurrency=concurrency,
middleware=middleware)
OneSpiderRetryTimes = OneSpiderRetryTimes + 1
logger.info("Try Spider " + uper.UserName + " " +
str(OneSpiderRetryTimes) + " times.")
RandomSleep()
logger.info(uper.UserName + " Spider Done.")
if OneSpiderRetryTimes > 10:
logger.error(uper.UserName + " Spider Retry " +
str(OneSpiderRetryTimes) + "times.")
logger.error("Error Url:")
for eUrl in uper.ErrorUrl_Dic:
logger.error(eUrl)
else:
# 本地应该原有+准备要下载的 != 网络总数，需要提示
if len(uper.VideoInfoDic_NetFileName) != len(
uper.VideoInfoDic_loaclFileName):
logger.warn("VideoInfoDic_NetFileName Count: " +
str(len(uper.VideoInfoDic_NetFileName)) +
" != VideoInfoDic_loaclFileName Count: " +
str(len(uper.VideoInfoDic_loaclFileName)))
uper.ErrorUrl_Dic.clear()
logger.info("Spider All Done.")
# --------------------------------------------------------------
logger.info("Start Download" + "----" * 20)
# 开始下载
# 先对 local 与 net 的字典进行同步
logger.info("Start Sync Dic")
for uper in pp.uperList:
iNeedDl = 0
for fileName, oneVideo in zip(
uper.VideoInfoDic_loaclFileName.keys(),
uper.VideoInfoDic_loaclFileName.values()):
# 匹配到对应的项目才进行处理
findList = list(
filter(lambda d: d.split('_')[1] == fileName,
uper.VideoInfoDic_NetFileName.keys()))
if any(findList):
uper.VideoInfoDic_NetFileName[
findList[0]].isDownloaded = oneVideo.isDownloaded
if oneVideo.isDownloaded == False:
iNeedDl = iNeedDl + 1
logger.info(uper.UserName + "NetFile / LocalFile -- NeedDl: " +
str(len(uper.VideoInfoDic_NetFileName)) + " / " +
str(len(uper.VideoInfoDic_loaclFileName)) + " -- " +
str(iNeedDl))
logger.info("End Sync Dic")
for uper in pp.uperList:
directory = os.path.join(saveRootPath, uper.UserName)
for fileName, oneVideo in zip(
uper.VideoInfoDic_NetFileName.keys(),
uper.VideoInfoDic_NetFileName.values()):
if oneVideo.isDownloaded == True:
continue
DownloadRetryTimes = 0
oneRe = False
while oneRe is False and DownloadRetryTimes <= 10:
oneRe = Downloader(logger, directory, oneVideo.time,
oneVideo.title,
oneVideo.url).ProcessOne()
DownloadRetryTimes = DownloadRetryTimes + 1
logger.info("Try Download " + str(DownloadRetryTimes) +
" times.")
RandomSleep()
if OneSpiderRetryTimes > 10:
logger.error("Retry Download " + str(DownloadRetryTimes) +
" times.")
logger.error("Error Url: " + oneVideo.url)
# 标记下载完成
if oneRe:
oneVideo.isDownloaded = True
uper.ThisTimeDownloadCount = uper.ThisTimeDownloadCount + 1
except Exception as ex:
errInfo = "Catch Exception: " + str(ex)
logger.error(errInfo)
finally:
logger.info("finally" + "----" * 20)
for uper in pp.uperList:
logger.info("This Time Download: " + uper.UserName + " -- " +
str(uper.ThisTimeDownloadCount))
for uper in pp.uperList:
for fileName, oneVideo in zip(
uper.VideoInfoDic_NetFileName.keys(),
uper.VideoInfoDic_NetFileName.values()):
if oneVideo.isDownloaded == False:
logger.error('Download Fail:' + uper.UserName)
logger.error(oneVideo.url)
logger.info("This Time Done.")
def download_callback(params):
downloader = Downloader(params['url'])
downloader.start()
class MainWindow( QtGui.QMainWindow ):
## Constructeur
# Le constructeur va creer la fenetre principale en y ajoutant tous les widgets necessaires au programme
def __init__( self ):
# Appel au constructeur de la classe mere
QtGui.QMainWindow.__init__( self )
###########
# Fenetre #
###########
###
# Reglages de la fenetre principale
###
# Nom de la fenetre
self.setWindowTitle( "TVDownloader" )
# Mise en place de son icone
self.setWindowIcon( QtGui.QIcon( "ico/TVDownloader.png" ) )
###
# Mise en place des widgets dans la fenetre
###
# Widget central qui contiendra tout
self.centralWidget = QtGui.QWidget( self )
#
# Barre du haut
#
# Layout horizontal qui contiendra les listes deroulantes
self.horizontalLayoutBarreHaut = QtGui.QHBoxLayout()
# Liste deroulante pour choisir le site (plugin)
self.comboBoxSite = QtGui.QComboBox( self.centralWidget )
self.horizontalLayoutBarreHaut.addWidget( self.comboBoxSite )
# Liste deroulante pour choisir une chaine du site courant
self.comboBoxChaine = QtGui.QComboBox( self.centralWidget)
self.horizontalLayoutBarreHaut.addWidget( self.comboBoxChaine )
# Liste deroulante pour choisir une emission de la chaine courante
self.comboBoxEmission = QtGui.QComboBox( self.centralWidget )
self.horizontalLayoutBarreHaut.addWidget( self.comboBoxEmission )
#
# Onglets
#
# Gestionnaire onglets
self.tabWidget = QtGui.QTabWidget( self.centralWidget )
# Onglet Fichiers
self.tabFichiers = QtGui.QSplitter( self.centralWidget ) # L'onglet Fichier contient un splitter
self.tabWidget.addTab( self.tabFichiers, u"Choix des fichiers" )
# Onglet Telechargements
self.tabTelechargements = QtGui.QWidget( self.centralWidget )
self.tabWidget.addTab( self.tabTelechargements, u"Téléchargements" )
#
# Liste des fichiers
#
# Layout de grille qui contient le tableau qui liste les fichiers + boutons
self.gridLayoutFichiers = QtGui.QGridLayout( self.tabFichiers )
# Tableau qui contient la liste des fichiers disponibles pour l'emission courante
self.tableWidgetFichier = MyQTableWidget( self.tabFichiers )
# Il a 4 colonnes et 0 ligne (pour l'instant)
self.tableWidgetFichier.setColumnCount( 3 )
self.tableWidgetFichier.setRowCount( 0 )
# On ajoute les titres
self.tableWidgetFichier.setHorizontalHeaderItem( 0,
self.tableWidgetFichier.creerItem( "" ) )
self.tableWidgetFichier.setHorizontalHeaderItem( 1,
self.tableWidgetFichier.creerItem( "Date" ) )
self.tableWidgetFichier.setHorizontalHeaderItem( 2,
self.tableWidgetFichier.creerItem( "Emission" ) )
# On l'ajoute au layout
self.gridLayoutFichiers.addWidget( self.tableWidgetFichier, 0, 1, 6, 1 )
# Icones du tableWidget
self.iconeFichier = QtGui.QIcon( "ico/gtk-file.svg" )
self.iconeAjoute = QtGui.QIcon( "ico/gtk-add.svg" )
self.iconeTelecharge = QtGui.QIcon( "ico/gtk-apply.svg" )
# Bouton pour ajouter tous les fichiers a la liste des telechargements
self.pushButtonToutAjouter = MyQPushButton( self.tabFichiers )
self.pushButtonToutAjouter.setIcon( QtGui.QIcon( "ico/gtk-add.svg" ) )
self.pushButtonToutAjouter.setToolTip( u"Ajouter tous les fichiers à la liste des téléchargements" )
self.gridLayoutFichiers.addWidget( self.pushButtonToutAjouter, 0, 0, 2, 1 )
# Bouton pour rafraichir le plugin courant
self.pushButtonRafraichirPlugin = MyQPushButton( self.tabFichiers )
self.pushButtonRafraichirPlugin.setIcon( QtGui.QIcon( "ico/gtk-refresh.svg" ) )
self.pushButtonRafraichirPlugin.setToolTip( "Rafraichir le plugin" )
self.gridLayoutFichiers.addWidget( self.pushButtonRafraichirPlugin, 2, 0, 2, 1 )
# Bouton pour ouvrir la fenetre des preferences du plugin courant
self.pushButtonPreferencesPlugin = MyQPushButton( self.tabFichiers )
self.pushButtonPreferencesPlugin.setIcon( QtGui.QIcon( "ico/gtk-preferences.svg" ) )
self.pushButtonPreferencesPlugin.setToolTip( u"Ouvrir les préférences du plugin" )
self.gridLayoutFichiers.addWidget( self.pushButtonPreferencesPlugin, 4, 0, 2, 1 )
# On met en place ce layout sur un widget (pour le splitter)
self.widgetFichiers = QtGui.QWidget()
self.widgetFichiers.setLayout( self.gridLayoutFichiers )
#
# Descriptif des fichiers
#
# Layout de grille
self.gridLayoutDescriptif = QtGui.QGridLayout()
# Label pour afficher un logo
self.logoFichierDefaut = QtGui.QPixmap()
self.logoFichierDefaut.load( "img/gtk-dialog-question.svg" )
self.labelLogo = QtGui.QLabel( self.centralWidget )
self.labelLogo.setPixmap( self.logoFichierDefaut.scaled( QtCore.QSize( 150, 150 ), QtCore.Qt.KeepAspectRatio ) )
self.gridLayoutDescriptif.addWidget( self.labelLogo, 0, 0, 1, 1 )
# Zone de texte pour afficher un descriptif
self.plainTextEdit = QtGui.QPlainTextEdit( self.centralWidget )
self.gridLayoutDescriptif.addWidget( self.plainTextEdit, 0, 1, 1, 2 )
# On met en place ce layout sur un widget (pour le splitter)
self.widgetDescriptif = QtGui.QWidget()
self.widgetDescriptif.setLayout( self.gridLayoutDescriptif )
# Onrientation verticale du splitter
self.tabFichiers.setOrientation( QtCore.Qt.Vertical )
# On ajoute les 2 elements au splitter (qui est notre onglet)
self.tabFichiers.addWidget( self.widgetFichiers )
self.tabFichiers.addWidget( self.widgetDescriptif )
#
# Liste des telechargements
#
# Layout de grille qui contient le tableau qui liste les fichiers a telecharger + les boutons pour le controller
self.gridLayoutTelechargement = QtGui.QGridLayout( self.tabTelechargements )
# Tableau qui contient la liste des fichiers a telecharger
self.tableWidgetTelechargement = MyQTableWidget( self.tabTelechargements )
# Il a 5 colonnes et 0 ligne (pour l'instant)
self.tableWidgetTelechargement.setColumnCount( 3 )
self.tableWidgetTelechargement.setRowCount( 0 )
# On ajoute le titre des 5 colonnes
self.tableWidgetTelechargement.setHorizontalHeaderItem( 0,
self.tableWidgetTelechargement.creerItem( "Date" ) )
self.tableWidgetTelechargement.setHorizontalHeaderItem( 1,
self.tableWidgetTelechargement.creerItem( "Emission" ) )
self.tableWidgetTelechargement.setHorizontalHeaderItem( 2,
self.tableWidgetTelechargement.creerItem( "Etat" ) )
# On l'ajoute au layout
self.gridLayoutTelechargement.addWidget( self.tableWidgetTelechargement, 0, 1, 4, 1 )
# Bouton pour monter l'element selectionne tout en haut de la liste
self.pushButtonExtremiteMonter = MyQPushButton( self.tabTelechargements )
self.pushButtonExtremiteMonter.setIcon( QtGui.QIcon( "ico/gtk-jump-to-rtl.svg" ) )
self.pushButtonExtremiteMonter.setToolTip( u"Placer l'élément sélectionné tout en haut" )
self.gridLayoutTelechargement.addWidget( self.pushButtonExtremiteMonter, 0, 0, 1, 1 )
# Bouton pour monter l'element selectionne d'un cran dans la liste
self.pushButtonMonter = MyQPushButton( self.tabTelechargements )
self.pushButtonMonter.setIcon( QtGui.QIcon( "ico/gtk-go-up.svg" ) )
self.pushButtonMonter.setToolTip( u"Monter l'élément sélectionné" )
self.gridLayoutTelechargement.addWidget( self.pushButtonMonter, 1, 0, 1, 1 )
# Bouton pour descendre l'element selectionne d'un cran dans la liste
self.pushButtonDescendre = MyQPushButton( self.tabTelechargements )
self.pushButtonDescendre.setIcon( QtGui.QIcon( "ico/gtk-go-down.svg" ) )
self.pushButtonDescendre.setToolTip( u"Descendre l'élément selectionné" )
self.gridLayoutTelechargement.addWidget( self.pushButtonDescendre, 2, 0, 1, 1 )
# Bouton pour descendre l'element selectionne tout en bas de la liste
self.pushButtonExtremiteDescendre = MyQPushButton( self.tabTelechargements )
self.pushButtonExtremiteDescendre.setIcon( QtGui.QIcon( "ico/gtk-jump-to-ltr.svg" ) )
self.pushButtonExtremiteDescendre.setToolTip( u"Placer l'élément sélectionné tout en bas" )
self.gridLayoutTelechargement.addWidget( self.pushButtonExtremiteDescendre, 3, 0, 1, 1 )
# Bouton pour supprimer tous les elements de la liste
self.pushButtonToutSupprimer = MyQPushButton( self.tabTelechargements )
self.pushButtonToutSupprimer.setIcon( QtGui.QIcon( "ico/gtk-cancel.svg" ) )
self.pushButtonToutSupprimer.setToolTip( u"Supprimer tous les téléchargements" )
self.gridLayoutTelechargement.addWidget( self.pushButtonToutSupprimer, 0, 2, 1, 1 )
# Bouton pour supprimer de la liste les telechargements termines
self.pushButtonNettoyer = MyQPushButton( self.tabTelechargements )
self.pushButtonNettoyer.setIcon( QtGui.QIcon( "ico/gtk-delete-full.svg" ) )
self.pushButtonNettoyer.setToolTip( u"Supprimer les téléchargement terminés" )
self.gridLayoutTelechargement.addWidget( self.pushButtonNettoyer, 1, 2, 1, 1 )
# Bouton pour ouvrir le dossier des telechargements
self.pushButtonOuvrirDossierTelechargement = MyQPushButton( self.tabTelechargements )
self.pushButtonOuvrirDossierTelechargement.setIcon( QtGui.QIcon( "ico/gtk-folder.svg" ) )
self.pushButtonOuvrirDossierTelechargement.setToolTip( u"Ouvrir le dossier des téléchargements" )
self.gridLayoutTelechargement.addWidget( self.pushButtonOuvrirDossierTelechargement, 2, 2, 1, 1 )
#
# Barre progression de telechargement d'un fichier
#
self.progressBarTelechargementFichier = QtGui.QProgressBar( self.centralWidget )
self.progressBarTelechargementFichier.setProperty( "value", 0 )
#
# Barre de progression de telechargement des fichiers
#
self.progressBarTelechargement = QtGui.QProgressBar( self.centralWidget )
self.progressBarTelechargement.setProperty( "value", 0 )
#
# Boutons du bas pour gerer ajouter/supprimer/lancer telechargements
#
# Layout horizontal qui contiendra les boutons
self.horizontalLayoutBarreBas = QtGui.QHBoxLayout()
# Bouton pour lancer les telechargements
self.pushButtonLancer = QtGui.QPushButton( QtGui.QIcon( "ico/gtk-media-play-ltr.svg" ), u"Lancer téléchargement", self.centralWidget )
self.horizontalLayoutBarreBas.addWidget( self.pushButtonLancer )
# Bouton pour stopper les telechargements
self.pushButtonStop = QtGui.QPushButton( QtGui.QIcon( "ico/gtk-media-stop.svg" ), u"Stopper le téléchargement", self.centralWidget )
self.pushButtonStop.setEnabled( False )
self.horizontalLayoutBarreBas.addWidget( self.pushButtonStop )
###
# Positionnement des differents widgets/layouts sur le layout de grille
###
# Layout de grille dans lequel on va placer nos widgets/layouts
self.gridLayout = QtGui.QGridLayout( self.centralWidget )
# On ajoute la barre du haut
self.gridLayout.addLayout( self.horizontalLayoutBarreHaut, 0, 0, 1, 3 )
# On ajoute le gestionnaire d'onglets
self.gridLayout.addWidget( self.tabWidget, 1, 0, 1, 3 )
# On ajoute la barre de progression de telechargement d'un fichier
self.gridLayout.addWidget( self.progressBarTelechargementFichier, 2, 0, 1, 3 )
# On ajoute la barre de progression de telechargement des fichiers
self.gridLayout.addWidget( self.progressBarTelechargement, 3, 0, 1, 3 )
# On ajoute les boutons ajouter/supprimer/lancer
self.gridLayout.addLayout( self.horizontalLayoutBarreBas, 4, 0, 1, 3 )
###
# Mise en place le central widget dans la fenetre
###
self.setCentralWidget( self.centralWidget )
###
# Mise en place du menu
###
# Menu barre
self.menubar = QtGui.QMenuBar( self )
self.menubar.setGeometry( QtCore.QRect( 0, 0, 480, 25 ) )
# Menu Fichier
self.menuFichier = QtGui.QMenu( "&Fichier", self.menubar )
self.menubar.addAction( self.menuFichier.menuAction() )
# Action Fichier -> Quitter
self.actionQuitter = QtGui.QAction( QtGui.QIcon( "ico/gtk-quit.svg" ), "&Quitter", self )
self.actionQuitter.setIconVisibleInMenu( True )
self.menuFichier.addAction( self.actionQuitter )
# Menu Edition
self.menuEdition = QtGui.QMenu( "&Edition", self.menubar )
self.menubar.addAction( self.menuEdition.menuAction() )
# Action Edition -> Mise a jour
self.actionMAJ = QtGui.QAction( QtGui.QIcon( "ico/gtk-refresh.svg" ), u"&Mise à jour des plugins", self )
self.actionMAJ.setIconVisibleInMenu( True )
self.menuEdition.addAction( self.actionMAJ )
# Action Edition -> Preferences
self.actionPreferences = QtGui.QAction( QtGui.QIcon( "ico/gtk-preferences.svg" ), u"&Préférences", self )
self.actionPreferences.setIconVisibleInMenu( True )
self.menuEdition.addAction( self.actionPreferences )
# Menu Aide
self.menuAide = QtGui.QMenu( "&Aide", self.menubar )
self.menubar.addAction( self.menuAide.menuAction() )
# Action Aide -> A propos
self.actionAPropos = QtGui.QAction( QtGui.QIcon( "ico/gtk-about.svg" ), u"À p&ropos", self )
self.actionAPropos.setIconVisibleInMenu( True )
self.menuAide.addAction( self.actionAPropos )
# Ajout du menu a l'interface
self.setMenuBar( self.menubar )
###
# Signaux provenants de l'interface
###
QtCore.QObject.connect( self.tableWidgetFichier,
QtCore.SIGNAL( "cellClicked(int,int)" ),
self.afficherInformationsFichier )
QtCore.QObject.connect( self.tableWidgetFichier,
QtCore.SIGNAL( "cellDoubleClicked(int,int)" ),
self.gererTelechargement )
QtCore.QObject.connect( self.pushButtonToutAjouter,
QtCore.SIGNAL( "clicked()" ),
self.ajouterTousLesFichiers )
QtCore.QObject.connect( self.pushButtonRafraichirPlugin,
QtCore.SIGNAL( "clicked()" ),
self.rafraichirPlugin )
QtCore.QObject.connect( self.tableWidgetTelechargement,
QtCore.SIGNAL( "cellDoubleClicked(int,int)" ),
self.supprimerTelechargement )
QtCore.QObject.connect( self.pushButtonExtremiteMonter,
QtCore.SIGNAL( "clicked()" ),
lambda versLeHaut = True, extremite = True : self.tableWidgetTelechargement.deplacerLigne( versLeHaut, extremite ) )
QtCore.QObject.connect( self.pushButtonMonter,
QtCore.SIGNAL( "clicked()" ),
lambda versLeHaut = True, extremite = False : self.tableWidgetTelechargement.deplacerLigne( versLeHaut, extremite ) )
QtCore.QObject.connect( self.pushButtonDescendre,
QtCore.SIGNAL( "clicked()" ),
lambda versLeHaut = False, extremite = False : self.tableWidgetTelechargement.deplacerLigne( versLeHaut, extremite ) )
QtCore.QObject.connect( self.pushButtonExtremiteDescendre,
QtCore.SIGNAL( "clicked()" ),
lambda versLeHaut = False, extremite = True : self.tableWidgetTelechargement.deplacerLigne( versLeHaut, extremite ) )
QtCore.QObject.connect( self.pushButtonToutSupprimer,
QtCore.SIGNAL( "clicked()" ),
self.supprimerTousLesTelechargements )
QtCore.QObject.connect( self.pushButtonNettoyer,
QtCore.SIGNAL( "clicked()" ),
self.nettoyer )
QtCore.QObject.connect( self.pushButtonLancer,
QtCore.SIGNAL( "clicked()" ),
self.lancerTelechargement )
QtCore.QObject.connect( self.pushButtonStop,
QtCore.SIGNAL( "clicked()" ),
self.stopperTelechargement )
QtCore.QObject.connect( self.actionQuitter,
QtCore.SIGNAL( "triggered()" ),
self.close )
################################################
# Instanciations + initialisation de variables #
################################################
# Fenetre About
self.aProposDialog = None
# Fenetre des preferences du logiciel
self.preferencesDialog = None
# Fenetre de mise a jour des plugins
self.updateManagerDialog = None
# Nom plugin courant
self.nomPluginCourant = ""
# Liste des fichiers
self.listeFichiers = []
# Liste des fichiers a telecharger
self.listeFichiersATelecharger = []
# Cache des images descriptive
# Clef : urlImage Valeur : image (binaire)
self.cacheImage = {}
# On intancie le lanceur de signaux
self.signaux = Signaux()
# On instancie le gestionnaire de preferences
self.preferences = Preferences()
# On instancie le gestionnaire de preferences des plugins
self.preferencesPluginDialog = PreferencePluginDialog( self )
# On instancie le gestionnaire de download
self.downloader = Downloader( self.signaux )
# On recupere l'instance de API
self.api = API.getInstance()
# On instancie le gestionnaire d'historique
self.historique = Historique()
# On instancie la fenetre d'attente
self.fenetreAttenteProgressDialog = FenetreAttenteProgressDialog( self )
# On instancie le gest
#
# Fenetre de confirmation pour quitter le logiciel
#
self.quitterMessageBox = QtGui.QMessageBox( self )
self.quitterMessageBox.setWindowTitle( "Fermeture de TVDownloader" )
self.quitterMessageBox.setText( u"Voulez-vous réellement quitter TVDownloader ?" )
self.quitterMessageBox.setInformativeText( u"Votre liste de téléchargement sera perdue" )
self.quitterMessageBox.addButton( "Oui", QtGui.QMessageBox.AcceptRole )
self.quitterMessageBox.addButton( "Non", QtGui.QMessageBox.RejectRole )
############################################################
# On connecte les signaux des instances precedements crees #
############################################################
QtCore.QObject.connect( self.pushButtonOuvrirDossierTelechargement,
QtCore.SIGNAL( "clicked()" ),
self.ouvrirRepertoireTelechargement )
QtCore.QObject.connect( self.comboBoxSite,
QtCore.SIGNAL( "activated(QString)" ),
self.listerChaines )
QtCore.QObject.connect( self.comboBoxChaine,
QtCore.SIGNAL( "activated(QString)" ),
self.listerEmissions )
QtCore.QObject.connect( self.comboBoxEmission,
QtCore.SIGNAL( "activated(QString)" ),
self.listerFichiers )
QtCore.QObject.connect( self.pushButtonPreferencesPlugin,
QtCore.SIGNAL( "clicked()" ),
self.ouvrirPreferencesPlugin )
QtCore.QObject.connect( self.actionPreferences,
QtCore.SIGNAL( "triggered()" ),
self.ouvrirPreferencesLogiciel )
QtCore.QObject.connect( self.actionMAJ,
QtCore.SIGNAL( "triggered()" ),
self.ouvrirFenetreMiseAJour )
QtCore.QObject.connect( self.actionAPropos,
QtCore.SIGNAL( "triggered()" ),
self.ouvrirFenetreAPropos )
QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "debutActualisation(PyQt_PyObject)" ) , self.fenetreAttenteProgressDialog.ouvrirFenetreAttente )
QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "finActualisation()" ) , self.fenetreAttenteProgressDialog.fermerFenetreAttente )
QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "actualiserListesDeroulantes()" ) , self.actualiserListesDeroulantes )
QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "listeChaines(PyQt_PyObject)" ) , self.ajouterChaines )
QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "listeEmissions(PyQt_PyObject)" ) , self.ajouterEmissions )
QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "listeFichiers(PyQt_PyObject)" ) , self.ajouterFichiers )
QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "nouvelleImage(PyQt_PyObject)" ) , self.mettreEnPlaceImage )
QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "debutTelechargement(int)" ) , self.debutTelechargement )
QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "finTelechargement(int)" ) , self.finTelechargement )
QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "finDesTelechargements()" ) , self.activerDesactiverInterface )
QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "pourcentageFichier(int)" ) , self.progressBarTelechargementFichier.setValue )
#########
# Début #
#########
# La fenetre prend la dimension qu'elle avait a sa fermeture
taille = self.preferences.getPreference( "tailleFenetre" )
self.resize( taille[ 0 ], taille[ 1 ] )
# Si aucun plugin n'est active, on ouvre la fenetre des preferences
if( len( self.preferences.getPreference( "pluginsActifs" ) ) == 0 ):
self.ouvrirPreferencesLogiciel()
# On actualise tous les plugins
self.rafraichirTousLesPlugins()
## Methode qui execute les actions necessaires avant de quitter le programme
def actionsAvantQuitter( self ):
# On sauvegarde les options des plugins
self.api.fermeture()
# On sauvegarde la taille de la fenetre
taille = self.size()
self.preferences.setPreference( "tailleFenetre", [ taille.width(), taille.height() ] )
# On sauvegarde les options du logiciel
self.preferences.sauvegarderConfiguration()
# On sauvegarde l'historique
self.historique.sauverHistorique()
# On stoppe les telechargements
self.stopperTelechargement()
#########################################
# Surcharge des methodes de QMainWindow #
#########################################
## Surcharge de la methode appelee lors de la fermeture de la fenetre
# Ne doit pas etre appele explicitement
# @param evenement Evenement qui a provoque la fermeture
def closeEvent( self, evenement ):
# On affiche une fenetre pour demander la fermeture si des fichiers sont dans la liste de telechargement
if( self.tableWidgetTelechargement.rowCount() > 0 ):
# On affiche une fenetre qui demande si on veut quitter
retour = self.quitterMessageBox.exec_()
# Si on veut quitter
if( retour == 0 ):
# On execute les actions necessaires
self.actionsAvantQuitter()
# On accept la fermeture
evenement.accept()
else:
# On refuse la fermeture
evenement.ignore()
else: # S'il n'y a pas de fichier
# On execute les actions necessaires
self.actionsAvantQuitter()
# On accept la fermeture
evenement.accept()
##############################################
# Methodes pour remplir les menus deroulants #
##############################################
## Methode qui actualise les listes deroulantes
def actualiserListesDeroulantes( self ):
# On lance juste l'ajout des sites en se basant sur les plugins actifs
self.ajouterSites( self.preferences.getPreference( "pluginsActifs" ) )
## Methode qui lance le listage des chaines
# @param site Nom du plugin/site pour lequel on va lister les chaines
def listerChaines( self, site ):
def threadListerChaines( self, nomPlugin ):
self.signaux.signal( "debutActualisation", nomPlugin )
listeChaines = self.api.getPluginListeChaines( nomPlugin )
self.signaux.signal( "listeChaines", listeChaines )
self.signaux.signal( "finActualisation" )
if( site != "" ):
self.nomPluginCourant = qstringToString( site )
threading.Thread( target = threadListerChaines, args = ( self, self.nomPluginCourant ) ).start()
# On active (ou pas) le bouton de preference du plugin
self.pushButtonPreferencesPlugin.setEnabled( self.api.getPluginListeOptions( self.nomPluginCourant ) != [] )
## Methode qui lance le listage des emissions
# @param chaine Nom de la chaine pour laquelle on va lister les emissions
def listerEmissions( self, chaine ):
def threadListerEmissions( self, nomPlugin, chaine ):
self.signaux.signal( "debutActualisation", nomPlugin )
listeEmissions = self.api.getPluginListeEmissions( nomPlugin, chaine )
self.signaux.signal( "listeEmissions", listeEmissions )
self.signaux.signal( "finActualisation" )
if( chaine != "" ):
threading.Thread( target = threadListerEmissions, args = ( self, self.nomPluginCourant, qstringToString( chaine ) ) ).start()
## Methode qui lance le listage des fichiers
# @param emission Nom de l'emission pour laquelle on va lister les fichiers
def listerFichiers( self, emission ):
def threadListerFichiers( self, nomPlugin, emission ):
self.signaux.signal( "debutActualisation", nomPlugin )
listeFichiers = self.api.getPluginListeFichiers( nomPlugin, emission )
self.signaux.signal( "listeFichiers", listeFichiers )
self.signaux.signal( "finActualisation" )
if( emission != "" ):
threading.Thread( target = threadListerFichiers, args = ( self, self.nomPluginCourant, qstringToString( emission ) ) ).start()
## Methode qui met en place une liste de sites sur l'interface
# @param listeSites Liste des sites a mettre en place
def ajouterSites( self, listeSites ):
# On efface la liste des sites
self.comboBoxSite.clear()
# On met en place les sites
for site in listeSites:
self.comboBoxSite.addItem( stringToQstring( site ) )
# On selectionne par defaut celui choisis dans les preference
index = self.comboBoxSite.findText( stringToQstring( self.preferences.getPreference( "pluginParDefaut" ) ) )
if( index != -1 ):
self.comboBoxSite.setCurrentIndex( index )
# On lance l'ajout des chaines
self.listerChaines( self.comboBoxSite.currentText() )
## Methode qui met en place une liste de chaines sur l'interface
# @param listeChaines Liste des chaines a mettre en place
def ajouterChaines( self, listeChaines ):
# On trie la liste des chaines
listeChaines.sort()
# On efface la liste des chaines
self.comboBoxChaine.clear()
# On efface la liste des emissions
self.comboBoxEmission.clear()
# On efface la liste des fichiers
self.tableWidgetFichier.toutSupprimer()
# On met en place les chaines
for chaine in listeChaines:
self.comboBoxChaine.addItem( stringToQstring( chaine ) )
# Si on a juste une seule chaine
if( self.comboBoxChaine.count() == 1 ):
# On lance l'ajout des emissions
self.listerEmissions( self.comboBoxChaine.currentText() )
else:
# On ne selectionne pas de chaine
self.comboBoxChaine.setCurrentIndex( -1 )
## Methode qui met en place une liste d'emissions sur l'interface
# @param listeEmissions Liste des emissions a mettre en place
def ajouterEmissions( self, listeEmissions ):
# On trie la liste des emissions
listeEmissions.sort()
# On efface la liste des emissions
self.comboBoxEmission.clear()
# On efface la liste des fichiers
self.tableWidgetFichier.toutSupprimer()
# On met en place la liste des emissions
for emission in listeEmissions:
self.comboBoxEmission.addItem( stringToQstring( emission ) )
# Si on a juste une seule emission
if( self.comboBoxEmission.count() == 1 ):
# On lance l'ajout des fichiers
self.listerFichiers( self.comboBoxEmission.currentText() )
else:
# On ne selectionne pas d'emission
self.comboBoxEmission.setCurrentIndex( -1 )
###############################################
# Methodes pour remplir la liste des fichiers #
###############################################
## Methode pour ajouter des fichiers a l'interface
# @param listeFichiers Liste des fichiers a ajouter
def ajouterFichiers( self, listeFichiers ):
self.listeFichiers = listeFichiers
# On efface la liste des fichiers
self.tableWidgetFichier.toutSupprimer()
# On commence au depart
ligneCourante = 0
# On met en place chacun des fichiers
for fichier in listeFichiers:
# On ajoute une ligne
self.tableWidgetFichier.insertRow( ligneCourante )
# On ajoute les informations au tableWidgetFichier
liste = []
liste.append( self.tableWidgetFichier.creerItem( "" ) )
liste.append( self.tableWidgetFichier.creerItem( getattr( fichier, "date" ) ) )
liste.append( self.tableWidgetFichier.creerItem( getattr( fichier, "nom" ) ) )
self.tableWidgetFichier.setLigne( ligneCourante, liste )
# On met en place l'icone qui va bien
self.gererIconeListeFichier( fichier )
ligneCourante += 1
# On adapte la taille des colonnes
self.tableWidgetFichier.adapterColonnes()
## Methode qui rafraichit le plugin courant
def rafraichirPlugin( self ):
def threadRafraichirPlugin( self, nomPlugin ):
self.signaux.signal( "debutActualisation", nomPlugin )
self.api.pluginRafraichir( nomPlugin )
self.signaux.signal( "finActualisation" )
threading.Thread( target = threadRafraichirPlugin, args = ( self, self.nomPluginCourant ) ).start()
## Methode qui met en place l'image de la description d'un fichier
# @param image Image a mettre en place (binaire)
def mettreEnPlaceImage( self, image ):
logoFichier = QtGui.QPixmap()
logoFichier.loadFromData( image )
self.labelLogo.setPixmap( logoFichier.scaled( QtCore.QSize( 150, 150 ), QtCore.Qt.KeepAspectRatio ) )
## Methode qui affiche des informations sur le fichier selectionne
def afficherInformationsFichier( self, ligne, colonne ):
def threadRecupererImage( self, urlImage ):
image = self.api.getPage( urlImage )
self.cacheImage[ urlImage ] = image
self.signaux.signal( "nouvelleImage", image )
fichier = self.listeFichiers[ ligne ]
# On recupere le lien de l'image et le texte descriptif
urlImage = getattr( fichier, "urlImage" )
texteDescriptif = getattr( fichier, "descriptif" )
self.plainTextEdit.clear()
# Si on a un texte descriptif, on l'affiche
if( texteDescriptif != "" ):
self.plainTextEdit.appendPlainText( stringToQstring( texteDescriptif ) )
else:
self.plainTextEdit.appendPlainText( u"Aucune information disponible" )
# Si on n'a pas d'image
if( urlImage == "" ):
# On met en place celle par defaut
self.logoFichier = self.logoFichierDefaut
self.labelLogo.setPixmap( self.logoFichier.scaled( QtCore.QSize( 150, 150 ), QtCore.Qt.KeepAspectRatio ) )
else: # Si on en a une
# Si elle est dans le cache des images
if( self.cacheImage.has_key( urlImage ) ):
self.mettreEnPlaceImage( self.cacheImage[ urlImage ] )
else: # Sinon
# On lance le thread pour la recuperer
threading.Thread( target = threadRecupererImage, args = ( self, urlImage ) ).start()
## Methode qui gere l'icone d'un fichier dans la liste des telechargements
# Il y a 3 icones possible :
# - C'est un fichier
# - C'est un fichier present dans l'historique (donc deja telecharge)
# - C'est un fichier present dans la liste des telechargements
# @param fichier Fichier a gerer
def gererIconeListeFichier( self, fichier ):
if( fichier in self.listeFichiers ):
ligneFichier = self.listeFichiers.index( fichier )
# On cherche quel icone mettre en place
if( fichier in self.listeFichiersATelecharger ):
icone = self.iconeAjoute
elif( self.historique.comparerHistorique( fichier ) ):
icone = self.iconeTelecharge
else:
icone = self.iconeFichier
# On met en place l'icone
self.tableWidgetFichier.item( ligneFichier, 0 ).setIcon( icone )
######################################################
# Methodes pour remplir la liste des telechargements #
######################################################
## Methode qui gere la liste des telechargements
# @param ligne Numero de la ligne (dans la liste des fichiers) de l'element a ajouter
# @param colonne Numero de colonne (inutile, juste pour le slot)
def gererTelechargement( self, ligne, colonne = 0 ):
fichier = self.listeFichiers[ ligne ]
# Si le fichier est deja dans la liste des telechargements
if( fichier in self.listeFichiersATelecharger ):
ligneTelechargement = self.listeFichiersATelecharger.index( fichier )
self.supprimerTelechargement( ligneTelechargement )
else: # S'il n'y est pas, on l'ajoute
self.listeFichiersATelecharger.append( fichier )
numLigne = self.tableWidgetTelechargement.rowCount()
# On insere une nouvelle ligne dans la liste des telechargements
self.tableWidgetTelechargement.insertRow( numLigne )
# On y insere les elements qui vont biens
self.tableWidgetTelechargement.setLigne( numLigne,
[ self.tableWidgetTelechargement.creerItem( getattr( fichier, "date" ) ),
self.tableWidgetTelechargement.creerItem( getattr( fichier, "nom" ) ),
self.tableWidgetTelechargement.creerItem( u"En attente de téléchargement" )
]
)
# On adapte la taille des colonnes
self.tableWidgetTelechargement.adapterColonnes()
# On modifie l'icone dans la liste des fichiers
self.gererIconeListeFichier( fichier )
## Methode qui ajoute tous les fichiers a la liste des telechargements
def ajouterTousLesFichiers( self ):
for i in range( self.tableWidgetFichier.rowCount() ):
self.gererTelechargement( i )
## Methode qui supprime un fichier de la liste des telechargements
# @param ligne Numero de la ligne a supprimer
# @param colonne Numero de colonne (inutile, juste pour le slot)
def supprimerTelechargement( self, ligne, colonne = 0 ):
fichier = self.listeFichiersATelecharger[ ligne ]
# On supprime l'element du tableWidgetTelechargement
self.tableWidgetTelechargement.removeRow( ligne )
# On supprime l'element de la liste des fichiers a telecharger
self.listeFichiersATelecharger.remove( fichier )
# On modifie l'icone dans la liste des fichiers
self.gererIconeListeFichier( fichier )
## Methode qui supprime tous les telechargement de la liste des telechargements
def supprimerTousLesTelechargements( self ):
for i in range( self.tableWidgetTelechargement.rowCount() -1, -1, -1 ):
self.supprimerTelechargement( i )
## Methode qui lance le telechargement des fichiers
def lancerTelechargement( self ):
# On liste les emissions a telecharger avec leurs numeros de ligne
listeFichiers = []
for i in range( self.tableWidgetTelechargement.rowCount() ): # Pour chaque ligne
fichier = self.listeFichiersATelecharger[ i ]
listeFichiers.append( [ i,
getattr( fichier, "lien" ),
getattr( fichier, "nomFichierSortie" ) ] )
nbATelecharger = len( listeFichiers )
# Si on a des elements a charger
if( nbATelecharger > 0 ):
# On met en place la valeur du progressBar
self.progressBarTelechargement.setMaximum( nbATelecharger )
self.progressBarTelechargement.setValue( 0 )
# On lance le telechargement
threading.Thread( target = self.downloader.lancerTelechargement, args = ( listeFichiers, ) ).start()
# On active/desactive ce qui va bien sur l'interface
self.activerDesactiverInterface( True )
## Methode qui stoppe le telechargement
def stopperTelechargement( self ):
# On stoppe le telechargement
self.downloader.stopperTelechargement()
############################################
# Methodes pour ouvrir les autres fenetres #
############################################
#
# Fenetre About
#
## Methode pour afficher la fenetre About
def ouvrirFenetreAPropos( self ):
if( self.aProposDialog == None ):
self.aProposDialog = AProposDialog()
self.aProposDialog.show()
#
# Fenetre de preference du logiciel
#
## Methode pour ouvrir les preferences du logiciel
def ouvrirPreferencesLogiciel( self ):
if( self.preferencesDialog == None ):
self.preferencesDialog = PreferencesDialog( self, self.signaux )
self.preferencesDialog.afficher()
#
# Fenetre de mise a jour des plugins
#
## Methode pour ouvrir la fenetre de mise a jour des plugins
def ouvrirFenetreMiseAJour( self ):
if( self.updateManagerDialog == None ):
self.updateManagerDialog = UpdateManagerDialog( self )
self.updateManagerDialog.afficher()
#
# Fenetre de preference des plugins
#
## Methode pour ouvrir les preferences du plugin courant
def ouvrirPreferencesPlugin( self ):
listeOptions = self.api.getPluginListeOptions( self.nomPluginCourant )
self.preferencesPluginDialog.ouvrirDialogPreferences( self.nomPluginCourant, listeOptions )
#########
# Slots #
#########
## Methode qui ouvre le repertoire de telechargement
def ouvrirRepertoireTelechargement( self ):
QtGui.QDesktopServices.openUrl( QtCore.QUrl.fromLocalFile( self.preferences.getPreference( "repertoireTelechargement" ) ) )
## Methode qui rafraichit le plugin courant
def rafraichirPlugin( self ):
def threadRafraichirPlugin( self, nomPlugin ):
self.signaux.signal( "debutActualisation", nomPlugin )
self.api.pluginRafraichir( nomPlugin )
self.signaux.signal( "finActualisation" )
threading.Thread( target = threadRafraichirPlugin, args = ( self, self.nomPluginCourant ) ).start()
## Methode qui rafraichit tous les plugins
# A utiliser au lancement du programme
def rafraichirTousLesPlugins( self ):
def threadRafraichirTousLesPlugins( self ):
self.signaux.signal( "debutActualisation", "TVDownloader" )
self.api.pluginRafraichirAuto()
self.signaux.signal( "finActualisation" )
self.signaux.signal( "actualiserListesDeroulantes" )
threading.Thread( target = threadRafraichirTousLesPlugins, args = ( self, ) ).start()
## Slot qui active/desactive des elements de l'interface pendant un telechargement
# @param telechargementEnCours Indique si on telecharge ou pas
def activerDesactiverInterface( self, telechargementEnCours = False ):
# Les boutons
self.pushButtonLancer.setEnabled( not telechargementEnCours )
self.pushButtonStop.setEnabled( telechargementEnCours )
self.pushButtonExtremiteMonter.setEnabled( not telechargementEnCours )
self.pushButtonMonter.setEnabled( not telechargementEnCours )
self.pushButtonDescendre.setEnabled( not telechargementEnCours )
self.pushButtonExtremiteDescendre.setEnabled( not telechargementEnCours )
self.pushButtonToutSupprimer.setEnabled( not telechargementEnCours )
self.pushButtonNettoyer.setEnabled( not telechargementEnCours )
# Le table widget
self.tableWidgetTelechargement.setEnabled( not telechargementEnCours )
## Slot appele lors ce qu'un le debut d'un telechargement commence
# @param numero Position dans la liste des telechargement du telechargement qui commence
def debutTelechargement( self, numero ):
self.tableWidgetTelechargement.item( numero, 2 ).setText( stringToQstring( u"Téléchargement en cours..." ) )
self.tableWidgetTelechargement.adapterColonnes()
self.progressBarTelechargementFichier.setValue( 0 )
## Slot appele lorsqu'un telechargement se finit
# @param numero Position dans la liste des telechargement du telechargement qui se finit
def finTelechargement( self, numero ):
fichier = self.listeFichiersATelecharger[ numero ]
# On ajoute le fichier a l'historique
self.historique.ajouterHistorique( fichier )
# On modifie l'icone dans la liste des fichiers
self.gererIconeListeFichier( fichier )
# On modifie l'interface
self.tableWidgetTelechargement.item( numero, 2 ).setText( stringToQstring( u"Fini !" ) )
self.progressBarTelechargement.setValue( self.progressBarTelechargement.value() + 1 )
self.tableWidgetTelechargement.adapterColonnes()
self.progressBarTelechargementFichier.setValue( 100 )
## Slot qui nettoie la liste des telechargements de tous les telechargements finis
def nettoyer( self ):
for i in range( self.tableWidgetTelechargement.rowCount() - 1, -1, -1 ): # [ nbLignes - 1, nbLignes - 2, ..., 1, 0 ]
if( self.tableWidgetTelechargement.item( i, 2 ).text() == u"Fini !" ): # Si c'est telecharge
self.supprimerTelechargement( i )
def download(cookies, url, folder):
downloader = Downloader(cookies, url, folder)
downloader.run()
class Watcher:
def __init__(self,
filename,
period='1yr',
stockPicks=10,
avoidDowntrends=True,
sellAllOnCrash=True,
hyperparameters={}):
self.filename = filename
self.period = period
self.stockPicks = stockPicks
self.hyperparameters = hyperparameters
self.sl = hyperparameters['sl']
self.tp = hyperparameters['tp']
self.ts = hyperparameters['ts']
self.avoidDowntrends = avoidDowntrends
self.sellAllOnCrash = sellAllOnCrash
def create(self, balance):
# Create the portolio
self.portfolio = Portfolio(cash=balance, sl=self.sl, tp=self.tp)
self.portfolio.save(self.filename)
def start(self):
# Open the portolio
self.portfolio = Portfolio(cash=0, sl=self.sl, tp=self.tp)
self.portfolio.load(self.filename)
# Download the stock data
self.downloader = Downloader(cache=False)
self.downloader.download(period=self.period)
date = self.downloader.prices.index[-1]
prices = self.downloader.prices
print("#", date)
# Refresh the portfolio with the latest prices
self.portfolio.refresh(prices=prices, date=date)
# Get the portfolio Summary
portfolio_summary = self.portfolio.summary()
# Get the opened positions
opened = self.portfolio.holdings[self.portfolio.holdings['status'] ==
'open']
# Get the current trend
zt = TradingZones(prices=prices)
zones = zt.get()
output = pd.DataFrame()
# Activate the trailing stops
targets = self.portfolio.getPositionsBelowTS(self.ts)
if len(targets) > 0:
sell_orders = self.portfolio.getSellOrdersFromSubset(
subset=targets)
print("\n----- TS -----\n", sell_orders)
#self.portfolio.sell(sell_orders, prices, date, label='TS')
# Activate the stop loss
if self.sl < 0:
poor_perfs = opened[opened['profits_pct'] <= self.sl]
if len(poor_perfs) > 0:
sell_list = poor_perfs['symbol'].unique()
sell_orders = self.portfolio.getSellAllOrders(
symbols=sell_list)
print("\n----- SL -----\n", sell_orders)
#self.portfolio.sell(sell_orders, prices, date, label='SL')
# Sell all on drop
if self.sellAllOnCrash == True and zones.iloc[-1][
'signal'] == 'DOWN' and len(opened) > 0:
print("Sell all")
print("\n----- Sell All -----\n")
#self.portfolio.sellAll(prices, date, label='Market Drop')
# Get the ones that reached take-profit
if self.tp > 0:
good_perfs = opened[opened['profits_pct'] >= self.tp]
if len(good_perfs) > 0:
sell_list = good_perfs['symbol'].unique()
sell_orders = self.portfolio.getSellAllOrders(
symbols=sell_list)
print("\n----- TP -----\n", sell_orders)
#self.portfolio.sell(sell_orders, prices, date, label='TP')
# Avoid trading in a general market downtrend
if self.avoidDowntrends == False or (self.avoidDowntrends == True and
zones.iloc[-1]['signal'] == 'UP'):
# Pick the stocks
picker = Picker(prices=prices)
picker.pick(count=self.stockPicks)
#print(">", picker.selected)
# Get the order size
portfolio_summary = self.portfolio.summary()
order = Order(prices=prices, picks=picker.selected)
order.get(budget=portfolio_summary['cash'])
#print("!", portfolio_summary['cash'], order.orders)
print("\n----- BUY -----\n", order.orders)
# Update our portfolio
#self.portfolio.update(orders=order.orders, prices=prices, date=date)
else:
# No new positions in a downtrend!
# Update our portfolio
self.portfolio.update(orders=pd.DataFrame(),
prices=prices,
date=date)
def stats(self):
# Open the portolio
self.portfolio = Portfolio(cash=0, sl=self.sl, tp=self.tp)
self.portfolio.load(self.filename)
symbols = list(self.portfolio.holdings['symbol'].unique())
if len(symbols) == 0:
print("Nothing to update")
return False
elif len(symbols) == 1:
symbols = symbols + [
'MSFT', 'TSLA'
] # otherwise single symbol = no multi-index = breaking
# Download the stock data
self.downloader = Downloader(cache=False)
self.downloader.download(period=self.period, symbols=symbols)
date = self.downloader.prices.index[-1]
prices = self.downloader.prices
print("#", date)
# Refresh the portfolio with the latest prices
self.portfolio.refresh(prices=prices, date=date)
# Save the portfolio with the latest prices
self.portfolio.save(self.filename)
print(self.portfolio.summary())
print('')
g = self.portfolio.holdings[self.portfolio.holdings['status'] ==
'open'].copy().groupby('symbol').sum()
g['profits_pct'] = (g['current_price'] -
g['purchase_price']) / g['purchase_price'] * 100
g['price'] = prices.loc[date]
print(g)
print(list(g.index))
print(prices.loc[date])
def check(self, symbol='TSLA', period='6mo'):
# Download the stock data
self.downloader = Downloader(cache=False)
if symbol not in self.downloader.symbols:
self.downloader.add(symbol)
print('Symbol added to the watch list')
self.downloader.download(period=period, symbols=[symbol])
print(self.downloader.prices[symbol].tail(50))
log.setLevel(level)
log2.setLevel(level)
fmt = logging.Formatter('%(levelname)s:%(name)s:%(message)s')
h = logging.StreamHandler()
h.setFormatter(fmt)
log.addHandler(h)
log2.addHandler(h)
#mongodb
app.config["MONGO_URI"] = "mongodb://"+Config.get('DB', 'mongoHost')+":"+Config.get('DB', 'mongoPort')+"/"+Config.get('DB', 'mongoDbName')
mongo = PyMongo(app)
# objects doing the work
downloader = Downloader(mongo, Config.get('PARSER', 'videoFolder'))
parser = Parser(mongo, Config.get('PARSER', 'videoFolder'), Config.get('PARSER', 'characterFolder'), Config.get('PARSER', 'templateFolder'))
parserTest = ParserTest(mongo, Config.get('PARSER', 'videoFolder'), Config.get('PARSER', 'characterFolder'), Config.get('PARSER', 'templateFolder'))
# fire the main job
@app.route('/firemainjob')
def index():
print("fire scheduler")
scheduler.get_job("mainJob").modify(next_run_time=datetime.datetime.now())
return jsonify({"fire": "ok"})
# download only
@app.route('/onlydl/<idvid>')
def onlydl(idvid):
print("download:"+idvid)
downloader.downloadVideo(idvid)
from MangaSiteFactory import get_manga_site, MangaSiteEnum
from MangaSite import MangaSite
from Manga import Manga, Chapter, MangaIndexTypeEnum
from Downloader import Downloader
from PySide2 import QtWidgets, QtCore
from typing import List, Dict
from functools import partial
def call_timeout(loop):
loop.quit()
raise TimeoutError
get_mhg = partial(get_manga_site, MangaSiteEnum.ManHuaGui,
Downloader(None, './downloads'))
class TestManHuaGui(unittest.TestCase):
def test_init(self):
mhg = get_mhg()
self.assertEqual(mhg.name, '漫畫鬼')
self.assertEqual(mhg.url, 'https://www.manhuagui.com/')
def test_seach_manga(self):
def search_callback(loop, result: List[Manga]):
self.assertIsInstance(result, list)
self.assertEqual(len(result), 2)
foundFlag = False
for manga in result:
if manga.name == '火影忍者':
def scrapeKupujemProdajem():
# UTF-8 support
reload(sys)
sys.setdefaultencoding('utf-8')
now = datetime.now()
ads = []
try:
down = Downloader(
'http://www.kupujemprodajem.com/search.php?action=list&data[category_id]=&data[group_id]=&data[location_id]=&data[keywords]=&submit[search]=Tra%C5%BEi'
)
content = down.get_content()
html = unicode(content)
link = ""
title = ""
imageUrl = ""
description = "/"
category = "/"
subcategory = "/"
value = "/"
currency = "/"
region = "/"
date = str(now.year) + "-" + str(now.month) + "-" + str(now.day)
linkovi = xpath.search(html, '//div[@class="item clearfix"]')
highlighted = xpath.search(
html, '//div[@class="item clearfix adHighlighted"]')
for h in highlighted:
linkovi.append(h)
for l in linkovi:
try:
link = "http://www.kupujemprodajem.com/" + xpath.get(
l, '//a[@class="adName"]/@href')
title = xpath.get(l, '//a[@class="adName"]')
region = xpath.get(l,
'//section[@class="locationSec"]').strip()
region = region.split(" | ")
region = region[0]
price = xpath.get(l, '//span[@class="adPrice"]')
price = price.split(" ")
if len(price) == 2:
value = price[0]
value = value.replace(".", "")
value = value.split(",")
value = value[0]
currency = price[1]
else:
value = "/"
currency = "/"
if currency == "€":
currency = "EUR"
elif currency == "din":
currency = "DIN"
down = Downloader(link)
content = down.get_content()
category = xpath.get(content, '//a[@class="crumbs"]')
description = xpath.get(
l, '//section[@class="nameSec"]/p[@class="adDescription"]')
category = category.split("|")
category = category[0]
category = category.strip()
imageUrl = xpath.get(
content, '//div[@class="adThumbnailHolder"]/a/img/@src')
imageUrl = imageUrl.replace("//", "/")
imageUrl = imageUrl[1::]
if imageUrl == "":
imageUrl = "/"
description = description.replace("...<p>", "")
description = description.strip()
country = u"Србија"
ad = Ad(link, title, imageUrl, description, category,
subcategory, value, currency, region, date, country)
ads.append(ad)
except:
pass
except:
pass
return adsToJson(ads)
#print scrapeKupujemProdajem()
def __init__(self, DLlist):
# On instancie le gestionnaire de preferences et sa fenetre
self.preferences = Preferences()
# On instancie le gestionnaire de download
self.downloader = Downloader()
# On instancie le gestionnaire d'historique et sa fenetre
self.historique = Historique()
current=0
global choice
choice=''
while choice!='r' and choice!='R':
if choice=='s' or choice=='S':
os.system(['clear','cls'][os.name == 'nt'])
header ('','','Liste de telechargement\t')
print "\n\n\n\tSupprimer tous les fichiers de la liste de telchargement"
#pour chaque fichier de la liste ***(len(DLlist))!=0
while len(DLlist)>0:
print "Supprimer le fichier :",DLlist[0].nom
#supprimer le fichier de la liste
DLlist.remove(DLlist[0])
#ajouter le fichier au log
time.sleep (1)
if choice=='t' or choice=='T':
os.system(['clear','cls'][os.name == 'nt'])
header ('','','Liste de telechargement\t')
print "\n\n\n\ttelecharger tous les fichiers"
#pour chaque fichier de la liste ***(len(DLlist))!=0
while len(DLlist)>0:
if not self.historique.comparerHistorique(DLlist[0]):
os.system(['clear','cls'][os.name == 'nt'])
header ('','','Liste de telechargement\t')
print "\n\n\n\ttelecharger le fichier :",DLlist[0].nom
if(DLlist[0].nomFichierSortie=="" ):
DLlist[0].nomFichierSortie=os.path.basename(getattr(DLlist[0],"lien" ))
#telecharger le fichier
self.downloader.lancerTelechargement([[0,DLlist[0].lien,DLlist[0].nomFichierSortie]])
#ajouter le fichier a l'historique de telechargement
self.historique.ajouterHistorique(DLlist[0])
else:
os.system(['clear','cls'][os.name == 'nt'])
header ('','','Liste de telechargement\t')
print "\n\n\n\tFichier deja telecharge"
#supprimer le fichier de la liste
DLlist.remove(DLlist[0])
time.sleep (1)
elif choice=='q' or choice=='Q':
quitter()
elif choice.isdigit() and len(DLlist)>int(choice)+10*current and int(choice)>=0:
value=''
while (value!='r' and value!='t' and value!='s'):
os.system(['clear','cls'][os.name == 'nt'])
header ('','','Liste de telechargement\t')
print "\n\n\n\tFichier :",DLlist[int(choice)+10*current].nom,"\n\n\tQue voulez vous faire?\n\n\t\tt:telecharger le fichier\n\t\ts:supprimer le fichier de la liste de telechargement\n\t\tr:retour a liste de telechargement\n\n\n\n\n"
value=getch()
if value=='t':
if not self.historique.comparerHistorique(DLlist[int(choice)+10*current]):
os.system(['clear','cls'][os.name == 'nt'])
header ('','','Liste de telechargement\t')
print "\n\n\n\ttelecharger le fichier :",DLlist[int(choice)].nom
# Si le nom du fichier de sortie n'existe pas, on l'extrait de l'URL
if(DLlist[int(choice)+10*current].nomFichierSortie=="" ):
DLlist[int(choice)+10*current].nomFichierSortie=os.path.basename(getattr(DLlist[int(choice)+10*current],"lien" ))
#telecharger le fichier
self.historique.ajouterHistorique(DLlist[int(choice)+10*current])
self.downloader.lancerTelechargement([[0,DLlist[int(choice)].lien,DLlist[int(choice)].nomFichierSortie]])
#ajouter le fichier a l'historique de telechargement
else:
os.system(['clear','cls'][os.name == 'nt'])
header ('','','Liste de telechargement\t')
print "\n\n\n\tFichier deja telecharge"
time.sleep(1)
#supprimer le fichier de la liste
DLlist.remove(DLlist[int(choice)])
elif value=='s':
os.system(['clear','cls'][os.name == 'nt'])
header ('','','Liste de telechargement\t')
print "\n\n\n\tSuppression de la liste de telechargement du fichier :\n\n\t\t",DLlist[int(choice)].nom
#supprimer le fichier de la liste
DLlist.remove(DLlist[int(choice)])
elif value=='r':
os.system(['clear','cls'][os.name == 'nt'])
header ('','','Liste de telechargement\t')
print "\n\n\n\tRetour a la liste de telechargement"
#ajouter le fichier au log
time.sleep(1)
elif choice=='*' :
value=''
os.system(['clear','cls'][os.name == 'nt'])
header ('','','Liste de telechargement\t')
print "\n\n\n\n\n\tQue voulez vous faire?\n\n\t\tt:telecharger les fichiers\n\t\ts:supprimer les fichiers de la liste de telechargement\n\n\n\n\n\n"
value=getch()
while len(DLlist)>0:
if value=='t':
if not self.historique.comparerHistorique(DLlist[0]):
os.system(['clear','cls'][os.name == 'nt'])
header ('','','Liste de telechargement\t')
print "\n\n\n\ttelecharger le fichier :",DLlist[0].nom
# Si le nom du fichier de sortie n'existe pas, on l'extrait de l'URL
if(DLlist[0].nomFichierSortie=="" ):
DLlist[0].nomFichierSortie=os.path.basename(getattr(DLlist[0],"lien" ))
#telecharger le fichier
self.downloader.lancerTelechargement([[0,DLlist[0].lien,DLlist[0].nomFichierSortie]])
#ajouter le fichier a l'historique de telechargement
self.historique.ajouterHistorique(DLlist[0])
else:
os.system(['clear','cls'][os.name == 'nt'])
header ('','','Liste de telechargement\t')
print "\n\n\n\tFichier deja telecharge"
time.sleep(0.5)
#supprimer le fichier de la liste
DLlist.remove(DLlist[0])
#ajouter le fichier au log
elif choice=='+':
if len(DLlist)>current*10+10: current+=1
elif choice=='-':
if current!=0: current-=1
choice=''
#sauvegarder l'historique de telechargement
self.historique.sauverHistorique()
#affichage a l'ecran de la liste
header ('','','Liste de telechargement\t')
for i in range(10):
if len(DLlist)>i+10*current:
if len(DLlist[int(i+10*current)].nom)>74: print " ",i,":",DLlist[int(i+10*current)].nom[:71]+"..."
elif len(DLlist[int(i+10*current)].nom)<=74: print " ",i,":",DLlist[int(i+10*current)].nom
else: print ""
if len(DLlist)>10:print "\n\t+:fichiers suivants\t-:fichiers precedents (page",current+1,"/",len(DLlist)/10+1,",",len(DLlist),"chaines)"
else:print"\n"
print "\n\tt:telecharger tous les fichiers s:supprimer tous les fichiers"
footer()
if len(DLlist)==0:
os.system(['clear','cls'][os.name == 'nt'])
header ('','','Liste de telechargement\t')
print "\n\n\n\n\n\t\tAucun fichier dans la liste"
footer()
choice='r'
time.sleep(1)
if not choice:choice=getch()
def download(self, collects, keyword):
self.print_with_color("Download Images...", "b")
image_count = 0
keyword = 'images/' + keyword
# Make directory and path
if not os.path.isdir(keyword):
print("Make a directory.")
os.makedirs(keyword)
dir = os.getcwd() + "/" + str(keyword)
# os.chdir(dir)
self.print_with_color("Download path is %s" % dir, "g")
if not dir.endswith("/"):
dir += '/'
thread_count = self.num_of_thread
queue = Queue()
self.printProgressBar(0,
len(collects),
prefix='Progress:',
suffix='Complete')
for i in range(thread_count):
downloader = Downloader(queue, dir, self.collectorName,
self.error_list, len(collects))
downloader.daemon = True
downloader.start()
for i in range(len(collects)):
url = collects[i]
queue.put(url)
queue.join()
self.printProgressBar(len(collects),
len(collects),
prefix='Progress:',
suffix='Complete')
# Start download
# self.printProgressBar(0, len(collects), prefix='Progress:', suffix='Complete')
# for i in range(len(collects)):
# self.printProgressBar(i+1, len(collects), prefix='Progress:', suffix='Complete')
# # print("Download Image (%d/%d)" % (i+1, len(collects)))
# col = collects[i]
# if (col == '알림'):
# pass
# else:
# full_name = self.collectorName + str(i+1) + ".jpg"
# save_path = os.path.join(dir, full_name) # 저장폴더
#
# try:
# with eventlet.Timeout(self.DOWNLOAD_TIMEOUT):
# urllib2.urlretrieve(col, save_path)
# except:
# self.error_list.append(col)
self.print_with_color(
"Save %d images" % (len(collects) - len(self.error_list)), "g")
gui = GUI(u'script-%s-main.xml' % addon_name, addon_path).doModal()
homewindow.clearProperty("homewindowactive")
del gui
elif window == "colorconfig":
from ColorConfigDialog import ColorConfigDialog
gui = ColorConfigDialog(u'script-%s-colorconfig.xml' % addon_name, addon_path).doModal()
del gui
elif window == "trailers":
from TrailerWindow import TrailerWindow
gui = TrailerWindow(u'script-%s-trailers.xml' % addon_name, addon_path).doModal()
del gui
elif window == "favourites":
from FavWindow import FavWindow
gui = FavWindow(u'script-globalsearch-main.xml', addon_path).doModal()
del gui
elif window == "dialogalbuminfo":
from DialogAlbumInfo import DialogAlbumInfo
gui = DialogAlbumInfo(u'script-%s-dialogalbuminfo.xml' % addon_name, addon_path).doModal()
del gui
elif window == "fullscreeninfo":
from FullscreenInfo import FullscreenInfo
gui = FullscreenInfo(u'script-%s-fullscreeninfo.xml' % addon_name, addon_path).doModal()
del gui
elif window == "downloader":
from Downloader import Downloader
dialog = Downloader()
dialog.show_download_dialog()
del dialog
else:
MoveProperties(container, focuscontrol)
def download_from_urls(url_file, out_dir):
print("Downloading data from URLS in %s to %s" % (url_file, out_dir))
dl = Downloader(json.loads(open(url_file).read()))
dl.save_files(out_dir)
class SchedulerGrid(Scheduler):
def __init__(self, name):
Scheduler.__init__(self, name)
self.states = [ "Acl", "cancelReason", "cancelling","ce_node","children", \
"children_hist","children_num","children_states","condorId","condor_jdl", \
"cpuTime","destination", "done_code","exit_code","expectFrom", \
"expectUpdate","globusId","jdl","jobId","jobtype", \
"lastUpdateTime","localId","location", "matched_jdl","network_server", \
"owner","parent_job", "reason","resubmitted","rsl","seed",\
"stateEnterTime","stateEnterTimes","subjob_failed", \
"user tags" , "status" , "status_code","hierarchy"]
return
def configure(self, cfg_params):
self.cfg_params = cfg_params
self.jobtypeName = cfg_params.get('CRAB.jobtype', '')
self.schedulerName = cfg_params.get('CRAB.scheduler', '')
Scheduler.configure(self, cfg_params)
self.proxyValid = 0
self.dontCheckProxy = int(cfg_params.get("GRID.dont_check_proxy", 0))
self.space_token = cfg_params.get("USER.space_token", None)
try:
self.proxyServer = Downloader(
"http://cmsdoc.cern.ch/cms/LCG/crab/config/").config(
"myproxy_server.conf")
self.proxyServer = self.proxyServer.strip()
if self.proxyServer is None:
raise CrabException("myproxy_server.conf retrieved but empty")
except Exception, e:
common.logger.info(
"Problem setting myproxy server endpoint: using myproxy.cern.ch"
)
common.logger.debug(e)
self.proxyServer = 'myproxy.cern.ch'
self.group = cfg_params.get("GRID.group", None)
self.role = cfg_params.get("GRID.role", None)
removeBList = cfg_params.get("GRID.remove_default_blacklist", 0)
blackAnaOps = None
if int(removeBList) == 0:
blacklist = Downloader(
"http://cmsdoc.cern.ch/cms/LCG/crab/config/")
result = blacklist.config("site_black_list.conf")
if result != None:
blackAnaOps = result
common.logger.debug("Enforced black list: %s " % blackAnaOps)
else:
common.logger.info("WARNING: Skipping default black list!")
self.EDG_ce_black_list = None
if cfg_params.has_key(
'GRID.ce_black_list') and cfg_params['GRID.ce_black_list']:
self.EDG_ce_black_list = cfg_params.get('GRID.ce_black_list')
if int(removeBList) == 0 and blackAnaOps:
self.EDG_ce_black_list += ",%s" % blackAnaOps
elif int(removeBList) == 0 and blackAnaOps:
self.EDG_ce_black_list = blackAnaOps
if self.EDG_ce_black_list:
self.EDG_ce_black_list = str(self.EDG_ce_black_list).split(',')
self.EDG_ce_white_list = cfg_params.get('GRID.ce_white_list', None)
if (self.EDG_ce_white_list):
self.EDG_ce_white_list = str(self.EDG_ce_white_list).split(',')
self.VO = cfg_params.get('GRID.virtual_organization', 'cms')
self.EDG_clock_time = cfg_params.get('GRID.max_wall_clock_time', None)
# Default minimum CPU time to >= 130 minutes
self.EDG_cpu_time = cfg_params.get('GRID.max_cpu_time', '130')
## Add EDG_WL_LOCATION to the python path
#if not self.CRAB_useServer and not self.CRAB_serverName:
# if not os.environ.has_key('EDG_WL_LOCATION'):
# msg = "Error: the EDG_WL_LOCATION variable is not set."
# raise CrabException(msg)
# path = os.environ['EDG_WL_LOCATION']
# libPath=os.path.join(path, "lib")
# sys.path.append(libPath)
# libPath=os.path.join(path, "lib", "python")
# sys.path.append(libPath)
self.checkProxy()
return
from Controlador import Controlador
from Downloader import Downloader
if __name__ == '__main__':
control = Controlador()
Vista = Downloader(control)
control.setVista(Vista)
control.iniciar()
class SchedulerGrid(Scheduler):
def __init__(self, name):
Scheduler.__init__(self,name)
self.states = [ "Acl", "cancelReason", "cancelling","ce_node","children", \
"children_hist","children_num","children_states","condorId","condor_jdl", \
"cpuTime","destination", "done_code","exit_code","expectFrom", \
"expectUpdate","globusId","jdl","jobId","jobtype", \
"lastUpdateTime","localId","location", "matched_jdl","network_server", \
"owner","parent_job", "reason","resubmitted","rsl","seed",\
"stateEnterTime","stateEnterTimes","subjob_failed", \
"user tags" , "status" , "status_code","hierarchy"]
return
def configure(self, cfg_params):
self.cfg_params = cfg_params
self.jobtypeName = cfg_params.get('CRAB.jobtype','')
self.schedulerName = cfg_params.get('CRAB.scheduler','')
Scheduler.configure(self,cfg_params)
self.proxyValid=0
self.dontCheckProxy=int(cfg_params.get("GRID.dont_check_proxy",0))
self.space_token = cfg_params.get("USER.space_token",None)
try:
self.proxyServer = Downloader("http://cmsdoc.cern.ch/cms/LCG/crab/config/").config("myproxy_server.conf")
self.proxyServer = self.proxyServer.strip()
if self.proxyServer is None:
raise CrabException("myproxy_server.conf retrieved but empty")
except Exception, e:
common.logger.info("Problem setting myproxy server endpoint: using myproxy.cern.ch")
common.logger.debug(e)
self.proxyServer= 'myproxy.cern.ch'
self.group = cfg_params.get("GRID.group", None)
self.role = cfg_params.get("GRID.role", None)
removeBList = cfg_params.get("GRID.remove_default_blacklist", 0 )
blackAnaOps = None
if int(removeBList) == 0:
blacklist = Downloader("http://cmsdoc.cern.ch/cms/LCG/crab/config/")
result = blacklist.config("site_black_list.conf")
if result != None:
blackAnaOps = result
common.logger.debug("Enforced black list: %s "%str(blacklist))
else:
common.logger.info("WARNING: Skipping default black list!")
self.EDG_ce_black_list = None
if cfg_params.has_key('GRID.ce_black_list') and cfg_params['GRID.ce_black_list']:
self.EDG_ce_black_list = cfg_params.get('GRID.ce_black_list')
if int(removeBList) == 0 and blackAnaOps:
self.EDG_ce_black_list += ",%s"%blackAnaOps
elif int(removeBList) == 0 and blackAnaOps:
self.EDG_ce_black_list = blackAnaOps
if self.EDG_ce_black_list:
self.EDG_ce_black_list = str(self.EDG_ce_black_list).split(',')
self.EDG_ce_white_list = cfg_params.get('GRID.ce_white_list',None)
if (self.EDG_ce_white_list): self.EDG_ce_white_list = str(self.EDG_ce_white_list).split(',')
self.VO = cfg_params.get('GRID.virtual_organization','cms')
self.EDG_clock_time = cfg_params.get('GRID.max_wall_clock_time',None)
# Default minimum CPU time to >= 130 minutes
self.EDG_cpu_time = cfg_params.get('GRID.max_cpu_time', '130')
## Add EDG_WL_LOCATION to the python path
#if not self.CRAB_useServer and not self.CRAB_serverName:
# if not os.environ.has_key('EDG_WL_LOCATION'):
# msg = "Error: the EDG_WL_LOCATION variable is not set."
# raise CrabException(msg)
# path = os.environ['EDG_WL_LOCATION']
# libPath=os.path.join(path, "lib")
# sys.path.append(libPath)
# libPath=os.path.join(path, "lib", "python")
# sys.path.append(libPath)
self.checkProxy()
return
addon_path).doModal()
del gui
elif window == "trailers":
from TrailerWindow import TrailerWindow
gui = TrailerWindow(u'script-%s-trailers.xml' % addon_name,
addon_path).doModal()
del gui
elif window == "favourites":
from FavWindow import FavWindow
gui = FavWindow(u'script-globalsearch-main.xml',
addon_path).doModal()
del gui
elif window == "dialogalbuminfo":
from DialogAlbumInfo import DialogAlbumInfo
gui = DialogAlbumInfo(
u'script-%s-dialogalbuminfo.xml' % addon_name,
addon_path).doModal()
del gui
elif window == "fullscreeninfo":
from FullscreenInfo import FullscreenInfo
gui = FullscreenInfo(u'script-%s-fullscreeninfo.xml' % addon_name,
addon_path).doModal()
del gui
elif window == "downloader":
from Downloader import Downloader
dialog = Downloader()
dialog.show_download_dialog()
del dialog
else:
MoveProperties(container, focuscontrol)
def scrapeHaloOglasi():
# UTF-8 support
reload(sys)
sys.setdefaultencoding('utf-8')
now = datetime.now()
down = Downloader(
'http://www.halooglasi.com/naslovna.240.html?search_text=&sortColumn=VremeDodavanja'
)
content = down.get_content()
html = unicode(content)
celo = xpath.get(html, '//div[@class="results_container"]')
linkovi = xpath.search(celo, '//div[@class="result_brza"]')
ads = []
for l in linkovi:
link = xpath.get(l, '//div[@style="height:auto;"]/h2/a/@href')
link = "http://www.halooglasi.com" + link
download = Downloader(link)
cont = download.get_content()
cont = unicode(cont)
title = xpath.get(cont, '//div[@class="detail_bar_nek"]/h2').strip()
if title == "":
title = xpath.get(cont, '//div[@class="detail_bar"]/h2').strip()
imageUrl = xpath.get(l, '//a[@class="thumb"]/img/@src')
imageUrl = "http://www.halooglasi.com" + imageUrl
description = xpath.get(l, '//div[@class="text_ogl"]/p')
kategorija = xpath.get(l, '//div[@class="brza_link"]').strip()
kategorija = kategorija.split("\r\n\t\t\t\t\t\t\r\n\t\t\t\t\t\t")
kategorija = kategorija[1]
kategorija = kategorija.split(" > ")
category = kategorija[0]
if len(kategorija) > 2:
subcategory = kategorija[1]
else:
kategorija = kategorija[1].split("'>")
kategorija = kategorija[1]
kategorija = kategorija.split("<")
subcategory = kategorija[0]
price = xpath.get(cont, '//div[@class="price"]').strip() #price
if price == "":
price = xpath.get(
cont, '//div[@class="price deal"]').strip() #price deal
price = price.replace(".", "")
price = price.replace("din", " DIN")
price = price.replace("€", " EUR")
if price == "Dogovor":
value = "/"
currency = "/"
else:
price = price.split(" ")
value = price[0]
currency = price[1]
date_loc = xpath.search(l, '//div[@class="datum_grad"]/h6/span')
date_loc[0] = date_loc[0].strip()
date = date_loc[0].split("\r\n")
date = date[0]
date = date.replace(".", "")
date = date.split(" ")
if date[1] == "Jan":
date[1] = "1"
elif date[1] == "Feb":
date[1] = "2"
elif date[1] == "Mar":
date[1] = "3"
elif date[1] == "Apr":
date[1] = "4"
elif date[1] == "Maj":
date[1] = "5"
elif date[1] == "Jun":
date[1] = "6"
elif date[1] == "Jul":
date[1] = "7"
elif date[1] == "Avg":
date[1] = "8"
elif date[1] == "Sep":
date[1] = "9"
elif date[1] == "Okt":
date[1] = "10"
elif date[1] == "Nov":
date[1] = "11"
elif date[1] == "Dec":
date[1] = "12"
date = date[2] + "-" + date[1] + "-" + date[0]
l = date_loc[1].strip()
l = l.split(" ")
region = l[0]
country = u"Србија"
ad = Ad(link, title, imageUrl, description, category, subcategory,
value, currency, region, date, country)
ads.append(ad)
return adsToJson(ads)
#print scrapeHaloOglasi()
class ServerConfig:
def __init__(self, serverName):
import string
self.serverName = string.lower(serverName)
common.logger.debug('Calling ServerConfig '+self.serverName)
url ='http://cmsdoc.cern.ch/cms/LCG/crab/config/'
self.downloader = Downloader(url)
def config(self):
"""
"""
if 'default' in self.serverName:
self.serverName = self.selectServer()
if 'server_' in self.serverName:
configFileName = '%s.conf'%self.serverName
else:
configFileName = 'server_%s.conf'%self.serverName
serverConfig = eval(self.downloader.config(configFileName))
if not serverConfig:
serverConfig = {}
serverConfig['serverGenericName']=self.serverName
return serverConfig
def selectServer(self):
"""
"""
common.logger.debug('getting serverlist from web')
# get a list of available servers
serverListFileName ='AvailableServerList'
serverListFile = self.downloader.config(serverListFileName)
if not serverListFile:
msg = 'List of avalable Server '+serverListFileName+' from '+self.url+' is empty\n'
msg += 'Please report to CRAB feedback hypernews [email protected]'
raise CrabException(msg)
# clean up empty lines and comments
serverList=[]
[serverList.append(string.split(string.strip(it))) for it in serverListFile.split('\n') if (it.strip() and not it.strip()[0]=="#")]
common.logger.debug('All avaialble servers: '+str(serverList))
# select servers from client version
compatibleServerList=[]
for s in serverList:
#vv=string.split(s[1],'-')
if len(s)<2:
continue
vv=s[1].split('-')
if len(vv[0])==0: vv[0]='0.0.0'
if len(vv[1])==0: vv[1]='99.99.99'
for i in 0,1:
tmp=[]
[tmp.append(int(t)) for t in vv[i].split('.')]
vv[i]=tuple(tmp)
if vv[0]<=common.prog_version and common.prog_version<=vv[1] and common.scheduler.name()==string.lower(s[2]):
compatibleServerList.append(s[0])
common.logger.debug('All avaialble servers compatible with %s: '%common.prog_version_str +str(compatibleServerList))
if len(compatibleServerList)==0:
msg = "No compatible server for client version %s and scheduler %s\n"%(common.prog_version_str,common.scheduler.name())
msg += "Exiting"
common.logger.info(msg)
raise CrabException(msg)
# if more than one, pick up a random one, waiting for something smarter (SiteDB)
import random
serverName = random.choice(compatibleServerList)
common.logger.debug('Avaialble servers: '+str(compatibleServerList)+' choosen: '+serverName)
return serverName
def threader_crawler(seed_url,
resource_regiex=None,
link_regiex=".*",
delay=5,
cache=None,
download_source_callback=None,
user_agent="wswp",
proxies=None,
num_retries=1,
max_threads=10,
timeout=60,
max_url=500):
downloaded = []
crawl_queue = [seed_url]
seen = set([seed_url])
D = Downloader(cache=cache,
delay=delay,
user_agent=user_agent,
proxies=proxies,
num_retries=num_retries,
timeout=timeout)
print(user_agent)
def process_queue():
while True:
links = []
try:
url = crawl_queue.pop()
except IndexError:
break
else:
html = D(url)
downloaded.append(url)
if download_source_callback:
if resource_regiex and re.match(resource_regiex, url):
download_source_callback(url, html)
links.extend([
link for link in get_links(html)
if re.match(link_regiex, link)
])
for link in links:
link = normalize(seed_url, link)
if link not in seen:
seen.add(link)
if same_domain(seed_url, link):
crawl_queue.append(link)
print("已经发现的总网页数目为", len(seen))
print("已经下载过的网页数目为", len(downloaded))
print("还没有遍历过的网页数目为", len(crawl_queue))
threads = []
while threads or crawl_queue:
if len(downloaded) == max_url:
return
for thread in threads:
if not thread.is_alive():
threads.remove(thread)
while len(threads) < max_threads and crawl_queue:
print("线程数量为", len(threads))
thread = threading.Thread(target=process_queue)
thread.setDaemon(True)
thread.start()
print("线程数量为", len(threads))
threads.append(thread)
def run(self):
tempPath = path + self.video.name + ".mp4"
self.downLoader = Downloader(self.video.videoUrl, tempPath)
self.downLoader.downLoad()
self.singal.emit()
def __init__(self):
print 'init'
self.urlManager = UrlManager()
self.downloader = Downloader()
self.praser = HtmlPraser()
self.outputer = Output()
def download32bit(self):
url32bit = 'https://go.microsoft.com/fwlink/?LinkID=623231'
Downloader(self.directoryName,self.IDEName,url32bit)
class ServerConfig:
def __init__(self, serverName):
import string
self.serverName = string.lower(serverName)
common.logger.debug('Calling ServerConfig ' + self.serverName)
url = 'http://cmsdoc.cern.ch/cms/LCG/crab/config/'
self.downloader = Downloader(url)
def config(self):
"""
"""
if 'default' in self.serverName:
self.serverName = self.selectServer()
if 'server_' in self.serverName:
configFileName = '%s.conf' % self.serverName
else:
configFileName = 'server_%s.conf' % self.serverName
serverConfig = eval(self.downloader.config(configFileName))
if not serverConfig:
serverConfig = {}
serverConfig['serverGenericName'] = self.serverName
return serverConfig
def selectServer(self):
"""
"""
common.logger.debug('getting serverlist from web')
# get a list of available servers
serverListFileName = 'AvailableServerList'
serverListFile = self.downloader.config(serverListFileName)
if not serverListFile:
msg = 'List of avalable Server ' + serverListFileName + ' from ' + self.url + ' is empty\n'
msg += 'Please report to CRAB feedback hypernews [email protected]'
raise CrabException(msg)
# clean up empty lines and comments
serverList = []
[
serverList.append(string.split(string.strip(it)))
for it in serverListFile.split('\n')
if (it.strip() and not it.strip()[0] == "#")
]
common.logger.debug('All avaialble servers: ' + str(serverList))
# select servers from client version
compatibleServerList = []
for s in serverList:
#vv=string.split(s[1],'-')
if len(s) < 2:
continue
vv = s[1].split('-')
if len(vv[0]) == 0: vv[0] = '0.0.0'
if len(vv[1]) == 0: vv[1] = '99.99.99'
for i in 0, 1:
tmp = []
[tmp.append(int(t)) for t in vv[i].split('.')]
vv[i] = tuple(tmp)
if vv[0] <= common.prog_version and common.prog_version <= vv[
1] and common.scheduler.name() == string.lower(s[2]):
compatibleServerList.append(s[0])
common.logger.debug('All avaialble servers compatible with %s: ' %
common.prog_version_str +
str(compatibleServerList))
if len(compatibleServerList) == 0:
msg = "No compatible server for client version %s and scheduler %s\n" % (
common.prog_version_str, common.scheduler.name())
msg += "Exiting"
common.logger.info(msg)
raise CrabException(msg)
# if more than one, pick up a random one, waiting for something smarter (SiteDB)
import random
serverName = random.choice(compatibleServerList)
common.logger.debug('Avaialble servers: ' + str(compatibleServerList) +
' choosen: ' + serverName)
return serverName
def download64bit(self):
url64bit = 'https://go.microsoft.com/fwlink/?Linkid=850641'
Downloader(self.directoryName,self.IDEName,url64bit)
def link_crawler(seed_url,
link_regex=None,
delay=5,
max_depth=-1,
max_urls=-1,
headers=None,
proxies=None,
cookies=None,
num_retries=5,
scrape_callback=None,
cache=None,
max_threads=10,
timeout=60):
"""Crawl from the given seed URL following links matched by link_regex
Crawl this website in mltiple threads
seed_url:种子链接，爬虫的初始链接
delay:避免请求太频繁，所采取的延时
max_depth:从种子链接向下，所链接的深度
max_urls:连续max_urls次下载同一个网页，得到404后，停止下载，跳出循环
headers:请求头
proxy:传入ProxiesPool类
num_retries:当网页的返回识别码在400和500之间时，重复下载的数，如果还不能下载返回空字符串
cahce:是否存在缓存，用法例如：
cache = MongoCache(expires=timedelta())
D = Downloader(cache=cache)
html = D('http://esf.nb.fang.com/')
"""
# the queue of URL's that still need to be crawled
crawl_queue = deque([seed_url])
# the URL's that have been seen and at what depth
seen = {seed_url: 0}
# track how many URL's have been downloaded
num_urls = {seed_url: 0}
# rp = get_robots(seed_url)
headers = headers
D = Downloader(delay=5,
headers=headers,
cookies=cookies,
proxies=proxies,
num_retries=num_retries,
cache=cache)
def process_queue():
while True:
try:
url = crawl_queue.pop()
except IndexError:
break
else:
download_time = 1
while download_time <= 5:
html = D(url)
download_time = download_time + 1
print('we download the url %s times' % download_time)
links = []
if scrape_callback:
links.extend(scrape_callback(url, html) or [])
next_urls = get_links(html)
if next_urls:
seen[seed_url] = seen[seed_url] + 1
break
if link_regex:
links.extend(link for link in next_urls
if re.match(link_regex, link))
else:
links.extend(link for link in next_urls)
for link in links:
if seen[seed_url] != max_depth:
link = normalize(seed_url, link)
# check whether already crawled this link
if link not in seen and same_domain(seed_url, link):
# check link is within same domain
# success! add this new link to queue
crawl_queue.append(link)
# check whether have reached downloaded maximum
try:
num_urls[url] += 1
except KeyError:
num_urls[url] = 1
if num_urls[url] == max_urls:
break
threads = []
while threads or crawl_queue:
# the crawl is still activate
for thread in threads:
if not thread.is_alive():
# remove the stopped threads
threads.remove(thread)
while len(threads) < max_threads and crawl_queue:
# can start some more threads
thread = threading.Thread(target=process_queue)
# set daemon so main thread can exit when receives ctrl-c
thread.setDaemon(True)
thread.start()
# thread.run()
threads.append(thread)
def scrapeAvtooglasi():
# UTF-8 support
reload(sys)
sys.setdefaultencoding('utf-8')
now = datetime.now()
down = Downloader('http://www.avtooglasi.com.mk/rezultati/show/?vid=0&orderby=0')
content = down.get_content()
html = unicode(content)
sliki = xpath.search(html, '//div[@class="resultLeft"]')
ostanato = xpath.search(html, '//div[@class="oglasInfoTopContent"]')
ceni = xpath.search(html, '//a[@class="btn btn-info btn-xs oglasInfoAdditionalPrice"]')
link = {}
title = {}
imageUrl = {}
description = {}
category = {}
subcategory = {}
value = {}
currency = {}
region = {}
date = {}
i = 0
ads = []
for slika in sliki:
imageUrl[i] = xpath.search(slika, '//a[@class="thumbnail resultImg"]/img/@src')[0]
i = i + 1
i = 0
for cena in ceni:
price = xpath.get(cena,'//span/span').strip()
price=price.split(" ")
if len(price)>1:
if price[0]=="По":
price[0]="/"
if price[1]=="договор":
price[1]="/"
value[i]=price[0]
currency[i]=price[1]
if currency[i]=="€":
currency[i]="EUR"
i = i + 1
i = 0
for advert in ostanato:
link[i] = xpath.get(advert, '//a[@class="resultMainLink"]/@href')
title[i] = xpath.get(advert, '//a[@class="resultMainLink"]/span').strip().replace("\"", "")
path = xpath.search(getDescription(link[i],'//div[@class="centerC"]'), '/div/div[@class="padded"]')
description[i] = path[1]
subcategory[i]="/"
category[i] = u"Возила"
dodatok = xpath.get(advert, '//span[@class="oglasInfoAdditionalInfo"]')
dodatok = dodatok.split(" | ")
region[i] = dodatok[0]
country = u"Македеонија"
description[i] = dodatok[1] + u" година, "+ dodatok[2] +", "+ dodatok[3] +", "+ dodatok[4] +", "+ dodatok[5] +", "+ description[i]
description[i] = description[i].strip().replace("\"", "")
date[i]=""
#print description[i]
datum = dodatok[6].strip()
datum = datum.split(" ")
if datum[0]=="Денес":
datum [0]= str(now.year)+"-"+str(now.month)+"-"+str(now.day)
date[i]=datum[0]+" "+datum[2]
elif datum[0]=="Вчера":
da=datetime.now()-timedelta(days=1)
datum[0]=str(da.year)+"-"+str(da.month)+"-"+str(da.day)
date[i]=datum[0]+" "+datum[2]
elif datum[0]=="пред":
if datum[2]=="дена":
da=datetime.now()-timedelta(days=int(datum[1]))
datum[0]=str(da.year)+"-"+str(da.month)+"-"+str(da.day)
date[i]=datum[0]
else:
if datum[1]=="1":
da=datetime.now()-timedelta(days=30)
datum[0]=str(da.year)+"-"+str(da.month)+"-"+str(da.day)
date[i]=datum[0]
else:
da=datetime.now()-timedelta(days=60)
datum[0]=str(da.year)+"-"+str(da.month)+"-"+str(da.day)
date[i]=datum[0]
else:
date[i]=datum[0]+" "+datum[1]
#print date[i]
i = i + 1
for i in link:
ad = Ad(link[i], title[i], imageUrl[i], description[i], category[i], subcategory[i], value[i], currency[i], region[i], date[i], country)
ads.append(ad)
return adsToJson(ads)
# print scrapeAvtooglasi()
def start(self):
# Open the portolio
self.portfolio = Portfolio(cash=0, sl=self.sl, tp=self.tp)
self.portfolio.load(self.filename)
# Download the stock data
self.downloader = Downloader(cache=False)
self.downloader.download(period=self.period)
date = self.downloader.prices.index[-1]
prices = self.downloader.prices
print("#", date)
# Refresh the portfolio with the latest prices
self.portfolio.refresh(prices=prices, date=date)
# Get the portfolio Summary
portfolio_summary = self.portfolio.summary()
# Get the opened positions
opened = self.portfolio.holdings[self.portfolio.holdings['status'] ==
'open']
# Get the current trend
zt = TradingZones(prices=prices)
zones = zt.get()
output = pd.DataFrame()
# Activate the trailing stops
targets = self.portfolio.getPositionsBelowTS(self.ts)
if len(targets) > 0:
sell_orders = self.portfolio.getSellOrdersFromSubset(
subset=targets)
print("\n----- TS -----\n", sell_orders)
#self.portfolio.sell(sell_orders, prices, date, label='TS')
# Activate the stop loss
if self.sl < 0:
poor_perfs = opened[opened['profits_pct'] <= self.sl]
if len(poor_perfs) > 0:
sell_list = poor_perfs['symbol'].unique()
sell_orders = self.portfolio.getSellAllOrders(
symbols=sell_list)
print("\n----- SL -----\n", sell_orders)
#self.portfolio.sell(sell_orders, prices, date, label='SL')
# Sell all on drop
if self.sellAllOnCrash == True and zones.iloc[-1][
'signal'] == 'DOWN' and len(opened) > 0:
print("Sell all")
print("\n----- Sell All -----\n")
#self.portfolio.sellAll(prices, date, label='Market Drop')
# Get the ones that reached take-profit
if self.tp > 0:
good_perfs = opened[opened['profits_pct'] >= self.tp]
if len(good_perfs) > 0:
sell_list = good_perfs['symbol'].unique()
sell_orders = self.portfolio.getSellAllOrders(
symbols=sell_list)
print("\n----- TP -----\n", sell_orders)
#self.portfolio.sell(sell_orders, prices, date, label='TP')
# Avoid trading in a general market downtrend
if self.avoidDowntrends == False or (self.avoidDowntrends == True and
zones.iloc[-1]['signal'] == 'UP'):
# Pick the stocks
picker = Picker(prices=prices)
picker.pick(count=self.stockPicks)
#print(">", picker.selected)
# Get the order size
portfolio_summary = self.portfolio.summary()
order = Order(prices=prices, picks=picker.selected)
order.get(budget=portfolio_summary['cash'])
#print("!", portfolio_summary['cash'], order.orders)
print("\n----- BUY -----\n", order.orders)
# Update our portfolio
#self.portfolio.update(orders=order.orders, prices=prices, date=date)
else:
# No new positions in a downtrend!
# Update our portfolio
self.portfolio.update(orders=pd.DataFrame(),
prices=prices,
date=date)
def threaded_crawler(seed_url,
delay=5,
cache=None,
scrape_callback=None,
proxies=None,
num_retries=1,
max_threads=10,
timeout=60):
"""Crawl this website in multiple threads
seed_url must be list
"""
# the queue of URL's that still need to be crawled
crawl_queue = deque(seed_url)
# the URL's that have been seen
seen = dict((url, 0) for url in seed_url)
D = Downloader(cache=cache,
delay=delay,
proxies=proxies,
num_retries=num_retries)
def process_queue():
while True:
try:
url = crawl_queue.pop()
except IndexError:
# crawl queue is empty
break
else:
html = D(url)
if scrape_callback:
try:
links = scrape_callback(url, html) or []
except Exception as e:
print('Error in callback for: {}: {}'.format(url, e))
else:
for link in links:
link = normalize(seed_url, link)
# check whether already crawled this link
if link not in seen:
# add this new link to queue
crawl_queue.appendleft(link)
# wait for all download threads to finish
threads = []
while threads or crawl_queue:
# the crawl is still active
for thread in threads:
if not thread.is_alive():
# remove the stopped threads
threads.remove(thread)
while len(threads) < max_threads and crawl_queue:
# can start some more threads
thread = threading.Thread(target=process_queue)
thread.setDaemon(
True
) # set daemon so main thread can exit when receives ctrl-c
thread.start()
threads.append(thread)
# all threads have been processed
# sleep temporarily so CPU can focus execution on other threads
time.sleep(SLEEP_TIME)
from Downloader import Downloader
url_metadata = "https://os.unil.cloud.switch.ch/fma/fma_metadata.zip"
url_small = "https://os.unil.cloud.switch.ch/fma/fma_small.zip"
url_medium = "https://os.unil.cloud.switch.ch/fma/fma_medium.zip"
url_large = "https://os.unil.cloud.switch.ch/fma/fma_large.zip"
url_full = "https://os.unil.cloud.switch.ch/fma/fma_full.zip"
sha1sum_metadata = 'f0df49ffe5f2a6008d7dc83c6915b31835dfe733'
sha1sum_small = 'ade154f733639d52e35e32f5593efe5be76c6d70'
sha1sum_medium = 'c67b69ea232021025fca9231fc1c7c1a063ab50b'
sha1sum_large = '497109f4dd721066b5ce5e5f250ec604dc78939e'
sha1sum_full = '0f0ace23fbe9ba30ecb7e95f763e435ea802b8ab'
threads = 10
code_geass = 'https://ca-east.uploadfiles.io/get/6tnazj0v'
obj = Downloader(url=code_geass,
number_of_threads=threads,
destination_directory='D:\\Downloads')
obj.start_download()
# obj.merge_chunks()
print(obj.get_metadata())
print(obj.get_remote_crc32c())
print(obj.get_downloaded_crc32c_large())
print(obj.get_downloaded_sha1_large())
class Sim:
def __init__(self,
period='2y',
timedelay=100,
window=100,
timestep=5,
budget=5000,
stockPicks=10,
sl=-0.1,
tp=0.25,
ts=0.2,
avoidDowntrends=True,
sellAllOnCrash=True,
neptune=None,
pick_kwargs={}):
self.neptune = neptune # neptune.ai logging
self.period = period
self.timedelay = timedelay
self.timestep = timestep
self.budget = budget
self.stockPicks = stockPicks
self.sl = sl
self.tp = tp
self.ts = ts
self.avoidDowntrends = avoidDowntrends
self.sellAllOnCrash = sellAllOnCrash
self.pick_kwargs = pick_kwargs
self.portfolio = Portfolio(cash=budget, sl=self.sl, tp=self.tp)
self.downloader = Downloader()
self.downloader.download(period=self.period)
self.current_index = timedelay
self.is_notebook = self.isNotebook()
def isNotebook(self):
try:
cfg = get_ipython().config
return True
except NameError:
return False
def progress(self, count, total=100, status=''):
if self.is_notebook == True:
return HTML("""
<progress
value='{value}'
max='{max}',
style='width: 50%'
>
{value}
</progress>
""".format(value=count, max=total))
else:
bar_len = 60
filled_len = int(round(bar_len * count / float(total)))
percents = round(100.0 * count / float(total), 1)
bar = '=' * filled_len + '-' * (bar_len - filled_len)
sys.stdout.write('[%s] %s%s\r' % (bar, percents, '%'))
sys.stdout.flush()
def run(self):
#out = display(self.progress(0, 100), display_id=True)
steps = (len(self.downloader.prices) - self.timedelay)
stepN = 0
self.stats = pd.DataFrame(columns=[
'invested', 'portfolio_value', 'Positions', 'closed', 'opened',
'cash', 'total_value', 'portfolio_resistances',
'portfolio_supports', 'drawdown', 'drawdown_pct'
])
while self.current_index + self.timestep < len(self.downloader.prices):
output = self.tick()
# Neptune logging
if self.neptune is not None:
for k in list(output.keys()):
self.neptune.log_metric(k, output[k])
output['date'] = self.downloader.prices.index[self.current_index]
self.stats = self.stats.append(output, ignore_index=True)
stepN = stepN + 1
pct = math.ceil((stepN / steps) * 100)
if self.is_notebook == True:
out.update(self.progress(pct, 100))
else:
self.progress(pct, 100)
#print(str(round(pct))+"%")
self.portfolio.sellAll(
prices=self.downloader.prices[:self.current_index],
date=self.downloader.prices.index[self.current_index],
label='End Sim')
self.stats = self.stats.set_index('date')
return self.stats
# Run every night after the market close
def tick(self):
#print('\n\n----------- tick -----------')
# Get the latest prices & date as of today
date = self.downloader.prices.index[self.current_index]
prices = self.downloader.prices[:self.current_index +
1] # Include the current price
#print("#", date)
# Refresh the portfolio with the latest prices
self.portfolio.refresh(prices=prices, date=date)
# Get the portfolio Summary
portfolio_summary = self.portfolio.summary()
# Get the opened positions
opened = self.portfolio.holdings[self.portfolio.holdings['status'] ==
'open']
# Get the current trend
zt = TradingZones(prices=prices)
zones = zt.get()
# Activate the trailing stops
targets = self.portfolio.getPositionsBelowTS(self.ts)
if len(targets) > 0:
sell_orders = self.portfolio.getSellOrdersFromSubset(
subset=targets)
#print("\n-----------\n",targets,"\n\n",sell_orders)
self.portfolio.sell(sell_orders, prices, date, label='TS')
# Activate the stop loss
if self.sl < 0:
poor_perfs = opened[opened['profits_pct'] <= self.sl]
if len(poor_perfs) > 0:
sell_list = poor_perfs['symbol'].unique()
sell_orders = self.portfolio.getSellAllOrders(
symbols=sell_list)
self.portfolio.sell(sell_orders, prices, date, label='SL')
# Sell all on drop
if self.sellAllOnCrash == True and zones.iloc[-1][
'signal'] == 'DOWN' and len(opened) > 0:
print("Sell all")
self.portfolio.sellAll(prices, date, label='Market Drop')
# Get the ones that reached take-profit
if self.tp > 0:
good_perfs = opened[opened['profits_pct'] >= self.tp]
if len(good_perfs) > 0:
sell_list = good_perfs['symbol'].unique()
sell_orders = self.portfolio.getSellAllOrders(
symbols=sell_list)
self.portfolio.sell(sell_orders, prices, date, label='TP')
# Avoid trading in a general market downtrend
if self.avoidDowntrends == False or (self.avoidDowntrends == True and
zones.iloc[-1]['signal'] == 'UP'):
# Pick the stocks
picker = Picker(prices=prices)
picker.pick(count=self.stockPicks, kwargs=self.pick_kwargs)
#print(">", picker.selected)
# Get the order size
portfolio_summary = self.portfolio.summary()
order = Order(prices=prices, picks=picker.selected)
order.get(budget=portfolio_summary['cash'])
#print("!", portfolio_summary['cash'], order.orders)
# Update our portfolio
self.portfolio.update(
orders=order.orders,
prices=prices,
date=self.downloader.prices.index[self.current_index])
else:
# No new positions in a downtrend!
# Update our portfolio
self.portfolio.update(
orders=pd.DataFrame(),
prices=prices,
date=self.downloader.prices.index[self.current_index])
# Update the index
if self.current_index + self.timestep < len(self.downloader.prices):
self.current_index = self.current_index + self.timestep
portfolio_summary = self.portfolio.summary()
return portfolio_summary
from Downloader import Downloader
from Builder import Builder
from Utils import loadconfig
from Controller import Controller
if __name__ == '__main__':
# -----------------load config--------------------
configpath = "Config/config.json"
configs = loadconfig(configpath)
dir_config = configs["Dir"]
file_config = configs["File"]
# -----------------create classes--------------
builder = Builder(configpath)
downloader = Downloader(dir_config["Save_Path"])
controller = Controller(file_config["MusicList"])
# ------------------------------------------
controller = controller.init(downloader, builder)
controller.AutoDownload()
def link_crawler(seed_url,
link_regex=".*",
resource_regex=".*",
delay=5,
max_depth=-1,
max_urls=-1,
headers=None,
user_agent="wswp",
proxies=None,
num_retries=1,
download_source_callback=None,
cache=None,
data=None):
'''
'''
#用于存储未被下载的页面
craw_queue = [seed_url]
#用于存储已经下载过的页面
seen = [seed_url]
#下载过的url数量
num_urls = 0
isdow = []
downloader = Downloader(delay=delay,
user_agent=user_agent,
proxies=proxies,
num_retries=num_retries,
cache=cache)
while craw_queue:
url = craw_queue.pop()
#depth = seen[]
html = downloader(url)
print(html)
links = []
isdow.append(url)
seen.append(url)
if download_source_callback and re.match(resource_regex, url):
#如果是资源页面,下载页面上的所有资源
print("属于资源,开始下载")
download_source_callback(url, html)
#if depth != max_depth:
links.extend(
[link for link in get_links(html) if re.search(link_regex, link)])
for link in links:
link = normalize(seed_url, link)
if link not in seen:
seen.append(link)
if same_domain(seed_url, link):
craw_queue.append(link)
print(link + "符合要求加入队列,现在队列的长度是" + str(len(craw_queue)))
#craw_queue.append(link)
#print(link + "符合要求加入队列,现在队列的长度是" + str(len(craw_queue)))
print("当前可怕去的url数量胃" + str(len(craw_queue)))
print("已经发现的总url数量为", str(len(set(seen))))
print("已经下载过的url数量为", str(len(isdow)))
num_urls += 1
if num_urls == max_urls:
break
print(seed_url + "上所有符合要求的网站下载完毕,一共遍历了" + str(len(seen)) + "个网页")
def __init__( self ):
# Appel au constructeur de la classe mere
QtGui.QMainWindow.__init__( self )
###########
# Fenetre #
###########
###
# Reglages de la fenetre principale
###
# Nom de la fenetre
self.setWindowTitle( "TVDownloader" )
# Mise en place de son icone
self.setWindowIcon( QtGui.QIcon( "ico/TVDownloader.png" ) )
###
# Mise en place des widgets dans la fenetre
###
# Widget central qui contiendra tout
self.centralWidget = QtGui.QWidget( self )
#
# Barre du haut
#
# Layout horizontal qui contiendra les listes deroulantes
self.horizontalLayoutBarreHaut = QtGui.QHBoxLayout()
# Liste deroulante pour choisir le site (plugin)
self.comboBoxSite = QtGui.QComboBox( self.centralWidget )
self.horizontalLayoutBarreHaut.addWidget( self.comboBoxSite )
# Liste deroulante pour choisir une chaine du site courant
self.comboBoxChaine = QtGui.QComboBox( self.centralWidget)
self.horizontalLayoutBarreHaut.addWidget( self.comboBoxChaine )
# Liste deroulante pour choisir une emission de la chaine courante
self.comboBoxEmission = QtGui.QComboBox( self.centralWidget )
self.horizontalLayoutBarreHaut.addWidget( self.comboBoxEmission )
#
# Onglets
#
# Gestionnaire onglets
self.tabWidget = QtGui.QTabWidget( self.centralWidget )
# Onglet Fichiers
self.tabFichiers = QtGui.QSplitter( self.centralWidget ) # L'onglet Fichier contient un splitter
self.tabWidget.addTab( self.tabFichiers, u"Choix des fichiers" )
# Onglet Telechargements
self.tabTelechargements = QtGui.QWidget( self.centralWidget )
self.tabWidget.addTab( self.tabTelechargements, u"Téléchargements" )
#
# Liste des fichiers
#
# Layout de grille qui contient le tableau qui liste les fichiers + boutons
self.gridLayoutFichiers = QtGui.QGridLayout( self.tabFichiers )
# Tableau qui contient la liste des fichiers disponibles pour l'emission courante
self.tableWidgetFichier = MyQTableWidget( self.tabFichiers )
# Il a 4 colonnes et 0 ligne (pour l'instant)
self.tableWidgetFichier.setColumnCount( 3 )
self.tableWidgetFichier.setRowCount( 0 )
# On ajoute les titres
self.tableWidgetFichier.setHorizontalHeaderItem( 0,
self.tableWidgetFichier.creerItem( "" ) )
self.tableWidgetFichier.setHorizontalHeaderItem( 1,
self.tableWidgetFichier.creerItem( "Date" ) )
self.tableWidgetFichier.setHorizontalHeaderItem( 2,
self.tableWidgetFichier.creerItem( "Emission" ) )
# On l'ajoute au layout
self.gridLayoutFichiers.addWidget( self.tableWidgetFichier, 0, 1, 6, 1 )
# Icones du tableWidget
self.iconeFichier = QtGui.QIcon( "ico/gtk-file.svg" )
self.iconeAjoute = QtGui.QIcon( "ico/gtk-add.svg" )
self.iconeTelecharge = QtGui.QIcon( "ico/gtk-apply.svg" )
# Bouton pour ajouter tous les fichiers a la liste des telechargements
self.pushButtonToutAjouter = MyQPushButton( self.tabFichiers )
self.pushButtonToutAjouter.setIcon( QtGui.QIcon( "ico/gtk-add.svg" ) )
self.pushButtonToutAjouter.setToolTip( u"Ajouter tous les fichiers à la liste des téléchargements" )
self.gridLayoutFichiers.addWidget( self.pushButtonToutAjouter, 0, 0, 2, 1 )
# Bouton pour rafraichir le plugin courant
self.pushButtonRafraichirPlugin = MyQPushButton( self.tabFichiers )
self.pushButtonRafraichirPlugin.setIcon( QtGui.QIcon( "ico/gtk-refresh.svg" ) )
self.pushButtonRafraichirPlugin.setToolTip( "Rafraichir le plugin" )
self.gridLayoutFichiers.addWidget( self.pushButtonRafraichirPlugin, 2, 0, 2, 1 )
# Bouton pour ouvrir la fenetre des preferences du plugin courant
self.pushButtonPreferencesPlugin = MyQPushButton( self.tabFichiers )
self.pushButtonPreferencesPlugin.setIcon( QtGui.QIcon( "ico/gtk-preferences.svg" ) )
self.pushButtonPreferencesPlugin.setToolTip( u"Ouvrir les préférences du plugin" )
self.gridLayoutFichiers.addWidget( self.pushButtonPreferencesPlugin, 4, 0, 2, 1 )
# On met en place ce layout sur un widget (pour le splitter)
self.widgetFichiers = QtGui.QWidget()
self.widgetFichiers.setLayout( self.gridLayoutFichiers )
#
# Descriptif des fichiers
#
# Layout de grille
self.gridLayoutDescriptif = QtGui.QGridLayout()
# Label pour afficher un logo
self.logoFichierDefaut = QtGui.QPixmap()
self.logoFichierDefaut.load( "img/gtk-dialog-question.svg" )
self.labelLogo = QtGui.QLabel( self.centralWidget )
self.labelLogo.setPixmap( self.logoFichierDefaut.scaled( QtCore.QSize( 150, 150 ), QtCore.Qt.KeepAspectRatio ) )
self.gridLayoutDescriptif.addWidget( self.labelLogo, 0, 0, 1, 1 )
# Zone de texte pour afficher un descriptif
self.plainTextEdit = QtGui.QPlainTextEdit( self.centralWidget )
self.gridLayoutDescriptif.addWidget( self.plainTextEdit, 0, 1, 1, 2 )
# On met en place ce layout sur un widget (pour le splitter)
self.widgetDescriptif = QtGui.QWidget()
self.widgetDescriptif.setLayout( self.gridLayoutDescriptif )
# Onrientation verticale du splitter
self.tabFichiers.setOrientation( QtCore.Qt.Vertical )
# On ajoute les 2 elements au splitter (qui est notre onglet)
self.tabFichiers.addWidget( self.widgetFichiers )
self.tabFichiers.addWidget( self.widgetDescriptif )
#
# Liste des telechargements
#
# Layout de grille qui contient le tableau qui liste les fichiers a telecharger + les boutons pour le controller
self.gridLayoutTelechargement = QtGui.QGridLayout( self.tabTelechargements )
# Tableau qui contient la liste des fichiers a telecharger
self.tableWidgetTelechargement = MyQTableWidget( self.tabTelechargements )
# Il a 5 colonnes et 0 ligne (pour l'instant)
self.tableWidgetTelechargement.setColumnCount( 3 )
self.tableWidgetTelechargement.setRowCount( 0 )
# On ajoute le titre des 5 colonnes
self.tableWidgetTelechargement.setHorizontalHeaderItem( 0,
self.tableWidgetTelechargement.creerItem( "Date" ) )
self.tableWidgetTelechargement.setHorizontalHeaderItem( 1,
self.tableWidgetTelechargement.creerItem( "Emission" ) )
self.tableWidgetTelechargement.setHorizontalHeaderItem( 2,
self.tableWidgetTelechargement.creerItem( "Etat" ) )
# On l'ajoute au layout
self.gridLayoutTelechargement.addWidget( self.tableWidgetTelechargement, 0, 1, 4, 1 )
# Bouton pour monter l'element selectionne tout en haut de la liste
self.pushButtonExtremiteMonter = MyQPushButton( self.tabTelechargements )
self.pushButtonExtremiteMonter.setIcon( QtGui.QIcon( "ico/gtk-jump-to-rtl.svg" ) )
self.pushButtonExtremiteMonter.setToolTip( u"Placer l'élément sélectionné tout en haut" )
self.gridLayoutTelechargement.addWidget( self.pushButtonExtremiteMonter, 0, 0, 1, 1 )
# Bouton pour monter l'element selectionne d'un cran dans la liste
self.pushButtonMonter = MyQPushButton( self.tabTelechargements )
self.pushButtonMonter.setIcon( QtGui.QIcon( "ico/gtk-go-up.svg" ) )
self.pushButtonMonter.setToolTip( u"Monter l'élément sélectionné" )
self.gridLayoutTelechargement.addWidget( self.pushButtonMonter, 1, 0, 1, 1 )
# Bouton pour descendre l'element selectionne d'un cran dans la liste
self.pushButtonDescendre = MyQPushButton( self.tabTelechargements )
self.pushButtonDescendre.setIcon( QtGui.QIcon( "ico/gtk-go-down.svg" ) )
self.pushButtonDescendre.setToolTip( u"Descendre l'élément selectionné" )
self.gridLayoutTelechargement.addWidget( self.pushButtonDescendre, 2, 0, 1, 1 )
# Bouton pour descendre l'element selectionne tout en bas de la liste
self.pushButtonExtremiteDescendre = MyQPushButton( self.tabTelechargements )
self.pushButtonExtremiteDescendre.setIcon( QtGui.QIcon( "ico/gtk-jump-to-ltr.svg" ) )
self.pushButtonExtremiteDescendre.setToolTip( u"Placer l'élément sélectionné tout en bas" )
self.gridLayoutTelechargement.addWidget( self.pushButtonExtremiteDescendre, 3, 0, 1, 1 )
# Bouton pour supprimer tous les elements de la liste
self.pushButtonToutSupprimer = MyQPushButton( self.tabTelechargements )
self.pushButtonToutSupprimer.setIcon( QtGui.QIcon( "ico/gtk-cancel.svg" ) )
self.pushButtonToutSupprimer.setToolTip( u"Supprimer tous les téléchargements" )
self.gridLayoutTelechargement.addWidget( self.pushButtonToutSupprimer, 0, 2, 1, 1 )
# Bouton pour supprimer de la liste les telechargements termines
self.pushButtonNettoyer = MyQPushButton( self.tabTelechargements )
self.pushButtonNettoyer.setIcon( QtGui.QIcon( "ico/gtk-delete-full.svg" ) )
self.pushButtonNettoyer.setToolTip( u"Supprimer les téléchargement terminés" )
self.gridLayoutTelechargement.addWidget( self.pushButtonNettoyer, 1, 2, 1, 1 )
# Bouton pour ouvrir le dossier des telechargements
self.pushButtonOuvrirDossierTelechargement = MyQPushButton( self.tabTelechargements )
self.pushButtonOuvrirDossierTelechargement.setIcon( QtGui.QIcon( "ico/gtk-folder.svg" ) )
self.pushButtonOuvrirDossierTelechargement.setToolTip( u"Ouvrir le dossier des téléchargements" )
self.gridLayoutTelechargement.addWidget( self.pushButtonOuvrirDossierTelechargement, 2, 2, 1, 1 )
#
# Barre progression de telechargement d'un fichier
#
self.progressBarTelechargementFichier = QtGui.QProgressBar( self.centralWidget )
self.progressBarTelechargementFichier.setProperty( "value", 0 )
#
# Barre de progression de telechargement des fichiers
#
self.progressBarTelechargement = QtGui.QProgressBar( self.centralWidget )
self.progressBarTelechargement.setProperty( "value", 0 )
#
# Boutons du bas pour gerer ajouter/supprimer/lancer telechargements
#
# Layout horizontal qui contiendra les boutons
self.horizontalLayoutBarreBas = QtGui.QHBoxLayout()
# Bouton pour lancer les telechargements
self.pushButtonLancer = QtGui.QPushButton( QtGui.QIcon( "ico/gtk-media-play-ltr.svg" ), u"Lancer téléchargement", self.centralWidget )
self.horizontalLayoutBarreBas.addWidget( self.pushButtonLancer )
# Bouton pour stopper les telechargements
self.pushButtonStop = QtGui.QPushButton( QtGui.QIcon( "ico/gtk-media-stop.svg" ), u"Stopper le téléchargement", self.centralWidget )
self.pushButtonStop.setEnabled( False )
self.horizontalLayoutBarreBas.addWidget( self.pushButtonStop )
###
# Positionnement des differents widgets/layouts sur le layout de grille
###
# Layout de grille dans lequel on va placer nos widgets/layouts
self.gridLayout = QtGui.QGridLayout( self.centralWidget )
# On ajoute la barre du haut
self.gridLayout.addLayout( self.horizontalLayoutBarreHaut, 0, 0, 1, 3 )
# On ajoute le gestionnaire d'onglets
self.gridLayout.addWidget( self.tabWidget, 1, 0, 1, 3 )
# On ajoute la barre de progression de telechargement d'un fichier
self.gridLayout.addWidget( self.progressBarTelechargementFichier, 2, 0, 1, 3 )
# On ajoute la barre de progression de telechargement des fichiers
self.gridLayout.addWidget( self.progressBarTelechargement, 3, 0, 1, 3 )
# On ajoute les boutons ajouter/supprimer/lancer
self.gridLayout.addLayout( self.horizontalLayoutBarreBas, 4, 0, 1, 3 )
###
# Mise en place le central widget dans la fenetre
###
self.setCentralWidget( self.centralWidget )
###
# Mise en place du menu
###
# Menu barre
self.menubar = QtGui.QMenuBar( self )
self.menubar.setGeometry( QtCore.QRect( 0, 0, 480, 25 ) )
# Menu Fichier
self.menuFichier = QtGui.QMenu( "&Fichier", self.menubar )
self.menubar.addAction( self.menuFichier.menuAction() )
# Action Fichier -> Quitter
self.actionQuitter = QtGui.QAction( QtGui.QIcon( "ico/gtk-quit.svg" ), "&Quitter", self )
self.actionQuitter.setIconVisibleInMenu( True )
self.menuFichier.addAction( self.actionQuitter )
# Menu Edition
self.menuEdition = QtGui.QMenu( "&Edition", self.menubar )
self.menubar.addAction( self.menuEdition.menuAction() )
# Action Edition -> Mise a jour
self.actionMAJ = QtGui.QAction( QtGui.QIcon( "ico/gtk-refresh.svg" ), u"&Mise à jour des plugins", self )
self.actionMAJ.setIconVisibleInMenu( True )
self.menuEdition.addAction( self.actionMAJ )
# Action Edition -> Preferences
self.actionPreferences = QtGui.QAction( QtGui.QIcon( "ico/gtk-preferences.svg" ), u"&Préférences", self )
self.actionPreferences.setIconVisibleInMenu( True )
self.menuEdition.addAction( self.actionPreferences )
# Menu Aide
self.menuAide = QtGui.QMenu( "&Aide", self.menubar )
self.menubar.addAction( self.menuAide.menuAction() )
# Action Aide -> A propos
self.actionAPropos = QtGui.QAction( QtGui.QIcon( "ico/gtk-about.svg" ), u"À p&ropos", self )
self.actionAPropos.setIconVisibleInMenu( True )
self.menuAide.addAction( self.actionAPropos )
# Ajout du menu a l'interface
self.setMenuBar( self.menubar )
###
# Signaux provenants de l'interface
###
QtCore.QObject.connect( self.tableWidgetFichier,
QtCore.SIGNAL( "cellClicked(int,int)" ),
self.afficherInformationsFichier )
QtCore.QObject.connect( self.tableWidgetFichier,
QtCore.SIGNAL( "cellDoubleClicked(int,int)" ),
self.gererTelechargement )
QtCore.QObject.connect( self.pushButtonToutAjouter,
QtCore.SIGNAL( "clicked()" ),
self.ajouterTousLesFichiers )
QtCore.QObject.connect( self.pushButtonRafraichirPlugin,
QtCore.SIGNAL( "clicked()" ),
self.rafraichirPlugin )
QtCore.QObject.connect( self.tableWidgetTelechargement,
QtCore.SIGNAL( "cellDoubleClicked(int,int)" ),
self.supprimerTelechargement )
QtCore.QObject.connect( self.pushButtonExtremiteMonter,
QtCore.SIGNAL( "clicked()" ),
lambda versLeHaut = True, extremite = True : self.tableWidgetTelechargement.deplacerLigne( versLeHaut, extremite ) )
QtCore.QObject.connect( self.pushButtonMonter,
QtCore.SIGNAL( "clicked()" ),
lambda versLeHaut = True, extremite = False : self.tableWidgetTelechargement.deplacerLigne( versLeHaut, extremite ) )
QtCore.QObject.connect( self.pushButtonDescendre,
QtCore.SIGNAL( "clicked()" ),
lambda versLeHaut = False, extremite = False : self.tableWidgetTelechargement.deplacerLigne( versLeHaut, extremite ) )
QtCore.QObject.connect( self.pushButtonExtremiteDescendre,
QtCore.SIGNAL( "clicked()" ),
lambda versLeHaut = False, extremite = True : self.tableWidgetTelechargement.deplacerLigne( versLeHaut, extremite ) )
QtCore.QObject.connect( self.pushButtonToutSupprimer,
QtCore.SIGNAL( "clicked()" ),
self.supprimerTousLesTelechargements )
QtCore.QObject.connect( self.pushButtonNettoyer,
QtCore.SIGNAL( "clicked()" ),
self.nettoyer )
QtCore.QObject.connect( self.pushButtonLancer,
QtCore.SIGNAL( "clicked()" ),
self.lancerTelechargement )
QtCore.QObject.connect( self.pushButtonStop,
QtCore.SIGNAL( "clicked()" ),
self.stopperTelechargement )
QtCore.QObject.connect( self.actionQuitter,
QtCore.SIGNAL( "triggered()" ),
self.close )
################################################
# Instanciations + initialisation de variables #
################################################
# Fenetre About
self.aProposDialog = None
# Fenetre des preferences du logiciel
self.preferencesDialog = None
# Fenetre de mise a jour des plugins
self.updateManagerDialog = None
# Nom plugin courant
self.nomPluginCourant = ""
# Liste des fichiers
self.listeFichiers = []
# Liste des fichiers a telecharger
self.listeFichiersATelecharger = []
# Cache des images descriptive
# Clef : urlImage Valeur : image (binaire)
self.cacheImage = {}
# On intancie le lanceur de signaux
self.signaux = Signaux()
# On instancie le gestionnaire de preferences
self.preferences = Preferences()
# On instancie le gestionnaire de preferences des plugins
self.preferencesPluginDialog = PreferencePluginDialog( self )
# On instancie le gestionnaire de download
self.downloader = Downloader( self.signaux )
# On recupere l'instance de API
self.api = API.getInstance()
# On instancie le gestionnaire d'historique
self.historique = Historique()
# On instancie la fenetre d'attente
self.fenetreAttenteProgressDialog = FenetreAttenteProgressDialog( self )
# On instancie le gest
#
# Fenetre de confirmation pour quitter le logiciel
#
self.quitterMessageBox = QtGui.QMessageBox( self )
self.quitterMessageBox.setWindowTitle( "Fermeture de TVDownloader" )
self.quitterMessageBox.setText( u"Voulez-vous réellement quitter TVDownloader ?" )
self.quitterMessageBox.setInformativeText( u"Votre liste de téléchargement sera perdue" )
self.quitterMessageBox.addButton( "Oui", QtGui.QMessageBox.AcceptRole )
self.quitterMessageBox.addButton( "Non", QtGui.QMessageBox.RejectRole )
############################################################
# On connecte les signaux des instances precedements crees #
############################################################
QtCore.QObject.connect( self.pushButtonOuvrirDossierTelechargement,
QtCore.SIGNAL( "clicked()" ),
self.ouvrirRepertoireTelechargement )
QtCore.QObject.connect( self.comboBoxSite,
QtCore.SIGNAL( "activated(QString)" ),
self.listerChaines )
QtCore.QObject.connect( self.comboBoxChaine,
QtCore.SIGNAL( "activated(QString)" ),
self.listerEmissions )
QtCore.QObject.connect( self.comboBoxEmission,
QtCore.SIGNAL( "activated(QString)" ),
self.listerFichiers )
QtCore.QObject.connect( self.pushButtonPreferencesPlugin,
QtCore.SIGNAL( "clicked()" ),
self.ouvrirPreferencesPlugin )
QtCore.QObject.connect( self.actionPreferences,
QtCore.SIGNAL( "triggered()" ),
self.ouvrirPreferencesLogiciel )
QtCore.QObject.connect( self.actionMAJ,
QtCore.SIGNAL( "triggered()" ),
self.ouvrirFenetreMiseAJour )
QtCore.QObject.connect( self.actionAPropos,
QtCore.SIGNAL( "triggered()" ),
self.ouvrirFenetreAPropos )
QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "debutActualisation(PyQt_PyObject)" ) , self.fenetreAttenteProgressDialog.ouvrirFenetreAttente )
QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "finActualisation()" ) , self.fenetreAttenteProgressDialog.fermerFenetreAttente )
QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "actualiserListesDeroulantes()" ) , self.actualiserListesDeroulantes )
QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "listeChaines(PyQt_PyObject)" ) , self.ajouterChaines )
QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "listeEmissions(PyQt_PyObject)" ) , self.ajouterEmissions )
QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "listeFichiers(PyQt_PyObject)" ) , self.ajouterFichiers )
QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "nouvelleImage(PyQt_PyObject)" ) , self.mettreEnPlaceImage )
QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "debutTelechargement(int)" ) , self.debutTelechargement )
QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "finTelechargement(int)" ) , self.finTelechargement )
QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "finDesTelechargements()" ) , self.activerDesactiverInterface )
QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "pourcentageFichier(int)" ) , self.progressBarTelechargementFichier.setValue )
#########
# Début #
#########
# La fenetre prend la dimension qu'elle avait a sa fermeture
taille = self.preferences.getPreference( "tailleFenetre" )
self.resize( taille[ 0 ], taille[ 1 ] )
# Si aucun plugin n'est active, on ouvre la fenetre des preferences
if( len( self.preferences.getPreference( "pluginsActifs" ) ) == 0 ):
self.ouvrirPreferencesLogiciel()
# On actualise tous les plugins
self.rafraichirTousLesPlugins()
def main(filenameList):
lock = _threading.Lock() # 获取线程锁，预防多个listener竞争lftp的使用
ftp = FTPFactory().GetFTP()
downloader = Downloader(ftp, lock, *filenameList)
downloader.Run()
YOUTUBE_VID_DIR = 'Youtube Videos' # Parent directory for all Yt videos !
current_user = os.getlogin()
if system()=='Windows':
DOWNLOADS = f'C:/Users/{current_user}/Downloads' # get the currently logged in user's download dir !
elif system()=='Linux':
DOWNLOADS = f'/home/{current_user}/Downloads'
else:
raise Exception("Not Implemented For Current Platform !")
FINAL_PATH = os.path.join(DOWNLOADS, YOUTUBE_VID_DIR) # joining the downloads and the new vids directory !
if not os.path.isdir(FINAL_PATH):
os.mkdir(FINAL_PATH)
print(f'New Download Directory Created At {FINAL_PATH}')
download = Downloader()
while True:
option = int(input('1)Video\n2)Playlist\n3)Exit\n>>'))
if option == 1:
link = input('Please Paste A Video Link : ')
download.download_single_video(FINAL_PATH, link)
elif option == 2:
playlist_link = input('Please Paste A Playlist Link :')
download.download_playlist(FINAL_PATH, playlist_link=playlist_link)
elif option == 3:
print('Exiting........!')
break
from sentinelsat import read_geojson, geojson_to_wkt
from Downloader import Downloader
import StringProcessor as process
import zipfile
import os
username = '******'
password = '******'
path = '/Users/DavidLei/PycharmProjects/untitled/testfile.geojson'
link: str = 'https://scihub.copernicus.eu/apihub/'
from_satelite: str = 'Sentinel-2'
package_info = list()
keys = []
downloader = Downloader(username, password, link)
wkt = geojson_to_wkt(read_geojson(path))
downloader.search_polygon(wkt,
'20190501',
'20190503',
str_platform_name=from_satelite,
percentage=(0, 100))
f = open('FileList.txt', 'w+')
for key, value in downloader.products.items():
if "S2B_MSIL1C" in str(value):
# print('downloading')
# print(key)
# print(value)
package_info.append(str(value))
keys.append(str(key))
print(value['title'])
downloader.download_one(
def download(self):
self.downloader = Downloader(self, self.url, self.path)
self.thread = threading.Thread(target = self.downloader.download)
self.thread.start()
def __init__(self, uuid):
# Movement_has_Category.delete().execute()
# Movement.delete().execute()
downloader = Downloader(uuid)
account_statements = downloader.download_account_data()
for statement in account_statements:
try:
movement = Movement.get(hash=statement['id'])
if movement.id:
continue
except:
date = datetime.strptime(statement['postDate'], '%Y-%m-%d')
if statement['__typename'] == 'BarcodePaymentEvent':
type = 'OUTCOME'
original_name = statement['title'] + " " + statement[
'detail']
elif statement['__typename'] == 'TransferOutEvent':
type = 'OUTCOME'
original_name = statement['title'] + " " + statement[
'destinationAccount']['name']
elif statement['__typename'] == 'TransferInEvent':
original_name = statement['title']
if statement['originAccount']:
original_name = original_name + " " + statement[
'originAccount']['name']
type = 'INCOME'
if statement['__typename'] == 'TransferOutReversalEvent':
value = float(statement['detail'].split(
'R$ ', 1)[1].replace('.', '').replace(',', '.'))
type = 'REVERSAL'
else:
value = statement['amount']
Movement.create(original_name=original_name,
date=date,
value=value,
status='SHOW',
type=type,
hash=statement['id'])
card_statements = downloader.download_credit_card_data()
for statement in card_statements:
try:
movement = Movement.get(hash=statement['id'])
if movement.id:
continue
except:
rest = statement['time'].split(
'T', 1)[0] # todo Haw ficou de corrigir para a data iso
date = datetime.strptime(rest, '%Y-%m-%d')
Movement.create(original_name=statement['description'],
date=date,
value=statement['amount'] / 100,
status='SHOW',
type='OUTCOME',
hash=statement['id'])
