def download(item): try: url = item.url #copy an instance runtime_ydl_option = copy.copy(ydl_option) runtime_ydl_option['outtmpl'] = Configuration.getDownloadPath() + '/' + str(item.id) + '-%(id)s.%(ext)s' ydl = youtube_dl.YoutubeDL(runtime_ydl_option) #process = subprocess.check_output(['youtube-dl', "-o downloads/video/%(uploader)s/%(title)s-%(id)s.%(ext)s", url], stderr=subprocess.STDOUT,shell=True) logging.info('[%s] Start downloading [%s]. ', threading.currentThread().name, url) #info = ydl.extract_info(url=url, download=False) info = ydl.extract_info(url=url) # Reduce the json info by removing unused format information #info['formats'] = None logging.info('[%s] after extract info', threading.currentThread().name) jsonFileName = Configuration.getDownloadPath() + '/{0}-{1}.json'.format(str(item.id), info['id']) with open(jsonFileName, 'w') as f: json.dump(info, f, indent=1) logging.info("[%s] ******Downloading finished: [%s] - [%s]", threading.currentThread().name, info['title'], info['format']) time.sleep(3) # Handle Exception except Exception as e: logging.warn("[%s] Exception: %s.", threading.currentThread().name, e.message) else: DownloadInfoService.markAsDownloaded(url) logging.info('[%s] Downloaded successfully, mark [%s] to downloaded in DB.', threading.currentThread().name, url)
def main(): # Initialize Configuration inputSearchText = raw_input( '1. Please input the key words you want to search.[default: ' + Configuration.runtime_search_text + ']\n') if inputSearchText.strip() != "": Configuration.runtime_search_text = inputSearchText inputMaxPageNumber = raw_input('2. Please input the maximum page number you want to search.[default: ' + str( Configuration.runtime_search_max_page_number) + ']\n') if inputMaxPageNumber.strip() != "": Configuration.runtime_search_max_page_number = int(inputMaxPageNumber) downloadPath = raw_input( '3. Please input the output folder of these videos.[default: ' + Configuration.runtime_download_root_path + ']\n') if downloadPath.strip() != "": Configuration.runtime_download_root_path = downloadPath inputDownloadThread = raw_input('4. Please input download threads number.[default: ' + str( Configuration.runtime_num_of_download_worker) + ']\n') if inputDownloadThread.strip() != "": Configuration.runtime_num_of_download_worker = int(inputDownloadThread) disableExtractor = raw_input('5. Disable video link extractor(y/n).[default: ' + ( 'y' if Configuration.runtime_disable_extractor else 'n') + ']\n') if disableExtractor.strip() == "y" or disableExtractor.strip() == "Y": Configuration.runtime_disable_extractor = True if not os.path.exists(Configuration.getDownloadPath()): mkdirs(Configuration.getDownloadPath()) DownloadTaskProducer.start() Downloader.start() if not Configuration.runtime_disable_extractor: VideoLinkExtractor.start() else: logging.info('Video link extractor is disabled.') logging.info('App started ...') while True: pass