def download(): if request.method == "POST": dir_name = os.path.join(CONF.get('SERVER', 'files_dir'), hex(int(time.time()))[2:]) if request.form.get('img_height') and request.form.get('img_width'): img_size = (min(640, int(request.form.get('img_height'))), min(640, int(request.form.get('img_width')))) max_count = min(5000, int(request.form.get('max_count', 100))) else: img_size = None max_count = min(1000, int(request.form.get('max_count', 100))) tag = request.form.get('tag', 100) photos = Photos.query.filter(Photos.tag == tag)\ .order_by(Photos.update_time.desc())\ .limit(max_count).all() urls = [ph.url for ph in photos] downloader.main(urls, dir_name, img_size) zip_file = downloader.zipdir(dir_name) return_data = io.BytesIO() with open(zip_file, 'rb') as fo: return_data.write(fo.read()) return_data.seek(0) os.remove(zip_file) wrapped_data = FileWrapper(return_data) return Response(wrapped_data, mimetype="application/zip", direct_passthrough=True) else: tags = db.session.query( func.count(Photos.tag).label('count'), Photos.tag).group_by(Photos.tag).all() return render_template('download.html', tag_list=tags)
def test_dowload_files_from_internet(tmpdir, caplog, file, links): """Test for downloader.main Assertions: all links exists in log files stored or exception exists in log """ output_dir = str(tmpdir.mkdir("download")) arguments = ["downloader", str(file), output_dir] downloader.main(arguments) caplog.setLevel(logging.INFO) out = caplog.records() info = "\n".join([o.message for o in out if o.levelname == "INFO"]) errors = [o.message for o in out if o.levelname == "ERROR"] error = "\n".join(errors) listdir = os.listdir(output_dir) # all links exists in log assert all(link in info for link in links) or all(link in error for link in links) for link in links: name = link.split("/")[-1] # files stored or exception exists in log assert name in listdir or link in error
def comm_create_database(self, bot: Bot, update: Update): bot.send_message(update.message.chat_id, "Creating DB") try: db = MySQLdb.connect(host="localhost", user=database_user, passwd=database_passwd, charset='utf8') except MySQLdb.Error as err: bot.send_message(update.message.chat_id, "{0}".format(err)) else: cursor = db.cursor() try: downloader.main([ '-i', 'local', '-o', 'database', '--if', dirname(abspath(__file__)) + '/../../CreateDatabase/pokemon_data.json', '--db', 'pokemon' ]) cursor.execute("USE pokemon") except MySQLdb.Error as err: return err.args[1] try: # fill database create_tables(cursor) fill_tables(cursor) db.commit() except MySQLdb.Error as err: bot.send_message(update.message.chat_id, "{0}".format(err)) except Timeout as err: bot.send_message(update.message.chat_id, "{0}".format(err)) else: bot.send_message(update.message.chat_id, "Created DB succesfull new")
def acquire_data_async(): try: main() except Exception: print("Failed to acquire data") print '-' * 60 traceback.print_exc(file=sys.stdout) print '-' * 60
def main(url): if urlparse(url).netloc == "www.newgrounds.com": ngpage = requests.get(url).content soup = BS(ngpage, "html.parser") nameLabel = soup.find_all("div", { "class": "pod-head", "id": "embed_header" })[0].find("h2") name = nameLabel.string.replace(":", ",") print(name) downloader.main(url=url, gameName=name) else: print("Its recomended to use newgrounds") downloader.main(url=url, gameName=input("Game name: "))
def call_main (): print(' Comment Thread 생성중 \n') print(' **************************************************************') print(' **************************************************************') print(' **************************************************************') print(' **************** 생성 완료 정보를 입력하세요. **************** ') print(' **************************************************************') print(' **************************************************************') print(' **************************************************************') a = downloader.main() return a
def createdb(): global cursor, isconnected if not isconnected: return "You need to connect with a database first!" else: try: downloader.main([ '-i', 'local', '-o', 'database', '--if', 'pokemon_data.json', '--db', 'pokemon' ]) cursor.execute("USE pokemon") except MySQLdb.Error as err: return err.args[1] try: #fill database create_tables(cursor) fill_tables(cursor) db.commit() except MySQLdb.Error as err: return err.args[1] except Timeout as err: return err.args[0] else: return "Created DB succesfull new"
fp = open("epis.txt", "r") for i in fp: ## std.addstr(count+2,1,i) flag = 1 time.sleep(1) for j in range(len(find)): if (find[j] in i): pass else: flag = 0 if (flag != 0): url = i break std.addstr(count + 2, 1, url) downloader.main(url.strip('\n')) ## std.addstr(11,1,str(link)) std.refresh() std.clear() std.refresh() std.addstr( 1, 1, "do you want to add this to you favorite ? 'y' for Yes and 'n' for No " ) std.addstr( 2, 1, "* adding this to favorite will automatically download all the latest episode on the arrival on our server" ) std.refresh() answer = std.getstr( 1,
feed = feedparser.parse(rss) # prepare database with open('database.pwp', 'r') as f: donelist = f.read().split('$') # check update update_pool = [] for i in feed.entries: i.link = bv2av(i.link.split('/')[-1]) if i.link not in donelist: print(i.link) update_pool.append(i) # update for i in update_pool: try: # download video ftitle = downloader.main(i.link + '?p=1') # convert into audio mp3path = 'output/' + re.sub(r'[\/\\:*?"<>|]', '', i.title) + '.mp3' { ffmpeg.input('bilibili_video/' + ftitle + '/' + ftitle + '.flv').output(mp3path, ab='1080k').run() } # insert cover try: cover.add_cover(cover.get_cover(i.link), mp3path) except: with open('./log.txt', 'a') as f: f.writelines('error when inserting cover to: ' + ftitle + i.link) except:
# -*- coding:utf-8 -*- import downloader import html_parser dst = 'subreddit_list.tsv' downloader.main() html_parser.main(dst)
def main(): parser = argparse.ArgumentParser(description="subtitle Downloader") parser.add_argument("-v", "--version", help="display version", action="store_true") parser.add_argument("-t", "--title", default=None, type=str.lower, help="Enter the movie title") parser.add_argument("-y", "--year", default=' ', type=str, help="Enter the released year") parser.add_argument( "-lang", "--language", type=str.lower, default="english", choices=[ "english", "arabic", "albanian", "arabic", "bengali", "turkish", "spanish", "swedish", "thai", "slovenian", "russian", "portuguese", "malay", "farsi/persian" ], help="Enter any language from choices, default value ='english'") parser.add_argument( "-l", "--limit", type=str.lower, default="1", help="No of files you want to download , default value = 1") parser.add_argument( "-r", "--resolution", type=str.lower, default="all", help= "Filter subtitles on resolution\nSelect any resolution from choices", choices=['360p', '480p', '720p', '1080p', '1440p']) parser.add_argument( "-p", "--path", type=str.lower, default="PWD", help= 'Select path for save subtitles, default value is current path (PWD)') args = parser.parse_args() if args.version: print("subtitle-Downloader 0.1") else: #info = {'Title' : args.title ,'Year':args.year,'Language':args.language,'limit':args.limit } #print("{}".format(info)) print("Connecting......") print(f"Searching for {args.title + ' ' + args.year}") print("Please wait...") downloader.main(title=args.title, year=args.year, language=args.language, limit=args.limit, resolution=args.resolution, path_to_save=args.path)
def yt_get_comments(self): driver = webdriver.Firefox(options=self.options) df = pd.read_excel('User_Inputs.xlsx') df = df['YT_Artist_Searches'] df.dropna(inplace=True) yt_url = [] yt_id = [] yt_title = [] yt_comments = [] yt_datetime = [] for artist in df: driver.get("https://www.youtube.com/results?search_query=" + artist + "&sp=CAI%253D") driver.execute_script('window.scrollTo(1, 150000);') time.sleep(1.5) soup = bs.BeautifulSoup(driver.page_source, 'lxml') for item in soup.find_all( 'a', class_='yt-simple-endpoint style-scope ytd-video-renderer' ): if artist.lower() in item['title'].lower(): try: dl.main([ '-y' + item['href'].replace('/watch?v=', ''), '-oyt_comments.json' ]) for line in open('yt_comments.json', 'r', encoding='utf-8'): yt_comments.append( json.loads(line.replace(';', ""))['text']) yt_datetime.append( json.loads(line.replace(';', ""))['time']) yt_url.append('https://www.youtube.com/' + item['href']) yt_title.append(item['title']) except: pass df_yt = pd.DataFrame( zip(yt_title, yt_url, yt_comments, yt_datetime), columns=['Mix', 'MixURL', 'Comments', 'Comments Datetime']) def transform_yt_datetime(x): if any(i in ['年', 'an', 'ans', 'year', 'years'] for i in x): return datetime.date.today() - relativedelta( years=int(re.search(r'\d+', x).group())) elif any(i in ['月', 'mois', 'month', 'months'] for i in x): return datetime.date.today() - relativedelta( months=int(re.search(r'\d+', x).group())) elif any(i in ['週', 'semaine', 'semaines', 'week', 'weeks'] for i in x): return datetime.date.today() - relativedelta( weeks=int(re.search(r'\d+', x).group())) elif any(i in ['日', 'jour', 'jours', 'day', 'days'] for i in x): return datetime.date.today() - relativedelta( days=int(re.search(r'\d+', x).group())) elif any(i in ['時', 'heure', 'heures', 'hour', 'hours'] for i in x): return datetime.date.today() - relativedelta( hours=int(re.search(r'\d+', x).group())) elif any(i in ['分', 'minute', 'minutes'] for i in x): return datetime.date.today() - relativedelta( minutes=int(re.search(r'\d+', x).group())) elif any(i in ['秒', 'seconde', 'secondes', 'heure', 'heures'] for i in x): return datetime.date.today() - relativedelta( seconds=int(re.search(r'\d+', x).group())) else: x df_yt['Comments Datetime'] = df_yt['Comments Datetime'].apply( lambda x: transform_yt_datetime(x)) driver.close() driver.quit() return df_yt
#!/usr/bin/env python3 import downloader downloader.main()
def test_main(self): actual = downloader.main()