def main():
# Parse arguments from file docstring
args = docopt.docopt(__doc__, version=__version__)
URLs = args['<url>']
cookie_file = args['--cookies']
proxy_url = args['--proxy']
username = args['--username']
password = args['--password']
quiet_mode = args['--quiet']
debug_mode = args['--debug']
use_download_archive = args['--use-download-archive']
get_comments = args['--get-comments']
ignore_existing_item = args['--ignore-existing-item']
if debug_mode:
# Display log messages.
root = logging.getLogger()
root.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'\033[92m[DEBUG]\033[0m %(asctime)s - %(name)s - %(levelname)s - '
'%(message)s')
ch.setFormatter(formatter)
root.addHandler(ch)
metadata = internetarchive.cli.argparser.get_args_dict(args['--metadata'])
tu = TubeUp(verbose=not quiet_mode,
output_template=args['--output'],
get_comments=get_comments)
try:
for identifier, meta in tu.archive_urls(URLs, metadata,
cookie_file, proxy_url,
username, password,
use_download_archive,
ignore_existing_item):
print('\n:: Upload Finished. Item information:')
print('Title: %s' % meta['title'])
print('Upload URL: https://archive.org/details/%s\n' % identifier)
except Exception:
print('\n\033[91m' # Start red color text
'An exception just occured, if you found this '
"exception isn't related with any of your connection problem, "
'please report this issue to '
'https://github.com/bibanon/tubeup/issues')
traceback.print_exc()
print('\033[0m') # End the red color text
sys.exit(1)
def test_get_resource_basenames(self):
tu = TubeUp(dir_path=os.path.join(current_path, 'test_tubeup_rootdir'))
copy_testfiles_to_tubeup_rootdir_test()
result = tu.get_resource_basenames(
['https://www.youtube.com/watch?v=KdsN9YhkDrY'])
expected_result = {
os.path.join(current_path, 'test_tubeup_rootdir', 'downloads',
'KdsN9YhkDrY')
}
self.assertEqual(expected_result, result)
def test_get_resource_basenames(self):
tu = TubeUp(dir_path=os.path.join(current_path,
'test_tubeup_rootdir'),
# HACK: A hack to make the test in Travis successful,
# We need to investigate more about this, it doesn't
# make sense that the verbose flag affect the
# youtubedl extract_info() process.
# See:
# https://travis-ci.org/bibanon/tubeup/builds/299091640
verbose=True)
copy_testfiles_to_tubeup_rootdir_test()
result = tu.get_resource_basenames(
['https://www.youtube.com/watch?v=6iRV8liah8A'])
expected_result = {os.path.join(
current_path, 'test_tubeup_rootdir', 'downloads',
'Mountain_3_-_Video_Background_HD_1080p-6iRV8liah8A')}
self.assertEqual(expected_result, result)
def main():
# Parse arguments from file docstring
args = docopt.docopt(__doc__)
URLs = args['<url>']
proxy_url = args['--proxy']
username = args['--username']
password = args['--password']
quiet_mode = args['--quiet']
debug_mode = args['--debug']
use_download_archive = args['--use-download-archive']
if debug_mode:
# Display log messages.
root = logging.getLogger()
root.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'\033[92m[DEBUG]\033[0m %(asctime)s - %(name)s - %(levelname)s - '
'%(message)s')
ch.setFormatter(formatter)
root.addHandler(ch)
metadata = internetarchive.cli.argparser.get_args_dict(args['--metadata'])
tu = TubeUp(verbose=not quiet_mode)
try:
for identifier, meta in tu.archive_urls(URLs, metadata, proxy_url,
username, password,
use_download_archive):
print('\n:: Upload Finished. Item information:')
print('Title: %s' % meta['title'])
print('Upload URL: https://archive.org/details/%s\n' % identifier)
except Exception:
sys.exit(1)
def test_generate_ydl_options_with_verbose_mode(self):
tu = TubeUp(verbose=True)
result = tu.generate_ydl_options(
mocked_ydl_progress_hook, ydl_username='******',
ydl_password='******')
expected_result = {
'outtmpl': os.path.join(
self.tu.dir_path['downloads'], '%(title)s-%(id)s.%(ext)s'),
'restrictfilenames': True,
'verbose': True,
'quiet': False,
'progress_with_newline': True,
'forcetitle': True,
'continuedl': True,
'retries': 9001,
'fragment_retries': 9001,
'forcejson': True,
'writeinfojson': True,
'writedescription': True,
'writethumbnail': True,
'writeannotations': True,
'writesubtitles': True,
'allsubtitles': True,
'ignoreerrors': True,
'fixup': 'warn',
'nooverwrites': True,
'consoletitle': True,
'prefer_ffmpeg': True,
'call_home': False,
'logger': tu.logger,
'progress_hooks': [mocked_ydl_progress_hook],
'username': '******',
'password': '******'}
self.assertEqual(result, expected_result)
def test_tubeup_attribute_logger_when_verbose_mode(self):
tu = TubeUp(verbose=True)
self.assertIsInstance(tu.logger, logging.Logger)
def test_archive_urls(self):
tu = TubeUp(dir_path=os.path.join(current_path, 'test_tubeup_rootdir'),
ia_config_path=get_testfile_path('ia_config_for_test.ini'))
videobasename = os.path.join(current_path, 'test_tubeup_rootdir',
'downloads', 'KdsN9YhkDrY')
copy_testfiles_to_tubeup_rootdir_test()
with requests_mock.Mocker() as m:
# Mock the request to s3.us.archive.org, so it will responds
# a custom json. `internetarchive` library sends GET request to
# that url to check that we don't violate the upload limit.
m.get('https://s3.us.archive.org',
content=b'{"over_limit": 0}',
headers={'content-type': 'application/json'})
m.get('https://archive.org/metadata/youtube-KdsN9YhkDrY',
content=b'{}',
headers={'content-type': 'application/json'})
# Mock the PUT requests for internetarchive urls that defined
# in mock_upload_response_by_videobasename(), so this test
# doesn't perform upload to the real archive.org server.
mock_upload_response_by_videobasename(m, 'youtube-KdsN9YhkDrY',
videobasename)
result = list(
tu.archive_urls(
['https://www.youtube.com/watch?v=KdsN9YhkDrY']))
expected_result = [('youtube-KdsN9YhkDrY', {
'mediatype':
'movies',
'creator':
'RelaxingWorld',
'collection':
'opensource_movies',
'title':
'Epic Ramadan - Video Background HD1080p',
'description':
('If you enjoy my work, please consider Subscribe to my NEW '
'channel for more videos: <br>'
'https://www.youtube.com/MusicForRelaxation?sub_confirmation=1 <br>'
'▷ If you use this video, please put credits to my channel '
'in description: <br>'
'Source from RelaxingWorld: https://goo.gl/HsW75m<br>'
'<br>'
'▷ Also, do not forget to Subscribe to my channel. Thanks! '
'<br/><br/>Source: <a '
'href="https://www.youtube.com/watch?v=KdsN9YhkDrY">'
'https://www.youtube.com/watch?v=KdsN9YhkDrY</a><br/>Uploader: '
'<a '
'href="http://www.youtube.com/channel/UCWpsozCMdAnfI16rZHQ9XDg">'
'RelaxingWorld</a>'),
'date':
'2016-06-25',
'year':
'2016',
'subject':
('Youtube;video;Film & Animation;Video Background;'
'Footage;Animation;Cinema;Royalty Free Videos;'
'Stock Video Footage;Video Backdrops;'
'Amazing Nature;youtube;HD;1080p;Creative Commons Videos;'
'relaxing music;Ramadan;'),
'originalurl':
'https://www.youtube.com/watch?v=KdsN9YhkDrY',
'licenseurl':
'',
'scanner':
SCANNER
})]
self.assertEqual(expected_result, result)
def test_upload_ia(self):
tu = TubeUp(
dir_path=os.path.join(current_path, 'test_tubeup_rootdir'),
# Use custom ia configuration file so we don't need
# to login with username and password.
ia_config_path=get_testfile_path('ia_config_for_test.ini'))
videobasename = os.path.join(
current_path, 'test_tubeup_rootdir', 'downloads',
'Mountain_3_-_Video_Background_HD_1080p-6iRV8liah8A')
copy_testfiles_to_tubeup_rootdir_test()
with requests_mock.Mocker() as m:
# Mock the request to s3.us.archive.org, so it will responds
# a custom json. `internetarchive` library sends GET request to
# that url to check that we don't violate the upload limit.
m.get('https://s3.us.archive.org',
content=b'{"over_limit": 0}',
headers={'content-type': 'application/json'})
m.get('https://archive.org/metadata/youtube-6iRV8liah8A',
content=b'{}',
headers={'content-type': 'application/json'})
# Mock the PUT requests for internetarchive urls that defined
# in mock_upload_response_by_videobasename(), so this test
# doesn't perform upload to the real archive.org server.
mock_upload_response_by_videobasename(m, 'youtube-6iRV8liah8A',
videobasename)
result = tu.upload_ia(videobasename)
expected_result = ('youtube-6iRV8liah8A', {
'mediatype':
'movies',
'creator':
'Video Background',
'collection':
'opensource_movies',
'title':
'Mountain 3 - Video Background HD 1080p',
'description':
('Mountain 3 - Video Background HD 1080p<br>If '
'you use this video please put credits to my'
' channel in description:<br>https://www.youtub'
'e.com/channel/UCWpsozCMdAnfI16rZHQ9XDg<br>© D'
'on\'t forget to SUBSCRIBE, LIKE, COMMENT an'
'd RATE. Hope you all enjoy! <br/><br/>Sourc'
'e: <a href="https://www.youtube.com/watch?v'
'=6iRV8liah8A">https://www.youtube.com/watch'
'?v=6iRV8liah8A</a><br/>Uploader: <a href="h'
'ttp://www.youtube.com/channel/UCWpsozCMdAnf'
'I16rZHQ9XDg">Video Background</a>'),
'date':
'2015-01-05',
'year':
'2015',
'subject': ('Youtube;video;Entertainment;Video Background;'
'Footage;Animation;Cinema;stock video footage;'
'Royalty free videos;Creative Commons videos;'
'free movies online;youtube;HD;1080p;Amazing '
'Nature;Mountain;'),
'originalurl':
'https://www.youtube.com/watch?v=6iRV8liah8A',
'licenseurl':
'https://creativecommons.org/licenses/by/3.0/',
'scanner':
SCANNER
})
self.assertEqual(expected_result, result)
def setUp(self):
self.tu = TubeUp()
self.maxDiff = 999999999
def test_archive_urls(self):
tu = TubeUp(
dir_path=os.path.join(current_path, 'test_tubeup_rootdir'),
ia_config_path=get_testfile_path('ia_config_for_test.ini'),
# HACK: A hack to make the test in Travis successful,
# We need to investigate more about this, it doesn't
# make sense that the verbose flag affect the
# youtubedl extract_info() process.
# See:
# https://travis-ci.org/bibanon/tubeup/builds/299091640
verbose=True)
videobasename = os.path.join(
current_path, 'test_tubeup_rootdir', 'downloads',
'Mountain_3_-_Video_Background_HD_1080p-6iRV8liah8A')
copy_testfiles_to_tubeup_rootdir_test()
with requests_mock.Mocker() as m:
# Mock the request to s3.us.archive.org, so it will responds
# a custom json. `internetarchive` library sends GET request to
# that url to check that we don't violate the upload limit.
m.get('https://s3.us.archive.org',
content=b'{"over_limit": 0}',
headers={'content-type': 'application/json'})
m.get('https://archive.org/metadata/youtube-6iRV8liah8A',
content=b'{}',
headers={'content-type': 'application/json'})
# Mock the PUT requests for internetarchive urls that defined
# in mock_upload_response_by_videobasename(), so this test
# doesn't perform upload to the real archive.org server.
mock_upload_response_by_videobasename(m, 'youtube-6iRV8liah8A',
videobasename)
result = list(
tu.archive_urls(
['https://www.youtube.com/watch?v=6iRV8liah8A']))
expected_result = [('youtube-6iRV8liah8A', {
'mediatype':
'movies',
'creator':
'Video Background',
'collection':
'opensource_movies',
'title':
'Mountain 3 - Video Background HD 1080p',
'description': ('Mountain 3 - Video Background HD 1080p\nIf '
'you use this video please put credits to my'
' channel in description:\nhttps://www.youtub'
'e.com/channel/UCWpsozCMdAnfI16rZHQ9XDg\n© D'
'on\'t forget to SUBSCRIBE, LIKE, COMMENT an'
'd RATE. Hope you all enjoy! <br/><br/>Sourc'
'e: <a href="https://www.youtube.com/watch?v'
'=6iRV8liah8A">https://www.youtube.com/watch'
'?v=6iRV8liah8A</a><br/>Uploader: <a href="h'
'ttp://www.youtube.com/channel/UCWpsozCMdAnf'
'I16rZHQ9XDg">Video Background</a>'),
'date':
'2015-01-05',
'year':
'2015',
'subject': ('Youtube;video;Entertainment;Video Background;'
'Footage;Animation;Cinema;stock video footage;'
'Royalty free videos;Creative Commons videos;'
'free movies online;youtube;HD;1080p;Amazing '
'Nature;Mountain;'),
'originalurl':
'https://www.youtube.com/watch?v=6iRV8liah8A',
'licenseurl':
'https://creativecommons.org/licenses/by/3.0/',
'scanner':
SCANNER
})]
self.assertEqual(expected_result, result)