def _test_file(name, plugin):
print '-'*100, "testing plugin {} with file {}".format(plugin, name)
path = os.path.join(rootpath, name)
with transaction:
package = core.Package(name=os.path.splitext(name)[0], complete_dir=rootpath)
file = core.File(package=package, name=name, url='file://{}'.format(path), host=FakeHosterPlugin(),
pmatch='asdf', state='download_complete')
event.wait_for_events(['fileplugin:done'], 5)
assert file.last_error is None
assert file.working is False
assert file.state == '{}_complete'.format(plugin)
assert plugin in file.completed_plugins
p = os.path.join(file.get_extract_path(), "1mb.bin")
assert os.path.exists(p)
try:
debugtools.assert_file_checksum('md5', p, '934a5866d0a738c32f040559eccbf567')
finally:
os.unlink(p)
file.delete()
def _test_file(name, plugin):
print '-' * 100, "testing plugin {} with file {}".format(plugin, name)
path = os.path.join(rootpath, name)
with transaction:
package = core.Package(name=os.path.splitext(name)[0],
complete_dir=rootpath)
file = core.File(package=package,
name=name,
url='file://{}'.format(path),
host=FakeHosterPlugin(),
pmatch='asdf',
state='download_complete')
event.wait_for_events(['fileplugin:done'], 5)
assert file.last_error is None
assert file.working is False
assert file.state == '{}_complete'.format(plugin)
assert plugin in file.completed_plugins
p = os.path.join(file.get_extract_path(), "1mb.bin")
assert os.path.exists(p)
try:
debugtools.assert_file_checksum('md5', p,
'934a5866d0a738c32f040559eccbf567')
finally:
os.unlink(p)
file.delete()
def finish_download(self, type, speed=0):
print "!!!", '{} #finish'.format(type)
interface.call('config', 'set', key='download.rate_limit', value=speed)
self.file_enabled(True)
event.wait_for_events(['file:download_complete'], 10)
self.assert_download_complete()
debugtools.assert_file_checksum('md5', self.file.get_complete_file(), httpserver.md5_10mb)
self.del_link()
def test_mirrors(self):
interface.call('config',
'set',
key='download.max_simultan_downloads',
value=4)
self.testurl = httpserver.url + '/10mb.bin'
interface.call('core',
'add_links',
links=[
httpserver.url + '/anyname/mirror1/flow-1.bin',
httpserver.url + '/anyname/mirror2/flow-1.bin',
httpserver.url + '/anyname/mirror3/flow-1.bin',
httpserver.url + '/anyname/mirror4/flow-1.bin'
])
self.wait_check()
assert [f.name for f in core.files()
] == ['flow-1.bin', 'flow-1.bin', 'flow-1.bin', 'flow-1.bin']
interface.call('config', 'set', key='download.rate_limit', value=32768)
interface.call('core', 'accept_collected')
event.wait_for_events(['download:spawn_tasks'], 5)
gevent.sleep(0.2)
assert len(core._packages) == 1
assert len(core._packages[0].files) == 4
assert sum(1 for f in core.files() if f.working) == 1
interface.call('config', 'set', key='download.rate_limit', value=0)
event.wait_for_events(['package:download_complete'], 5)
file = [f for f in core.files() if f.state == 'download_complete'][0]
assert file.package.state == 'download_complete'
for f in core.files():
if f != file:
assert f.state == 'download'
assert f.enabled is False
assert f.last_error.startswith('downloaded via')
interface.call('core', 'printr')
def _test_rar_multipart_start_stop():
print '-' * 100, 'test_rar_multipart_start_stop'
files = list()
add_files(files, 'download_complete', 'foo', 'foo')
gevent.spawn_later(1.5, interface.call, 'core', 'stop')
event.wait_for_events(['download:stopped'], 5)
interface.call('core', 'printr')
gevent.sleep(1)
interface.call('core', 'start')
event.wait_for_events(
['rarextract:part_complete', 'rarextract:waiting_for_part'], 5)
interface.call('core', 'printr')
with transaction:
files[1].state = 'download_complete'
event.wait_for_events(
['rarextract:part_complete', 'rarextract:waiting_for_part'], 5)
interface.call('core', 'printr')
with transaction:
files[2].state = 'download_complete'
event.wait_for_events(
['rarextract:part_complete', 'rarextract:waiting_for_part'], 5)
interface.call('core', 'printr')
gevent.sleep(0.1)
for f in files:
f.join()
assert f.last_error is None
assert f.working is False
assert f.state == 'rarextract_complete', "expected complete, but is {}".format(
f.state)
p = os.path.join(files[0].get_extract_path(), "1mb.bin")
assert os.path.exists(p)
try:
debugtools.assert_file_checksum('md5', p,
'934a5866d0a738c32f040559eccbf567')
finally:
os.unlink(p)
for f in files:
f.delete()
def test_mirrors(self):
interface.call('config', 'set', key='download.max_simultan_downloads', value=4)
self.testurl = httpserver.url+'/10mb.bin'
interface.call('core', 'add_links', links=[
httpserver.url+'/anyname/mirror1/flow-1.bin',
httpserver.url+'/anyname/mirror2/flow-1.bin',
httpserver.url+'/anyname/mirror3/flow-1.bin',
httpserver.url+'/anyname/mirror4/flow-1.bin'])
self.wait_check()
assert [f.name for f in core.files()] == ['flow-1.bin', 'flow-1.bin', 'flow-1.bin', 'flow-1.bin']
interface.call('config', 'set', key='download.rate_limit', value=32768)
interface.call('core', 'accept_collected')
event.wait_for_events(['download:spawn_tasks'], 5)
gevent.sleep(0.2)
assert len(core._packages) == 1
assert len(core._packages[0].files) == 4
assert sum(1 for f in core.files() if f.working) == 1
interface.call('config', 'set', key='download.rate_limit', value=0)
event.wait_for_events(['package:download_complete'], 5)
file = [f for f in core.files() if f.state == 'download_complete'][0]
assert file.package.state == 'download_complete'
for f in core.files():
if f != file:
assert f.state == 'download'
assert f.enabled is False
assert f.last_error.startswith('downloaded via')
interface.call('core', 'printr')
def _test_rar_multipart_start_stop():
print '-'*100, 'test_rar_multipart_start_stop'
files = list()
add_files(files, 'download_complete', 'foo', 'foo')
gevent.spawn_later(1.5, interface.call, 'core', 'stop')
event.wait_for_events(['download:stopped'], 5)
interface.call('core', 'printr')
gevent.sleep(1)
interface.call('core', 'start')
event.wait_for_events(['rarextract:part_complete', 'rarextract:waiting_for_part'], 5)
interface.call('core', 'printr')
with transaction:
files[1].state = 'download_complete'
event.wait_for_events(['rarextract:part_complete', 'rarextract:waiting_for_part'], 5)
interface.call('core', 'printr')
with transaction:
files[2].state = 'download_complete'
event.wait_for_events(['rarextract:part_complete', 'rarextract:waiting_for_part'], 5)
interface.call('core', 'printr')
gevent.sleep(0.1)
for f in files:
f.join()
assert f.last_error is None
assert f.working is False
assert f.state == 'rarextract_complete', "expected complete, but is {}".format(f.state)
p = os.path.join(files[0].get_extract_path(), "1mb.bin")
assert os.path.exists(p)
try:
debugtools.assert_file_checksum('md5', p, '934a5866d0a738c32f040559eccbf567')
finally:
os.unlink(p)
for f in files:
f.delete()
def test_rar_multipart():
print "-" * 100, 'test_rar_multipart'
files = list()
add_files(files, 'download_complete', 'download', 'download')
event.wait_for_events(
['rarextract:part_complete', 'rarextract:waiting_for_part'], 5)
interface.call('core', 'printr')
with transaction:
if not "rarextract" in files[1].completed_plugins:
print files[1].state
files[1].state = 'download_complete'
files[1].working = False
event.wait_for_events(
['rarextract:part_complete', 'rarextract:waiting_for_part'], 5)
interface.call('core', 'printr')
with transaction:
if not "rarextract" in files[2].completed_plugins:
print files[2].state
files[2].state = 'download_complete'
files[2].working = False
event.wait_for_events(
['rarextract:part_complete', 'rarextract:waiting_for_part'], 5)
interface.call('core', 'printr')
gevent.sleep(1)
for f in files:
f.join()
assert f.last_error is None
assert not f.working, "{} is working".format(f.name)
assert f.state == 'rarextract_complete', "expected complete, but is {}".format(
f.state)
p = os.path.join(files[0].get_extract_path(), "1mb.bin")
assert os.path.exists(p)
try:
debugtools.assert_file_checksum('md5', p,
'934a5866d0a738c32f040559eccbf567')
finally:
os.unlink(p)
for f in files:
f.delete()
def test_downloads(self):
interface.call('config',
'set',
key='download.max_simultan_downloads',
value=2)
self.testurl = httpserver.url + '/10mb.bin'
interface.call('core',
'add_links',
links=[
httpserver.url + '/anyname/flow-1.bin',
httpserver.url + '/anyname/flow-2.bin',
httpserver.url + '/anyname/flow-3.bin',
httpserver.url + '/anyname/flow-4.bin'
])
self.wait_check()
assert [f.name for f in core.files()
] == ['flow-1.bin', 'flow-2.bin', 'flow-3.bin', 'flow-4.bin']
interface.call('config', 'set', key='download.rate_limit', value=32768)
interface.call('core', 'accept_collected')
event.wait_for_events(['download:spawn_tasks'], 5)
gevent.sleep(0.2)
assert len(core._packages) == 1
assert len(core._packages[0].files) == 4
assert [f.working for f in core.files()] == [True, True, False, False]
interface.call('config', 'set', key='download.rate_limit', value=0)
event.wait_for_events(['file:download_complete'], 15)
interface.call('config', 'set', key='download.rate_limit', value=32768)
# these tests fail randomly so it is disabled
#assert sum(1 for f in core.files() if f.working) == 1
#assert sum(1 for f in core.files() if f.state == 'download_complete') == 1
assert sum(1 for f in core.files() if f.state == 'download') == 3
interface.call('config', 'set', key='download.rate_limit', value=0)
event.wait_for_events(['package:download_complete'], 15)
assert sum(1 for f in core.files() if f.working) == 0
assert sum(1 for f in core.files()
if f.state == 'download_complete') == 4
assert sum(1 for f in core.files() if f.last_error) == 0
assert sum(1 for f in core.files() if not f.enabled) == 0
interface.call('core', 'printr')
def test_rar_multipart():
print "-"*100, 'test_rar_multipart'
files = list()
add_files(files, 'download_complete', 'download', 'download')
event.wait_for_events(['rarextract:part_complete', 'rarextract:waiting_for_part'], 5)
interface.call('core', 'printr')
with transaction:
if not "rarextract" in files[1].completed_plugins:
print files[1].state
files[1].state = 'download_complete'
files[1].working = False
event.wait_for_events(['rarextract:part_complete', 'rarextract:waiting_for_part'], 5)
interface.call('core', 'printr')
with transaction:
if not "rarextract" in files[2].completed_plugins:
print files[2].state
files[2].state = 'download_complete'
files[2].working = False
event.wait_for_events(['rarextract:part_complete', 'rarextract:waiting_for_part'], 5)
interface.call('core', 'printr')
gevent.sleep(1)
for f in files:
f.join()
assert f.last_error is None
assert not f.working, "{} is working".format(f.name)
assert f.state == 'rarextract_complete', "expected complete, but is {}".format(f.state)
p = os.path.join(files[0].get_extract_path(), "1mb.bin")
assert os.path.exists(p)
try:
debugtools.assert_file_checksum('md5', p, '934a5866d0a738c32f040559eccbf567')
finally:
os.unlink(p)
for f in files:
f.delete()
def test_downloads(self):
interface.call('config', 'set', key='download.max_simultan_downloads', value=2)
self.testurl = httpserver.url+'/10mb.bin'
interface.call('core', 'add_links', links=[
httpserver.url+'/anyname/flow-1.bin',
httpserver.url+'/anyname/flow-2.bin',
httpserver.url+'/anyname/flow-3.bin',
httpserver.url+'/anyname/flow-4.bin'])
self.wait_check()
assert [f.name for f in core.files()] == ['flow-1.bin', 'flow-2.bin', 'flow-3.bin', 'flow-4.bin']
interface.call('config', 'set', key='download.rate_limit', value=32768)
interface.call('core', 'accept_collected')
event.wait_for_events(['download:spawn_tasks'], 5)
gevent.sleep(0.2)
assert len(core._packages) == 1
assert len(core._packages[0].files) == 4
assert [f.working for f in core.files()] == [True, True, False, False]
interface.call('config', 'set', key='download.rate_limit', value=0)
event.wait_for_events(['file:download_complete'], 15)
interface.call('config', 'set', key='download.rate_limit', value=32768)
# these tests fail randomly so they are disabled (race conditions?!)
#assert sum(1 for f in core.files() if f.working) == 1
#assert sum(1 for f in core.files() if f.state == 'download_complete') == 1
#assert sum(1 for f in core.files() if f.state == 'download') == 3
interface.call('config', 'set', key='download.rate_limit', value=0)
event.wait_for_events(['package:download_complete'], 15)
assert sum(1 for f in core.files() if f.working) == 0
assert sum(1 for f in core.files() if f.state == 'download_complete') == 4
assert sum(1 for f in core.files() if f.last_error) == 0
assert sum(1 for f in core.files() if not f.enabled) == 0
interface.call('core', 'printr')
else:
raise ValueError(
'{}: {}.{} = "{}" -- needs value: "{}"'.format(
step, obj._table_name, key, obj_value, value))
######## test code
interface.call('config', 'set', key='download.rate_limit', value=0)
interface.call('config', 'set', key='download.overwrite', value='overwrite')
try:
if "account" in test:
interface.call('account', 'add', name=plugin, **test['account'])
args, kwargs = event.wait_for_events(
['account:initialized', 'account:initialize_error'],
test['timeout']['account'])
check_values(test, 'result_account', args[1])
if "urls" in test:
ids = interface.call('core', 'add_links', links=test['urls'])
if test['type'] == 'decrypter':
while True:
args, kwargs = event.wait_for_events(
['file:greenlet_stop', 'file:created', 'file:deleted'],
test['timeout']['check'])
if args[0] == 'file:created':
check_values(test, 'result_new', args[1])
elif args[1].id in ids:
check_values(test, 'result_check', args[1])
raise ValueError('{}: {}.{} = "{}" -- needs on of these values: "{}"'.format(step, obj._table_name, key, obj_value, '", "'.join(values)))
else:
if obj_value == value:
print '{}: {}.{} = "{}"'.format(step, obj._table_name, key, obj_value)
else:
raise ValueError('{}: {}.{} = "{}" -- needs value: "{}"'.format(step, obj._table_name, key, obj_value, value))
######## test code
interface.call('config', 'set', key='download.rate_limit', value=0)
interface.call('config', 'set', key='download.overwrite', value='overwrite')
try:
if "account" in test:
interface.call('account', 'add', name=plugin, **test['account'])
args, kwargs = event.wait_for_events(['account:initialized', 'account:initialize_error'], test['timeout']['account'])
check_values(test, 'result_account', args[1])
if "urls" in test:
ids = interface.call('core', 'add_links', links=test['urls'])
if test['type'] == 'decrypter':
while True:
args, kwargs = event.wait_for_events(['file:greenlet_stop', 'file:created', 'file:deleted'], test['timeout']['check'])
if args[0] == 'file:created':
check_values(test, 'result_new', args[1])
elif args[1].id in ids:
check_values(test, 'result_check', args[1])
break
elif test['type'] == 'hoster':
