def test_service():
global cnt
settings = baseSettings
settings["service"] = True
settings["sleep"] = 1
settings["onlyOnce"] = True
cnt = 0
mId = 3
cycles = 10
def callback(file, id, processingCrawler):
global cnt
cnt = cnt + 1
assert id == mId, "Argurments doesn't match the expected. Got {} instead of {}".format(
id, mId)
if cnt >= cycles:
processingCrawler.settings["service"] = False
mCrawler = crawler.createCrawler(settings, callback)
startTime = datetime.datetime.now()
mCrawler.process(mId, mCrawler)
endTime = datetime.datetime.now()
diffTime = endTime - startTime
def checkTime(seconds):
if seconds > (cycles - 1) * settings["sleep"] and seconds < (
cycles + 1) * settings["sleep"]:
return True
return False
assert checkTime(
diffTime.seconds), "Test took {}s, expceted time would be {}s".format(
diffTime.seconds, cycles * settings["sleep"])
assert cnt == cycles, "Wrong number of cycles. Got {} instead of {}".format(
cnt, cycles)
def test_save():
dummyName = "dummyMemo"
mCrawler = crawler.createCrawler(baseSettings)
mCrawler.memo.append(dummyName)
mCrawler.save()
with open(baseSettings["memo"], 'rb') as f:
data = json.load(f)
assert dummyName in data, "Didn't found {} in {}".format(
dummyName, baseSettings["memo"])
def test_load():
dummyName = "dummyLoad"
data = [dummyName]
with open(baseSettings['memo'], 'w') as f:
json.dump(data, f, indent=4)
mCrawler = crawler.createCrawler(baseSettings)
assert len(
mCrawler.memo) == 1, "Crawler memo contains not exactly one item"
assert mCrawler.memo[
0] == dummyName, "Crawlers memo contains {} instead of {}".format(
mCrawler.memo[0], dummyName)
def test_createCrawlerFactory():
settings = baseSettings
tests = [["local", crawler.localCrawler],
["google", crawler.googleCrawler], ["git", crawler.gitCrawler]]
for test in tests:
settings["type"] = test[0]
myCrawler = crawler.createCrawler(settings)
testCrawler = test[1](settings)
assert type(myCrawler) == type(
testCrawler
), "Wrong crawler type was created. Created crawler was: {}".format(
type(myCrawler))
def find_list(baseSettings, tests):
for i, test in enumerate(tests):
settings = baseSettings
for key, val in test[0].items():
settings[key] = val
myCrawler = crawler.createCrawler(settings)
myCrawler.memo = []
try:
results = myCrawler.getList()
assert len(
results) == test[1], "Found {} instead of {} files".format(
len(results), test[1])
if len(test[2]) > 0:
for name in results:
assert name in test[
2], "Unexpected file ({}) in Test {} appeared in found files. Expected {}".format(
name, i, test[2])
except ValueError as VE:
assert settings["onlyOnce"] == False, "Unexpected exeption raises"
def callback_singleReturn(baseSettings, tests):
global singleReturnCnt
settings = baseSettings
settings["onlyOnce"] = False
for test in tests:
for key, val in test[0].items():
settings[key] = val
singleReturnCnt = 0
def callback(file):
global singleReturnCnt
if len(test[2]) > 0:
assert file in test[2], "Couldn't find file ({}) in {}".format(
file, test[2])
singleReturnCnt += 1
myCrawler = crawler.createCrawler(settings, callback)
myCrawler.process()
assert singleReturnCnt == test[
1], "Found {} instead of {} files".format(singleReturnCnt, test[1])
def find_gen(baseSettings, tests):
for i, test in enumerate(tests):
settings = baseSettings
for key, val in test[0].items():
settings[key] = val
myCrawler = crawler.createCrawler(settings)
myCrawler.memo = []
gen = myCrawler.generator()
cnt = 0
results = []
try:
while True:
name = next(gen)
results.append(name)
assert name in test[
2], "Unexpected file ({}) appeared in found files. During Test: {}".format(
name, i)
cnt += 1
except StopIteration:
assert cnt == test[1], "Found {} instead of {} {} files".format(
cnt, test[1], test[0])
def callback_listReturn(baseSettings, tests):
settings = baseSettings
settings["singleReturn"] = False
for test in tests:
for key, val in test[0].items():
settings[key] = val
settings["onlyOnce"] = True
def callback(files):
if len(test[2]) > 0:
for file in files:
assert file in test[
2], "Couldn't find file ({}) in {}".format(
file, test[2])
assert len(
files) == test[1], "Found {} instead of {} files".format(
len(files), test[1])
myCrawler = crawler.createCrawler(settings, callback)
myCrawler.memo = []
myCrawler.process()
def test_create_google_crawler():
settings = baseSettings
crawler.createCrawler(settings)