# Initialize variables
filename = "data/urls.json"
# Read JSON data into the urls variable
with open(filename, 'r') as f:
try:
urls = json.load(f)
except Exception as e:
print("got exception {e}".format(e=e))
minBatch = 10
# use a heuristics to prevent undersized batches from being used
numOfBatches = min(minBatch, (int(len(urls) / minBatch) + 1))
httpObj = HTTPClass()
MAXTIME = 1000 # account for possible network latency
MININTERVAL = 60 # I minute
def scrapWeb(url, interval, tol=5):
"""
scrabing the url
"""
print('Starting %s' % url)
before = time.time()
try:
# tol is slackness on timeout
httpObj.getContent(url, interval=interval - tol)
except Exception as e:
print("got exception {e}".format(e=e))
def __init__(self, xmlFileName="configuration/config.xml"):
self.myhttp = HTTPClass(xmlFileName) #create the HTTP object
def setUp(self):
"""
This method is called before each test
"""
self.httpObj = HTTPClass()
