class TestLinkDataBase(object):
def setup(self):
self.links_db = BerkeleyBTreeLinkDB("test_db", SimpleKeyPolicyModule)
def test_one_in_base(self):
_link = "www.test.com"
self.links_db.add_link(_link, 1, 1)
assert self.links_db.get_link() is not None
def test_get_from_empty_base(self):
assert self.links_db.get_link() is None
def test_empty_queue(self):
_link = "www.test.com"
self.links_db.add_link(_link, 1, 1)
self.links_db.get_link()
assert self.links_db.get_link() is None
def test_priority_set(self):
_link = "www.test.com"
self.links_db.add_link(_link, 1, 8)
_details = self.links_db.get_details(_link)
assert _details[2] == '8'
def test_depth_set(self):
_link = "www.test.com"
self.links_db.add_link(_link, 6, 4)
_details = self.links_db.get_details(_link)
assert _details[0] == '6'
def test_is_in_base(self):
_link = "www.test.com"
self.links_db.add_link(_link, 1, 1)
assert self.links_db.is_in_base(_link) is True
def test_is_not_in_base(self):
_link1 = "www.test.com"
_link2 = "www.test22222.com"
self.links_db.add_link(_link1, 1, 1)
assert self.links_db.is_in_base(_link2) is False
def test_best_priority_in_queue1(self):
'''
Kolejnosc liter.
'''
_link = "www.aaa.com"
self.links_db.add_link("www.abc.com", 1, 3)
self.links_db.add_link(_link, 1, 1)
assert self.links_db.get_link() == _link
def test_best_priority_in_queue2(self):
"""
Dluzszy link.
"""
_link = "www.aaa.com"
self.links_db.add_link(_link, 1, 1)
self.links_db.add_link("www.aaa.com/aaaa/aa", 1, 3)
assert self.links_db.get_link() == _link
def test_best_priority_in_queue3(self):
"""
Link krotszy, ale kolejnosc liter.
"""
_link = "www.aaa.com"
self.links_db.add_link(_link, 1, 1)
self.links_db.add_link("www.zz.com", 1, 3)
assert self.links_db.get_link() == _link
def test_best_priority_in_queue4(self):
"""
Priorytet.
"""
_link = "www.aaa.com"
self.links_db.add_link(_link, 1, 1)
_link2 = "www.zz.com"
self.links_db.add_link("www.zz.com", 4, 3)
assert self.links_db.get_link() == _link2
def test_best_priority_in_queue5(self):
"""
Priorytety o roznych liczbach cyfr.
"""
_link = "www.zzz.com"
self.links_db.add_link(_link, 12, 1)
self.links_db.add_link("www.aaa.com", 4, 3)
assert self.links_db.get_link() == _link
def test_best_priority_in_queue6(self):
"""
Priorytety o roznych liczbach cyfr.
"""
_link = "www.zzz.com"
self.links_db.add_link(_link, 12, 1)
self.links_db.add_link("www.aaa.com/asasf/34", 4, 3)
assert self.links_db.get_link() == _link
def test_set_as_fetched(self):
_link = "www.zzz.com"
self.links_db.add_link(_link, 12, 1)
_t0 = datetime.datetime.now()
self.links_db.set_as_fetched(_link)
_details = self.links_db.get_details(_link)
assert _t0 < datetime.datetime.strptime(_details[1], "%Y-%m-%d %H:%M:%S.%f")
def test_feedback(self):
_link = "www.zzz.com"
self.links_db.add_link(_link, 12, 1)
self.links_db.change_link_priority(_link, 5)
_details = self.links_db.get_details(_link)
assert _details[0] == '5'
def test_feedback_correct_keys(self):
_link = "www.zzz.com"
self.links_db.add_link(_link, 12, 1)
self.links_db.change_link_priority(_link, 5)
assert self.links_db.policy_module.generate_key(_link, 5) in self.links_db.priority_queue
def teardown(self):
self.links_db.clear()
class TaskServer(threading.Thread):
def __init__(self, web_server, task_id, manager_address, max_url_depth=1):
threading.Thread.__init__(self)
self.status_lock = threading.Lock()
self.cache_lock = threading.RLock()
self.data_lock = threading.RLock()
self.statistics_lock = threading.RLock()
self.web_server = web_server
self.manager_address = manager_address
self.link_db = BerkeleyBTreeLinkDB('link_db', SimpleKeyPolicyModule)
self.content_db = BerkeleyContentDB('content_db')
self.crawlers = {}
self.task_id = task_id
self.max_links = 0
self.expire_date = None
self.mime_type = []
self.uuid = ''
self.whitelist = []
self.blacklist = []
self.urls_per_min = 0
self.package_cache = {}
self.package_id = 0
self.processing_crawlers = []
self.max_url_depth = max_url_depth
self.status = Status.INIT
self.crawled_links = []
self.stats_reset_time = time.time()
self.logger = logging.getLogger('server')
_file_handler = logging.FileHandler('server%s.log' % task_id)
_formatter = logging.Formatter('<%(asctime)s>:%(levelname)s: %(message)s')
_file_handler.setFormatter(_formatter)
self.logger.addHandler(_file_handler)
self.logger.setLevel(logging.DEBUG)
def assign_crawlers(self, assignment):
"""
Sets actual crawler assignment.
Task server can send crawling requests only to these crawlers
and size of packages must be specified in assignment dict for each crawler.
It allows to control crawling efficiency of all task servers.
"""
self.data_lock.acquire()
self.crawlers = assignment
self.data_lock.release()
self.logger.debug('%d crawlers assigned' % len(assignment))
def assign_speed(self, speed):
"""
Sets task server's crawling speed.
After each speed change statistics are reset.
"""
self.data_lock.acquire()
self.urls_per_min = int(speed)
self._reset_stats()
self.data_lock.release()
self.logger.debug('Changed speed to %d', self.urls_per_min)
def get_address(self):
# TODO: change this address to external ip address (and in crawler too)
return 'http://' + self.web_server.get_host()
def _set_status(self, status):
"""
Sets task server state.
"""
self.status_lock.acquire()
self.status = status
self.status_lock.release()
self.logger.debug('Changed status to: %d' % status)
def _get_status(self):
"""
Returns actual task server state.
"""
self.status_lock.acquire()
_status = self.status
self.status_lock.release()
return _status
def _register_to_management(self):
"""
Sends register request to FCS main application.
Received data is used to set crawling parameters.
"""
r = requests.post(self.manager_address + '/autoscale/server/register/',
data={'task_id': self.task_id, 'address': self.get_address()})
self.logger.debug('Registering to management. Return code: %d, message: %s' % (r.status_code, r.content))
if r.status_code in [status.HTTP_412_PRECONDITION_FAILED, status.HTTP_404_NOT_FOUND]:
self.stop()
return
self._set_status(Status.RUNNING)
try:
data = r.json()
self.update(data)
self.add_links(data['start_links'], self.link_db.DEFAULT_PRIORITY, 0)
self.data_lock.acquire()
self.uuid = data['uuid']
self.data_lock.release()
self.logger.debug('Registered to management')
except (KeyError, ValueError) as e:
self.logger.debug('Error while registering: %s' % str(e))
self.stop()
def _unregister_from_management(self):
"""
Sends unregister request to FCS main application.
"""
#TODO: check why precondition failed (task not stopped)
r = requests.post(self.manager_address + '/autoscale/server/unregister/',
data={'task_id': self.task_id, 'uuid': self.uuid})
self.logger.debug('Unregistering from management. Return code: %d, message: %s' % (r.status_code, r.content))
def update(self, data):
"""
Updates crawling parameters and status.
It is called usually when user makes some changes in task data using GUI or API.
"""
self.logger.debug('Updating task server: %s' % json.dumps(data))
if self._get_status() in [Status.STOPPING, Status.STARTING]:
return
if data['finished']:
self.stop()
return
self.data_lock.acquire()
self.whitelist = data['whitelist']
self.blacklist = data['blacklist']
self.mime_type = data['mime_type']
self.max_links = int(data['max_links'])
self.expire_date = datetime.strptime(data['expire_date'], DATE_FORMAT)
self.data_lock.release()
if data['active']:
self.resume()
else:
self.pause()
def pause(self):
"""
Pauses task server if it was running.
"""
if self._get_status() == Status.RUNNING:
self._set_status(Status.PAUSED)
def resume(self):
"""
Resumes task server if it was paused.
"""
if self._get_status() == Status.PAUSED:
self._set_status(Status.RUNNING)
def stop(self):
"""
Sets STOPPING status
Task server in this state won't send crawling requests anymore.
It will wait WAIT_FOR_DOWNLOAD_TIME seconds for user to download gathered data.
"""
self._set_status(Status.STOPPING)
def kill(self):
"""
Sets KILLED status
Task server in this state will be stopped as soon as possible
"""
self._set_status(Status.KILLED)
def run(self):
"""
Main task server loop
"""
self._set_status(Status.STARTING)
self.web_server.start()
self._register_to_management()
try:
while self._get_status() not in [Status.STOPPING, Status.KILLED]:
if self._get_status() == Status.RUNNING and not self._efficiency_achieved():
for crawler in self.get_idle_crawlers():
package = self._get_links_package(crawler, self.crawlers[crawler])
if package:
try:
requests.post(crawler + '/put_links', json.dumps(package))
self.logger.debug('Sending links to crawler %s' % crawler)
except ConnectionError:
pass
self._check_cache()
self._check_limits()
self._clear_stats()
time.sleep(1)
shutdown_time = time.time()
while (time.time() - shutdown_time) < WAIT_FOR_DOWNLOAD_TIME and self.content_db.size() > 0 \
and self._get_status() != Status.KILLED:
time.sleep(30)
finally:
self._unregister_from_management()
#check what happens here sometimes that server doesn't shutdown
self._clear()
self.logger.debug('Stopping web interface')
self.web_server.stop()
self.logger.debug('Task server stopped')
def get_idle_crawlers(self):
"""
Returns list of crawlers which are not processing any requests.
"""
self.data_lock.acquire()
crawlers = self.crawlers
self.data_lock.release()
self.cache_lock.acquire()
processing = self.processing_crawlers
self.cache_lock.release()
return [crawler for crawler in crawlers if crawler not in processing]
def _check_limits(self):
"""
Checks if crawling limits are exceeded or no link left to crawl.
If so, tries to stop task.
"""
self.data_lock.acquire()
expire_date = self.expire_date
max_links = self.max_links
self.data_lock.release()
self.cache_lock.acquire()
packages_cached = len(self.package_cache)
self.cache_lock.release()
if datetime.now() > expire_date:
self.logger.debug('Task expired')
self._stop_task()
elif self.content_db.added_records_num() > max_links:
self.logger.debug('Task max links limit exceeded')
self._stop_task()
elif self.link_db.size() == 0 and packages_cached == 0:
self.logger.debug('No links to crawl')
self._stop_task()
def _stop_task(self):
"""
Sends request to FCS main application to stop this task server's task.
If case of error task server will be killed.
"""
r = requests.post(self.manager_address + '/autoscale/server/stop_task/',
data={'task_id': self.task_id, 'uuid': self.uuid})
self.logger.debug('Stopping task. Return code: %d, message: %s' % (r.status_code, r.content))
if r.status_code in [status.HTTP_412_PRECONDITION_FAILED, status.HTTP_404_NOT_FOUND]:
self.kill()
def _cache(self, package_id, crawler, links):
"""
Puts link package into cache and marks assigned crawler as 'processing'
"""
self.cache_lock.acquire()
has_timed_out = False
self.package_cache[package_id] = [time.time(), links, crawler, has_timed_out]
self.processing_crawlers.append(crawler)
self.cache_lock.release()
self.logger.debug('Cached package %d' % package_id)
def _get_links_package(self, crawler, size):
"""
Prepares link package of given size for given crawler and caches it.
"""
_links = []
for i in range(size):
_links.append(self.link_db.get_link())
_links = [link for link in _links if link]
self.logger.debug('Retrieved %d links from linkdb' % len(_links))
if _links:
address = self.get_address()
crawling_type = self.mime_type
package_id = self.package_id
#TODO: change 'crawling_type' to 'mime_type'
package = {'server_address': address, 'crawling_type': crawling_type, 'id': package_id, 'links': _links}
self._cache(package_id, crawler, _links)
self.package_id += 1
return package
else:
return None
def _check_cache(self):
"""
Scans package cache looking for timed out packages.
Marks timed out packages as such
and again puts links that they contained into database.
"""
cur_time = time.time()
self.cache_lock.acquire()
for package_id in self.package_cache.keys():
if (cur_time - self.package_cache[package_id][0] > URL_PACKAGE_TIMEOUT) and \
not self.package_cache[package_id][3]:
self.logger.debug('Package %d has timed out. Readding' % package_id)
self.readd_links(self.package_cache[package_id][1])
self.package_cache[package_id][3] = True
elif (cur_time - self.package_cache[package_id][0]) > 5 * URL_PACKAGE_TIMEOUT:
self._clear_cache(package_id)
self.cache_lock.release()
def _clear_cache(self, package_id):
"""
Removes entry from package cache for given package id.
It also marks crawler which was assigned to this crawling request as 'idle'
so next request can be sent to this crawler.
"""
self.cache_lock.acquire()
try:
self.processing_crawlers.remove(self.package_cache[package_id][2])
del self.package_cache[package_id]
self.logger.debug('Removed package %d from cache' % package_id)
except KeyError:
pass
self.cache_lock.release()
def feedback(self, regex, rate):
# TODO: change this method to feedback regex (which will be created soon)
#self.link_db.change_link_priority(regex, rate)
pass
def _evaluate_link(self, link):
"""
Checks blacklist and whitelist and decides
if link can be put into link queue.
"""
domain = urlparse(link).netloc
if not domain:
self.logger.debug('Link evaluation failed. Bad link format: %s' % link)
return False
for regex in self.blacklist:
if re.match(regex, domain):
return False
for regex in self.whitelist:
if re.match(regex, domain):
return True
return False
def add_links(self, links, priority, depth=0, source_url=""):
_counter = 0
self.logger.debug('Trying to add %d links' % len(links))
for link in links:
_link = URLProcessor.validate(link, source_url)
if self._evaluate_link(_link) and not self.link_db.is_in_base(_link):
#_depth = SimpleCrawlingDepthPolicy.calculate_depth(link, source_url, depth)
#_depth = RealDepthCrawlingDepthPolicy.calculate_depth(link, self.link_db)
_depth = IgnoreDepthPolicy.calculate_depth()
if _depth <= self.max_url_depth:
self.logger.debug("Added:%s with priority %d" % (_link, _depth))
self.link_db.add_link(_link, priority, _depth)
_counter += 1
self.logger.debug("Added %d new links into DB." % _counter)
def readd_links(self, links):
for link in links:
# adds link only when it was earlier in linkdb
# TODO : ???
self.link_db.change_link_priority(link, BerkeleyBTreeLinkDB.BEST_PRIORITY)
def _decode_content(self, content):
return Base64ContentCoder.decode(content)
def put_data(self, package_id, data):
"""
Handles data package received from crawler and puts it into content database
If received package isn't in package cache (or crawling request has timed out)
no data will be stored in database.
It also marks crawler which was assigned to this crawling request as 'idle'
so next request can be sent to this crawler.
"""
if package_id in self.package_cache:
has_timed_out = self.package_cache[package_id][3]
if not has_timed_out:
self.logger.debug('Putting content from package %d' % package_id)
start_time = self.package_cache[package_id][0]
end_time = time.time()
self._add_stats(start_time, end_time, len(data))
for entry in data:
self.logger.debug('Adding content from url %s' % entry['url'])
self.content_db.add_content(entry['url'], entry['links'], entry['content'])
# TODO: put correct depth and priority value (based on previous url)
_details = self.link_db.get_details(entry['url'])
_url_depth = _details is not None and _details[2] or 0
self.add_links(entry['links'], BerkeleyBTreeLinkDB.DEFAULT_PRIORITY, _url_depth, entry['url'])
self._clear_cache(package_id)
def _clear(self):
"""
Clears database files
"""
self.logger.debug('Clearing db files')
self.link_db.clear()
self.content_db.clear()
def get_data(self, size):
"""
Returns path to file with crawling results.
"""
self.logger.debug('Downloading content - %d size' % size)
return self.content_db.get_file_with_data_package(size)
def _add_stats(self, start_time, end_time, links):
"""
Adds new statistics entry - links number with current time
"""
self.statistics_lock.acquire()
self.crawled_links.append((start_time, end_time, links))
self.statistics_lock.release()
def _reset_stats(self):
"""
Removes all statistics entries and
resets time from which statistics can be measured.
"""
self.statistics_lock.acquire()
self.crawled_links = []
self.stats_reset_time = time.time()
self.statistics_lock.release()
def _get_stats(self, seconds):
"""
Returns statistics summarise for the last seconds in a dict.
Dict keys:
'seconds' - actual number of seconds for which measurement was done
'links' - number of links crawled during this time
'speed' - crawling speed assigned to this task server
"""
self.statistics_lock.acquire()
now = time.time()
from_time = now - seconds
if self.stats_reset_time > from_time:
from_time = self.stats_reset_time
links = 0
for entry in self.crawled_links:
if entry[0] > from_time:
links += entry[2]
elif entry[1] > from_time:
links += int(entry[2] * (entry[1] - from_time) / (entry[1] - entry[0]))
self.statistics_lock.release()
ret = dict()
ret['seconds'] = int(now - from_time)
ret['links'] = links
self.data_lock.acquire()
ret['urls_per_min'] = self.urls_per_min
self.data_lock.release()
return ret
def _clear_stats(self):
"""
Removes statistics entries which are too old
"""
self.statistics_lock.acquire()
from_time = max(time.time() - KEEP_STATS_SECONDS, self.stats_reset_time)
index = 0
for i in range(len(self.crawled_links)):
if self.crawled_links[i][0] > from_time:
index = i
break
self.crawled_links = self.crawled_links[index:]
self.stats_reset_time = from_time
self.statistics_lock.release()
def _efficiency_achieved(self):
"""
Checks if crawled links number in the last time is above expectation.
"""
stats = self._get_stats(CRAWLING_PERIOD)
if stats['seconds'] > 0:
return 60. * stats['links'] / stats['seconds'] >= stats['urls_per_min']
return False