def initialize(self):
# FIXME: isn't there a better way testing if etcd is writable?
# wait for etcd to be available
while not self.touch_member():
logging.info('waiting on etcd')
sleep(5)
# is data directory empty?
if self.postgresql.data_directory_empty():
# racing to initialize
if self.etcd.race('/initialize', self.postgresql.name):
self.postgresql.initialize()
self.etcd.take_leader(self.postgresql.name)
self.postgresql.start()
else:
# FIXME: touch_member?
while True:
leader = self.etcd.current_leader()
if leader and self.postgresql.sync_from_leader(leader):
self.postgresql.write_recovery_conf(leader)
self.postgresql.start()
break
sleep(5)
elif self.postgresql.is_running():
self.postgresql.load_replication_slots()
def crawl(scrapeQueue, writeQueue):
"""
:param scrapeQueue: queue for scrapping task
:param writeQueue: queue for writing task
:return:
"""
global STOP
reset()
while True and not STOP:
while scrapeQueue.hasItems and not STOP:
# get item
req = scrapeQueue.getItem()
if not req:
continue
# check if task has already been registered before, else register
if scrapeQueue.exists(req[0]):
continue
scrapeQueue.register(req[0], {})
# scrape the article
scrapped = scrapeArticle(*req, scrapeQueue)
if scrapped:
# relevant data found then put a task in wirting queue
fileName = '{}_{}.json'.format(scrapped['dir'], scrapped['title'].replace('/', '-'))
writeQueue.addTask([fileName, scrapped])
utils.logging.debug('queue empty')
utils.sleep(5)
def schedule_next_run(self):
self.next_run += self.nap_time
current_time = time.time()
nap_time = self.next_run - current_time
if nap_time <= 0:
self.next_run = current_time
else:
sleep(nap_time)
def get_etcd_client(self, config):
client = None
while not client:
try:
client = Client(config)
except etcd.EtcdException:
logger.info('waiting on etcd')
sleep(5)
return client
def get_etcd_client(self, config):
client = None
while not client:
try:
client = Client(config)
except etcd.EtcdException:
logger.info('waiting on etcd')
sleep(5)
return client
def __init__(self, hosts, port, uri_path='/exhibitor/v1/cluster/list', poll_interval=300):
self._exhibitor_port = port
self._uri_path = uri_path
self._poll_interval = poll_interval
self._exhibitors = hosts
self._master_exhibitors = hosts
self._zookeeper_hosts = ''
self._next_poll = None
while not self.poll():
logger.info('waiting on exhibitor')
sleep(5)
def __init__(self,
hosts,
port,
uri_path='/exhibitor/v1/cluster/list',
poll_interval=300):
self._exhibitor_port = port
self._uri_path = uri_path
self._poll_interval = poll_interval
self._exhibitors = hosts
self._master_exhibitors = hosts
self._zookeeper_hosts = ''
self._next_poll = None
while not self.poll():
logger.info('waiting on exhibitor')
sleep(5)
def query(self, sql, *params):
max_attempts = 0
while True:
ex = None
try:
cursor = self._cursor()
cursor.execute(sql, params)
return cursor
except psycopg2.InterfaceError as e:
ex = e
except psycopg2.OperationalError as e:
if self._connection and self._connection.closed == 0:
raise e
ex = e
if ex:
self.disconnect()
max_attempts += 1
if max_attempts >= 3:
raise ex
sleep(5)
def query(self, sql, *params):
max_attempts = 0
while True:
ex = None
try:
cursor = self._cursor()
cursor.execute(sql, params)
return cursor
except psycopg2.InterfaceError as e:
ex = e
except psycopg2.OperationalError as e:
if self._connection and self._connection.closed == 0:
raise e
ex = e
if ex:
self.disconnect()
max_attempts += 1
if max_attempts >= 3:
raise ex
sleep(5)
def writer(writeQueue):
"""
write the data in a json file, reads from the writing task queues and performs the task
:param writeQueue: writing task queue
:return:
"""
global STOP
reset()
while True and not STOP:
while writeQueue.hasItems() and not STOP:
req = writeQueue.getItem()
if not req:
continue
if writeQueue.exists(req[0]):
continue
utils.logging.debug('writing file: '+req[0])
createRecord(*req)
writeQueue.register(req[0], req[1])
utils.logging.debug('queue empty')
utils.sleep(5)
def initialize(self):
# wait for etcd to be available
while not self.touch_member():
logger.info('waiting on DCS')
sleep(5)
# is data directory empty?
if self.postgresql.data_directory_empty():
# racing to initialize
if self.ha.dcs.race('/initialize'):
self.postgresql.initialize()
self.ha.dcs.take_leader()
self.postgresql.start()
else:
while True:
leader = self.ha.dcs.current_leader()
if leader and self.postgresql.sync_from_leader(leader):
self.postgresql.write_recovery_conf(leader)
self.postgresql.start()
break
sleep(5)
elif self.postgresql.is_running():
self.postgresql.load_replication_slots()
def get_client_path(self, path, max_attempts=1):
attempts = 0
response = None
while True:
ex = None
try:
response = requests.get(self.client_url(path))
if response.status_code == 200:
break
except RequestException as e:
logger.exception('get_client_path')
ex = e
attempts += 1
if attempts < max_attempts:
logger.info('Failed to return %s, trying again. (%s of %s)', path, attempts, max_attempts)
sleep(3)
elif ex:
raise ex
else:
break
return response.json(), response.status_code
def test_sleep(self):
time.sleep = time_sleep
sleep(0.01)
def sleep(self, timeout):
sleep(timeout)
def test_sleep(self):
time.sleep = time_sleep
sleep(0.01)
