from totalimpact.models import Error
from totalimpact.tilogging import logging
from totalimpact.providers.provider import ProviderConfigurationError, ProviderTimeout, ProviderHttpError
from totalimpact.providers.provider import ProviderClientError, ProviderServerError, ProviderContentMalformedError
from totalimpact.providers.provider import ProviderValidationFailedError, ProviderRateLimitError
import daemon
import lockfile
from totalimpact.pidsupport import PidFile
from optparse import OptionParser
import os
logger = logging.getLogger('backend')
class TotalImpactBackend(object):
def __init__(self, dao, providers):
self.threads = []
self.dao = dao
self.providers = providers
def run(self):
self._spawn_threads()
try:
self._monitor()
except (KeyboardInterrupt, SystemExit):
logger.info("Interrupted ... exiting ...")
self._cleanup()
def main(logfile=None):
logger = logging.getLogger()
mydao = dao.Dao(
app.config["DB_NAME"],
app.config["DB_URL"],
app.config["DB_USERNAME"],
app.config["DB_PASSWORD"]
)
# Adding this by handle. fileConfig doesn't allow filters to be added
from totalimpact.backend import ctxfilter
handler = logging.handlers.RotatingFileHandler(logfile)
handler.level = logging.DEBUG
formatter = logging.Formatter("%(asctime)s %(levelname)8s %(item)8s %(thread)s%(provider)s - %(message)s")#,"%H:%M:%S,%f")
handler.formatter = formatter
handler.addFilter(ctxfilter)
logger.addHandler(handler)
ctxfilter.threadInit()
logger.debug("test")
from totalimpact.backend import TotalImpactBackend, ProviderMetricsThread, ProvidersAliasThread, StoppableThread, QueueConsumer
from totalimpact.providers.provider import Provider, ProviderFactory
# Start all of the backend processes
print "Starting alias retrieval thread"
providers = ProviderFactory.get_providers(app.config["PROVIDERS"])
alias_threads = []
thread_count = app.config["ALIASES"]["workers"]
for idx in range(thread_count):
at = ProvidersAliasThread(providers, mydao, idx)
at.thread_id = 'AliasThread(%i)' % idx
at.start()
alias_threads.append(at)
print "Starting metric retrieval threads..."
# Start each of the metric providers
metrics_threads = []
for provider in providers:
providers = ProviderFactory.get_providers(app.config["PROVIDERS"])
thread_count = app.config["PROVIDERS"][provider.provider_name]["workers"]
print " ", provider.provider_name
for idx in range(thread_count):
thread = ProviderMetricsThread(provider, mydao)
metrics_threads.append(thread)
thread.thread_id = thread.thread_id + '(%i)' % idx
thread.start()
# Install a signal handler so we'll break out of the main loop
# on receipt of relevant signals
class ExitSignal(Exception):
pass
def kill_handler(signum, frame):
raise ExitSignal()
import signal
signal.signal(signal.SIGTERM, kill_handler)
try:
while True:
time.sleep(1)
except (KeyboardInterrupt, ExitSignal), e:
pass
# To read global config
from totalimpact.api import app
TEST_DB_NAME = "test_dao"
from totalimpact.providers.provider import ProviderConfigurationError, ProviderTimeout, ProviderHttpError
from totalimpact.providers.provider import ProviderClientError, ProviderServerError, ProviderContentMalformedError
from totalimpact.providers.provider import ProviderValidationFailedError, ProviderRateLimitError
from test.mocks import ProviderMock, QueueMock, ItemMock
def slow(f):
f.slow = True
return f
logger = logging.getLogger(__name__)
CWD, _ = os.path.split(__file__)
def dao_init_mock(self, name, url):
pass
class InterruptTester(object):
def run(self, stop_after=0):
st = StoppableThread()
st.start()
time.sleep(stop_after)
st.stop()
st.join()
class ProviderNotImplemented(Provider):
