def main():
try:
import os
dr = os.path.dirname(os.path.realpath(__file__))
config = read_env('{}/config.cfg'.format(dr))
except:
config = read_env('config.cfg')
logger.info("Launching forwarder sucking and spitting")
#Create the sucking socket
try:
context = zmq.Context(1)
# Socket facing clients
frontend = context.socket(zmq.SUB)
frontend.bind(config['ZMQ_FORWARDER_SUCKS_IN'])
frontend.setsockopt(zmq.SUBSCRIBE, "") #subscribe to everything
except Exception, e:
logger.debug(str(e))
logger.debug("Bringing down sucking device")
def main():
try:
import os
dr = os.path.dirname(os.path.realpath(__file__))
config = read_env('{}/config.cfg'.format(dr))
except IOError:
config = read_env('config.cfg')
logger.info("Launching forwarder sucking and spitting")
# Create the sucking socket
try:
context = zmq.Context(1)
# Socket facing clients
frontend = context.socket(zmq.SUB)
frontend.bind(config['ZMQ_FORWARDER_SUCKS_IN'])
frontend.setsockopt(zmq.SUBSCRIBE, "") # subscribe to everything
except Exception, e:
logger.debug(str(e))
logger.debug("Bringing down sucking device")
def __init__(self, jobstore, url):
self.logger = setup(__name__)
self.logger.debug("Creating MessageScheduler")
self.logger.debug("id = {}".format(id(self)))
config = read_env('config.cfg')
self._scheduler = Scheduler(daemonic=True)
config_scheduler = {'apscheduler.jobstores.file.class': 'apscheduler.jobstores%s' % jobstore,
'apscheduler.jobstores.file.url': url}
self._scheduler.configure(config_scheduler)
#Open a publishing socket to the forwarder to pass messages out
self.broadcast_socket = zmq.Context().socket(zmq.PUB)
self.broadcast_socket.connect(config['ZMQ_FORWARDER_SUCKS_IN'])
def __init__(self, jobstore, url):
self.socket = zmq.Context().socket(zmq.SUB)
self.logger = setup(__name__)
self.logger.debug("Creating MessageScheduler")
self.logger.debug("id = {}".format(id(self)))
config = read_env('config.cfg')
self._scheduler = Scheduler(daemonic=True)
config_scheduler = {'apscheduler.jobstores.file.class': 'apscheduler.jobstores%s' % jobstore,
'apscheduler.jobstores.file.url': url}
self._scheduler.configure(config_scheduler)
# Open a publishing socket to the forwarder to pass messages out
self.broadcast_socket = zmq.Context().socket(zmq.PUB)
self.broadcast_socket.connect(config['ZMQ_FORWARDER_SUCKS_IN'])
def start_listener(self):
""" Connects to forwarder_device, runs forever. Launch in
separate process.
"""
config = read_env('config.cfg')
self.logger.debug("Scheduler listener start")
try:
self.socket.setsockopt(zmq.SUBSCRIBE, "scheduler")
self.socket.connect(config['ZMQ_FORWARDER_SPITS_OUT'])
except Exception, e:
print e
print "bringing down port {} device".format(port)
self.socket.close()
def start_listener(self):
""" Connects to forwarder_device, runs forever. Launch in
separate process.
"""
config = read_env('config.cfg')
self.logger.debug("Scheduler listener start")
try:
self.socket = zmq.Context().socket(zmq.SUB)
self.socket.setsockopt(zmq.SUBSCRIBE, "scheduler")
self.socket.connect(config['ZMQ_FORWARDER_SPITS_OUT'])
except Exception, e:
print e
print "bringing down port {} device".format(port)
self.socket.close()
def main():
"""Shows basic usage of the Sheets API.
Prints values from a sample spreadsheet.
"""
env = read_env()
print('start main')
lang_list = get_range(env['lang'])
sheet = sheets_api(credential(env['credential_file_path']))
result = sheet.values().get(spreadsheetId=SAMPLE_SPREADSHEET_ID,
range=SAMPLE_RANGE_NAME).execute()
print(result)
reader = Reader(result)
# print(reader.get_column_info())
# print(reader.get_lang_by_column('ko'))
'''
def run():
config = read_env('config.cfg')
scheduler = MessageScheduler(config['jobstore'], config['url'])
# start APscheduler daemon in own thread
scheduler.start_ap_daemon()
# shut scheduler threads cleanly at exit
atexit.register(lambda: scheduler.shutdown())
logger.info("About to launch scheduler listener")
# start listener for new schedule events from anywhere
try:
scheduler.start_listener()
except KeyboardInterrupt:
scheduler.shutdown()
except Exception, e:
logger.debug("exception in scheculer start_listener():{}".format(e))
def run():
config = read_env('config.cfg')
scheduler = MessageScheduler(config['jobstore'],config['url'])
# start APscheduler daemon in own thread
scheduler.start_ap_daemon()
# shut scheduler threads cleanly at exit
atexit.register(lambda: scheduler.shutdown())
logger.info("About to launch scheduler listener")
# start listener for new schedule events from anywhere
try:
scheduler.start_listener()
except KeyboardInterrupt:
scheduler.shutdown()
except Exception, e:
logger.debug("exception in scheculer start_listener():{}".format(e))
import time as timer
import random
from env import read_env
from logger import Logger
from crawler import Crawler
from exception import AuthException
logger = Logger()
read_env()
# We run this continuously
while True:
try:
crawler = Crawler()
crawler.run()
rntime = random.randrange(30, 60, 5)
logger.log('Sleeping for %s seconds ... ' % rntime)
timer.sleep(rntime)
except AuthException as e:
logger.error('Authentication error: %s' % str(e))
except AttributeError as e:
logger.error('Attribute error: %s' % str(e))
except Exception as e:
logger.error('Unknown error: %s' % str(e))