def load(self, dbpath, name): ''' load table :return: self ''' try: #initialize table parameters self.name = name self.path = join_paths(dbpath, name) self.table_file = join_paths(self.path, "table") self.data_file = join_paths(self.path, "data") self.table = self.desc() #load data file if not is_file(self.data_file): #create data file if not exists self._create_data_file() else: #replace old data file if needed with open(self.data_file) as fdata: nfields = strips(fdata.readline().split(",")) if self.table.nfields() != nfields: if is_subset(nfields, self.table.nfields()): self._upgrade_data_file() else: self._replace_data_file() Logger.info("loading table %s...success.", self.name) return self except Exception as e: Logger.info("loading table %s...failed. error: %s", self.name, str(e)) raise e
def handle(self): parser = reqparse.RequestParser() parser.add_argument('type') args = parser.parse_args() usage = '' if args.type is None: return CommonUtil.json_response(-1, '缺少参数:type') elif args.type == '0': usage = 'regAccount' # 注册账号 elif args.type == '1': usage = 'findPassword' # 找回密码 elif args.type == '2': usage = 'adminLogin' # 管理台登录 else: return CommonUtil.json_response(-1, 'type参数格式错误') # 用客户端ip来作为sendId是为了使频繁请求时可以替换这个key下面原来的验证码 md5 = hashlib.md5() md5.update("validimage_{}_{}".format(request.environ['REMOTE_ADDR'], usage).encode('utf-8')) sendId = md5.hexdigest() validImage = ValidImage.create() Redis.setex(sendId, 60, validImage['code']) data = {"img": validImage['img'], "sendId": sendId} Logger.log("生成图片验证码 ip:{} sendId:{} code:{}".format( request.environ['REMOTE_ADDR'], sendId, validImage['code'])) return CommonUtil.json_response(0, "success", data)
def output_json(data, code, headers=None): """Makes a Flask response with a JSON encoded body""" # 如果是app接口,且不是支付回调,加密后返回 Logger.log("请求id:%s 响应\n返回JSON:%s\n" % (session['requestId'], data)) resp = make_response(json.dumps(data), code) resp.headers.extend(headers or {}) return resp
def open(self, host, user, pwd, dbn, port=3306): ''' open database or create it if not exist :return: ''' #init store path try: self.host, self.port, self.user, self.pwd = host, port, user, pwd self.dbn = dbn self.dbc = pymysql.connect(host=host, user=user, passwd=pwd, port=port) if not self._exists(): #create database self._create() self._use() else: # load database self._use() self._load() self._rebuild_tindex() return self Logger.info("open store mysql://%s:%s@%s:%d/%s...success. %d tables.", user, pwd, host, port, self.dbn, len(self.tables)) except Exception as e: Logger.error("open store mysql://%s:%s@%s:%d/%s...failed. error: %s", user, pwd, host, port, self.dbn, str(e)) raise e
def test_print_result(self): device = FakeDevice(SourceType.IMAGE, 'dataset/png/') # device = FakeDevice(SourceType.IMAGE, '/media/zis/Dados/dev/datasets/putkk.poznan/Dataset_1_Kin_2') algorithms = [] algorithms.append( RGBDSegmentation( Parameter(Segmentation.GRAPH_CANNY, os.getcwd() + '/results/graph_canny/', resize=(640, 360), fix_proportion=False))) #algorithms.append(RGBDSegmentation(Parameter(Segmentation.RGBD_SALIENCY, os.getcwd() + '/results/rgbd_saliency/', resize=(324, 324)))) #algorithms.append(RGBDSegmentation(Parameter(Segmentation.FCN_TENSORFLOW, os.getcwd() + '/results/fcn_tensorflow/', resize=(224, 224)))) #algorithms.append(RGBDSegmentation(Parameter(Segmentation.FUSENET, os.getcwd() + '/results/fusenet/', resize=(224, 224)))) directory_rgb = device.datasetPath + 'rgb/' directory_depth = device.datasetPath + 'depth/' for algorithm in algorithms: Logger.info('############# Testing algorithm ' + str(algorithm.parameter.segmentation)) for i in range(0, 10): frame = RGBDFrame( RGBFrame(directory_rgb, 'rgb_' + format(i, '05') + '.png'), DepthFrame(directory_depth, 'depth_' + format(i, '05') + '.png')) algorithm.process(frame) algorithm.write_results() algorithm.finish() #break algorithm.release() break
def select(self): ''' select all data from table :return: ''' try: with Lock(self.lock): with open(self.data_file, "r") as fdata: models = [] #read field names nfields = strips(fdata.readline().strip().split(",")) #read data records data = fdata.readline() while data: data = data.strip() vfields = strips(data.split(",")) model = {} for idx in range(0, len(nfields)): model[nfields[idx]] = str2obj(vfields[idx], ',') models.append(model) data = fdata.readline() return models except Exception as e: Logger.info("select data from table %s...failed. error: %s", self.name, str(e)) raise e
def open_script_config(productModel): dict_script = { "default": "default", #没有适配的手机 "Xiaomi": "default", "samsung": "default", "360": "default", "Lenovo": "default", "vivo": "default", "HUAWEI": "default", "OPPO": "default", "gionee": "default", "xlj": "default", "yunso": "default", "oysin": "default" } try: if productModel in dict_script: return dict_script[productModel] else: return dict_script['default'] except: log = Logger() log.error("model_config.py, open_model_config : get model error") return dict_script['default']
def create(self, dbc, table): ''' create table :return self ''' try: #initialize table parameters self.dbc = dbc self.table = table self.name = table.name #check if table has exists if self._exists_table(): #exists table old_table = self.desc() if old_table != self.table: if is_subset(old_table.nfields(), self.table.nfields()): #upgrade table self._upgrade_table() else: #replace table self._replace_table else: #table is the same as in database pass else: #create new table self._create_table() Logger.info("create table %s...success.", self.name) return self except Exception as e: Logger.error("create table %s...failed. error: %s", self.name, str(e)) raise e
def open(self, path): ''' open store or create it if not exist :return: ''' try: with Lock(self.lock): #init store path self.path = path if not path_exists(self.path): #create database self._create() else: # load database self._load() self._rebuild_tindex() return self Logger.info("open store %s...success. %d tables.", self.path, len(self.tables)) except Exception as e: Logger.error("open store %s...failed. error: %s", self.path, str(e)) raise e
def __init__(self, queue_name, route, persister, wnum, port, logfile=sys.stdout): self.queue_name = queue_name self.route = route self.persister = persister self.stop = False self.pid = os.getpid() self.worker_id = '-'.join( ['worker', str(wnum), queue_name, route, str(self.pid)]) self.log = Logger(self.worker_id, logfile=logfile, loglevel=logging.DEBUG) self.log.info("starting") self.host = socket.gethostbyname(socket.gethostname()) self.port = port self.register() self.todo = None self.stop = False
def create_table(self, table): ''' create table in current database :param table: :return: ''' with Lock(self.lock): # test if the table has loaded for t in self.tables: if t.table == table: Logger.info("create table %s...exists.", table.name) return #create new table table = FSTable().create(self.path, table) for i in range(0, len(self.tables)): t = self.tables[i] if t.table.name == table.name: self.tables.pop(i) break self.tables.append(table) self._rebuild_tindex()
def select(self): ''' select all data from table :return: ''' try: nfields = self.table.nfields() sql, models = "select %s from %s;" % (",".join(quotes( nfields, '`')), self.name), [] cursor = self.dbc.cursor() cursor.execute(sql) results = cursor.fetchall() for result in results: model = {} for idx in range(0, len(result)): nfield = nfields[idx] vfield = result[idx] if isinstance(vfield, str): vfield = unescapes(vfield) model[nfield] = vfield models.append(model) Logger.info("select from table %s...success", self.name) return models except Exception as e: Logger.error("select from table %s...failed. error %s", self.name, str(e)) raise e
def get_request_token(self): Logger.info('Starting to fetch request token for Kite API') selenium = SeleniumDispatcher(headless=True) driver = selenium.get_driver() driver.get(self.kite.login_url()) time.sleep(4) username_field = driver.find_element_by_xpath("//input[@type='text']") username_field.send_keys(self.username) password_field = driver.find_element_by_xpath("//input[@type='password']") password_field.send_keys(self.password) password_field.send_keys(Keys.ENTER) time.sleep(2) pin_field = driver.find_element_by_xpath("//input[@type='password']") pin_field.send_keys(self.pin) pin_field.send_keys(Keys.ENTER) time.sleep(2) url = driver.current_url parsed = urlparse.urlparse(url) token = parse_qs(parsed.query)['request_token'][0] Logger.info('Request token received!') selenium.destroy_driver() return token
def indian_business_news(self): res = requests.get(self.business_url + '&pageSize=100&apiKey=' + self.api_key).json() total_results = res['totalResults'] self.total_articles = int(total_results) # 100 is the max pagesize for newsapi.org if total_results <= 100: self.insert_data_into_db(res['articles']) else: # break them in chunks of 10 number_of_pages = math.ceil(total_results) / 10 collection = self.client[Config.MONGO_DB][Config.NEWS_COLLECTION] for page in range(1, number_of_pages): res = requests.get( self.business_url + \ '&pageSize=10&page={}&apiKey='.format(page) + \ self.api_key).json() self.insert_data_into_db(res['articles']) if self.inserted_articles > 0: Logger.info( 'News data stored successfully!\nTotal articles: {}\nInserted articles: {}' .format(self.total_articles, self.inserted_articles)) else: Logger.info('Total articles: {}\nNot inserted articles: {}'.format( self.total_articles, self.not_inserted_articles))
def __init__(self, code: CodeBlock): self.code = BlockStmt(code) self.log = Logger("Algo") self.strict_typing = False self.callback_stop = lambda: () self.callback_input = None self.callback_print = None self.map = { DisplayStmt: self.exec_display, InputStmt: self.exec_input, AssignStmt: self.exec_assign, IfStmt: self.exec_if, ForStmt: self.exec_for, WhileStmt: self.exec_while, BreakStmt: self.exec_break, ContinueStmt: self.exec_continue, FuncStmt: self.exec_function, ReturnStmt: self.exec_return, CallStmt: self.exec_call, ElseStmt: self.exec_else, BaseStmt: lambda _: (), CommentStmt: lambda _: (), StopStmt: self.exec_stop, SleepStmt: self.exec_sleep }
def process(self, frame): time_elapsed = TimeElapsed() self.lastProcessedFrame = frame Logger.info('Processing frame - PCL: ' + frame.getFilePath()) cloud = pcl.load_XYZRGB(frame.getFilePath()) self.algorithmSegmentation.segment_image(cloud, True) time_elapsed.printTimeElapsed()
def main(args): loader = Loader(args.run_name) logger = Logger(args.run_name, create_if_exists=False) option = loader.load_option() layers = [int(l) for l in option["layers"].split(",")] params = loader.load_params() Model = get_model_cls_by_type(option["type"]) model = Model(layers, option["nc"], option["omega"]) model.update_net_params(params) if args.size == 0: orig_img_fn = loader.get_image_filename("original") img = Image.open(orig_img_fn) width = img.width height = img.height else: width = args.size height = args.size estimate_and_save_image(model, width, height, logger) if option["nc"] == 1: estimate_and_save_gradient(model, width, height, logger) estimate_and_save_laplacian(model, width, height, logger) if option["size"] != 0: # PIL resize as reference orig_pil_img = loader.load_pil_image("original") resized_pil = orig_pil_img.resize((width, height)) pil_output_name = "pil_{}x{}".format(width, height) logger.save_image(pil_output_name, resized_pil)
def send(self, data): # Importing it here to avoid error due to circular dependency from util.log import Logger '''Data can be dict, str, int, list''' Logger.info('Sending slack message to: ' + self.webhook_url) data = json.dumps({'text': data}) res = requests.post(self.webhook_url, data) return res
def __init__(self, expr: str): """Initializes the Parser instance. expr -- the expression to be parsed""" self.expression = expr self.tokens = [] self.index = 0 self.log = Logger("Parser")
def process(self, frame): time_elapsed = TimeElapsed() self.lastProcessedFrame = frame Logger.info('Processing frame - RGB: ' + frame.rgbFrame.getFilePath() + ', Depth: '+frame.depthFrame.getFilePath()) self.results = self.algorithmSegmentation.segment_image(self.get_image(frame.rgbFrame), self.get_image(frame.depthFrame)) Logger.info('Objects segmented: ' + str(self.algorithmSegmentation.get_num_objects())) time_elapsed.printTimeElapsed('Total segmentation - ')
def __init__(self): md5 = hashlib.md5() md5.update(os.urandom(24)) session['requestId'] = md5.hexdigest() Logger.log("请求 请求id:%s\n来源IP:%s\n请求方法:%s\n请求路径:%s\n请求参数:%s" % (session['requestId'], request.environ['REMOTE_ADDR'], request.environ['REQUEST_METHOD'], request.url, json.dumps(request.form))) Resource.__init__(self)
class PipelineLog: def __init__(self, logpath): self.logger = Logger(logpath) self.logpipeout = LogPipe.createAndStart('OUT', self.logger) self.logpipeerr = LogPipe.createAndStart('ERROR', self.logger) def close(self): self.logpipeout.close() self.logpipeerr.close() self.logger.close()
def drop(self): ''' drop table :return: ''' try: remove_dir(self.path) except Exception as e: Logger.error("drop table %s...failed. error %s", self.name, str(e)) raise e
def write_results(self): time_elapsed = TimeElapsed() img = self.write_objects() Logger.info('Saving result to ' + self.parameter.outputDir+self.lastProcessedFrame.rgbFrame.fileName) if not os.path.exists(self.parameter.outputDir): os.makedirs(self.parameter.outputDir) cv2.imwrite(self.parameter.outputDir+self.lastProcessedFrame.rgbFrame.fileName, img) time_elapsed.printTimeElapsed('Total writing file - ')
def util_load(self, file): file_name = os.path.basename(file) file_name = file_name.replace(".pyc", "_case").replace(".py", "_case") name_info = file_name.split("_", 1) self.case_id = name_info[0] self.case_name = name_info[1] self.logger = Logger(self.case_id) self.case_info = CaseInfo(self.case_id, self.case_name, "Functional testing") self.case_info.set_log_path(self.logger.log_file_rel_report)
def download_data_all(self): Logger.info('Download started') if self.expiry_strike_price_map is None: self.get_expiry_strike_price_map_for_all() downloaded_files = list() for expiry in self.expiry_strike_price_map.keys(): for strike_price in self.expiry_strike_price_map[expiry]: filepath = self.download_data_specific(expiry, strike_price) downloaded_files.append(filepath) Logger.info('Download finished') return downloaded_files
def desc(self): ''' descrite table from store :return: Table ''' try: with open(self.table_file) as ftable: table = Table().fromstr(ftable.read()) return table except Exception as e: Logger.info("describe table %s...failed. error: %s", self.name, str(e)) raise e
def drop(self): ''' drop table :return: ''' try: sql = "drop table if exists %s;" % self.name self.dbc.cursor().execute(sql) Logger.info("drop table %s...success", self.name) except Exception as e: Logger.error("drop table %s...failed. error %s", self.name, str(e)) raise e
def truncate(self): ''' truncate table :return: ''' try: with Lock(self.lock): remove_files(self.data_file) self._create_data_file() except Exception as e: Logger.error("truncate table %s...failed. error %s", self.name, str(e)) raise e
def __init__(self): md5 = hashlib.md5() md5.update(os.urandom(24)) session['requestId'] = md5.hexdigest() Logger.log( ">>>>>>>>>>>>>>>>>>>>>>> 请求 请求id:%s >>>>>>>>>>>>>>>>>>>>>>>\n%s|%s|%s|%s|%s" % (session['requestId'], time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), request.environ['REMOTE_ADDR'], request.environ['REQUEST_METHOD'], request.url, request.get_data())) Resource.__init__(self)
class Worker(object): def __init__(self, queue_name, route, persister, wnum, port, logfile=sys.stdout): self.queue_name = queue_name self.route = route self.persister = persister self.stop = False self.pid = os.getpid() self.worker_id = '-'.join(['worker', str(wnum), queue_name, route, str(self.pid)]) self.log = Logger(self.worker_id,logfile=logfile, loglevel=logging.DEBUG) self.log.info("starting") self.host = socket.gethostbyname(socket.gethostname()) self.port = port self.register() self.todo = None self.stop = False def register(self): self.persister.add_worker(self.worker_id, self.host, self.port) def unregister(self): self.persister.delete_worker(self.worker_id) def start_worker(self): while not self.stop: if self.todo: yield self.work(self.todo) self.todo = None def stop_worker(self): self.log.info("shutting down") self.unregister() self.stop = True def work(self, f): self.persister.set_working() ret = f() self.persister.unset_working() return ret
def __init__(self, queue_name, route, persister, wnum, port, logfile=sys.stdout): self.queue_name = queue_name self.route = route self.persister = persister self.stop = False self.pid = os.getpid() self.worker_id = '-'.join(['worker', str(wnum), queue_name, route, str(self.pid)]) self.log = Logger(self.worker_id,logfile=logfile, loglevel=logging.DEBUG) self.log.info("starting") self.host = socket.gethostbyname(socket.gethostname()) self.port = port self.register()
def __init__(self, queue_name, routing_keys=None, backend='mongodb', conn_url='localhost:27017', dbname='skunkqueue', logfile=sys.stdout, pidfile=None): """ routing_keys are a required parameter to specify an n-length list of routing keys, which will each be assigned to one worker """ self.stop = False self.queue_name = queue_name self.persister = get_backend(backend)(conn_url=conn_url, dbname=dbname) self.log = Logger('pool-'+queue_name, logfile=logfile) self.host = socket.gethostbyname(socket.gethostname()) self.workers = {} if pidfile: self.log.info("writing to pidfile %s" % pidfile) with open(pidfile) as f: f.write(str(self.pid)) f.close() # TODO this needs to be in shared memory wnums = {} self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.s.bind((self.host, 0)) self.s.listen(5) self.port = self.s.getsockname()[1] for key in routing_keys: if key not in wnums: wnums[key] = 0 wnums[key] += 1 worker = Worker(queue_name, key, self.persister, wnums[key], self.port, logfile) thread = KillableThread(target=worker.begin_execution) thread.start() self.workers[worker.worker_id] = (worker, thread)
class WorkerPool(object): def __init__(self, queue_name, routing_keys=None, backend='mongodb', conn_url='localhost:27017', dbname='skunkqueue', logfile=sys.stdout, pidfile=None): """ routing_keys are a required parameter to specify an n-length list of routing keys, which will each be assigned to one worker """ self.stop = False self.queue_name = queue_name self.persister = get_backend(backend)(conn_url=conn_url, dbname=dbname) self.log = Logger('pool-'+queue_name, logfile=logfile) self.host = socket.gethostbyname(socket.gethostname()) self.workers = {} if pidfile: self.log.info("writing to pidfile %s" % pidfile) with open(pidfile) as f: f.write(str(self.pid)) f.close() # TODO this needs to be in shared memory wnums = {} self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.s.bind((self.host, 0)) self.s.listen(5) self.port = self.s.getsockname()[1] for key in routing_keys: if key not in wnums: wnums[key] = 0 wnums[key] += 1 worker = Worker(queue_name, key, self.persister, wnums[key], self.port, logfile) thread = KillableThread(target=worker.begin_execution) thread.start() self.workers[worker.worker_id] = (worker, thread) def __enter__(self, *args, **kwargs): self.log.info("starting") def gentle(signum, frame): self.log.info("Received gentle shutdown signal %d" % signum) self.shutdown() sys.exit(0) def rough(signum, frame): self.log.warn("Received non-gentle kill signal %d" % signum) self.die() sys.exit(0) signal.signal(signal.SIGINT, rough ) signal.signal(signal.SIGHUP, gentle) signal.signal(signal.SIGTERM, gentle) signal.signal(signal.SIGALRM, gentle) signal.signal(signal.SIGQUIT, gentle) return self def __exit__(self, *args, **kwargs): self.shutdown() def shutdown(self): for _id in self.workers.keys(): worker = self.workers[_id] worker[0].stop_worker() del self.workers[_id] self.s.close() self.stop = True def die(self): for _id in self.workers.keys(): worker = self.workers[_id] try: worker[1].raise_exc(TriggeredInterrupt) self.log.warn("raised an exception in %s" % str(_id)) del self.workers[_id] except ValueError: # it was dead already self.log.debug("ignored killing thread %s" % str(_id)) continue self.s.close() self.stop = True def run_cmd(self, terminate=None, kill=None, suicide=None): # requests a termination # terminate looks like: <worker_id> if terminate: self.workers[terminate][0].stop_worker() del self.workers[terminate] # performs a hard kill # kill looks like: <worker_id> if kill: try: self.workers[kill][1].raise_exc(TriggeredInterrupt) del self.workers[kill] except ValueError: # it was already dead self.log.warn("tried to kill a thread when it was dead already") # shuts down the entire pool # suicide looks like: 1 if suicide == 1: self.stop = True def listen(self): while not self.stop: conn, addr = self.s.accept() datalen = conn.recv(2) data = "" while len(data) < int(datalen): data += conn.recv(1024) data = json.loads(data) self.run_cmd(**data) # no need to run the pool if it has no workers if len(self.workers) == 0: self.shutdown()
class Worker(object): def __init__(self, queue_name, route, persister, wnum, port, logfile=sys.stdout): self.queue_name = queue_name self.route = route self.persister = persister self.stop = False self.pid = os.getpid() self.worker_id = '-'.join(['worker', str(wnum), queue_name, route, str(self.pid)]) self.log = Logger(self.worker_id,logfile=logfile, loglevel=logging.DEBUG) self.log.info("starting") self.host = socket.gethostbyname(socket.gethostname()) self.port = port self.register() def begin_execution(self, *args): while not self.stop: try: job = self.persister.get_job_from_queue(self.queue_name, self.worker_id, self.route) if job: self.do_job(job) sleep(0.1) except TriggeredInterrupt: # we were cut off by an interrupt trigger self.log.warn("received interrupt request; stopping current job") self.log.warn("no result will be committed and this job will not be restarted") self.stop_worker() def register(self): self.persister.add_worker(self.worker_id, self.host, self.port) def unregister(self): self.persister.delete_worker(self.worker_id) def do_job(self, job): # depickle body = pickle.loads(job['body']) directory = body['dir'] # FIXME a horrible hack where we add ourselves to the pythonpath sys.path.append(directory) mod = __import__(body['mod']) self.log.debug("successfully imported module "+str(mod)) if job['fn_type'] == 'method': parent = dill.loads(body['parent']) fn = getattr(parent, body['fn']) else: fn = getattr(mod, body['fn']) args = body['args'] kwargs = body['kwargs'] # call it self.persister.set_working(self.worker_id) try: ret = fn(*args, **kwargs) self.persister.save_result(job['job_id'], ret, 'complete') self.log.info(ret) except Exception as e: self.persister.save_result(job['job_id'], None, 'error') self.log.error(str(e)) exc_t, exc_v, exc_tr = sys.exc_info() self.log.error(str( '\n'.join(traceback.format_exception(exc_t, exc_v, exc_tr)))) self.log.debug("python path is %s" % str(sys.path)) finally: a = sys.path.pop() self.persister.unset_working(self.worker_id) def stop_worker(self): self.log.info("shutting down") self.unregister() self.stop = True
def __init__(self, queue_name, routing_keys=None, backend='mongodb', conn_url='localhost:27017', dbname='fwbots', logfile=sys.stdout, pidfile=None): """ routing_keys are a required parameter to specify an n-length list of routing keys, which will each be assigned to one worker FWBOTS: we are using routing_keys as the account names to load """ self.stop = False self.name = queue_name self.log = Logger('pool-'+queue_name, logfile=logfile) self.persister = get_backend(backend)(conn_url,dbname) self.port = -1 # too lazy to actually remove this self.workers = {} self.manual = {} self.auto = {} self.insta = {} if pidfile: self.log.info("writing to pidfile %s" % pidfile) with open(pidfile) as f: f.write(str(self.pid)) f.close() # TODO this needs to be in shared memory wnums = {} self.ircbot = IrcListener(self,"#fwbots",self.name,"irc.freenode.net") for key in routing_keys: errnum = 0 try: acc = get_twitter_account(key) if acc.ty == 'auto': self.log.info('found auto for %s' % key) self.auto[acc.name] = acc # if there are multiple manual accs defined, pick only the last one elif acc.ty == 'manual': self.log.info('found manual for %s' % key) self.manual[acc.name] = acc except NameError: errnum += 1 try: acc = get_instagram_account(key) self.insta[acc.name] = acc except NameError: errnum += 1 if errnum > 1: self.log.warn("Could not find any account called %s"%key) if key not in wnums: wnums[key] = 0 wnums[key] += 1 worker = Worker(queue_name, key, self.persister, wnums[key], self.port, logfile) thread = KillableThread(target=worker.start_worker) thread.start() self.workers[worker.worker_id] = (worker, thread) # lastly, register yourself self.persister.add_pool(self)
def __init__(self, logpath): self.logger = Logger(logpath) self.logpipeout = LogPipe.createAndStart('OUT', self.logger) self.logpipeerr = LogPipe.createAndStart('ERROR', self.logger)
class WorkerPool(object): def __init__(self, queue_name, routing_keys=None, backend='mongodb', conn_url='localhost:27017', dbname='fwbots', logfile=sys.stdout, pidfile=None): """ routing_keys are a required parameter to specify an n-length list of routing keys, which will each be assigned to one worker FWBOTS: we are using routing_keys as the account names to load """ self.stop = False self.name = queue_name self.log = Logger('pool-'+queue_name, logfile=logfile) self.persister = get_backend(backend)(conn_url,dbname) self.port = -1 # too lazy to actually remove this self.workers = {} self.manual = {} self.auto = {} self.insta = {} if pidfile: self.log.info("writing to pidfile %s" % pidfile) with open(pidfile) as f: f.write(str(self.pid)) f.close() # TODO this needs to be in shared memory wnums = {} self.ircbot = IrcListener(self,"#fwbots",self.name,"irc.freenode.net") for key in routing_keys: errnum = 0 try: acc = get_twitter_account(key) if acc.ty == 'auto': self.log.info('found auto for %s' % key) self.auto[acc.name] = acc # if there are multiple manual accs defined, pick only the last one elif acc.ty == 'manual': self.log.info('found manual for %s' % key) self.manual[acc.name] = acc except NameError: errnum += 1 try: acc = get_instagram_account(key) self.insta[acc.name] = acc except NameError: errnum += 1 if errnum > 1: self.log.warn("Could not find any account called %s"%key) if key not in wnums: wnums[key] = 0 wnums[key] += 1 worker = Worker(queue_name, key, self.persister, wnums[key], self.port, logfile) thread = KillableThread(target=worker.start_worker) thread.start() self.workers[worker.worker_id] = (worker, thread) # lastly, register yourself self.persister.add_pool(self) def __enter__(self, *args, **kwargs): self.log.info("starting") def gentle(signum, frame): self.log.info("Received gentle shutdown signal %d" % signum) self.shutdown() sys.exit(0) def rough(signum, frame): self.log.warn("Received non-gentle kill signal %d" % signum) self.die() sys.exit(0) signal.signal(signal.SIGINT, rough ) signal.signal(signal.SIGHUP, gentle) signal.signal(signal.SIGTERM, gentle) signal.signal(signal.SIGALRM, gentle) signal.signal(signal.SIGQUIT, gentle) return self def __exit__(self, *args, **kwargs): self.shutdown() def shutdown(self): for _id in self.workers.keys(): worker = self.workers[_id] worker[0].stop_worker() del self.workers[_id] self.stop = True self.persister.delete_pool(self.name) def die(self): for _id in self.workers.keys(): worker = self.workers[_id] try: worker[1].raise_exc(TriggeredInterrupt) self.log.warn("raised an exception in %s" % str(_id)) del self.workers[_id] except ValueError: # it was dead already self.log.debug("ignored killing thread %s" % str(_id)) continue self.stop = True def work(self, f): w = self.persister.get_avail_workers()[0] w.todo = f def run_cmd(self, terminate=None, kill=None, suicide=None, tweet=None,rtp=None): # requests a termination # terminate looks like: <worker_id> if terminate: self.workers[terminate][0].stop_worker() del self.workers[terminate] # performs a hard kill # kill looks like: <worker_id> if kill: try: self.workers[kill][1].raise_exc(TriggeredInterrupt) del self.workers[kill] except ValueError: # it was already dead self.log.warn("tried to kill a thread when it was dead already") # shuts down the entire pool # suicide looks like: 1 if suicide == 1: self.stop = True # sends a tweet from self.manual # tweet looks like: <msg> if tweet: self.log.info("tweeting with manual account %s: '%s'"%(self.manual.name,tweet)) self.work(lambda: self.manual.make_tweet(tweet)) # if rtp is set, have dummies retweet it with probability rtp if rtp: def f(): for a in self.auto: if rtp < random.random(): a.retweet_last(self.manual.name) self.work(f) def listen(self): self.ircbot.start()