def run(self): logger.config_logging( logger_name="worker", file_name=("didgen_concurrent_write_test_%s" % self.process_id + '.log'), log_level="DEBUG", dir_name="logs", day_rotate=False, when="D", interval=1, max_size=20, backup_count=5, console=True) LLOG = logging.getLogger("worker") LLOG.propagate = False success_processed = 0 total_processed = 0 start_id = 10000000000000000000L end_id = 0 r = redis.StrictRedis(host='localhost', port=6389, db=0) n = 0 while True: if self.tasks.empty() == False: task = self.tasks.get() if task != "mission complete": total_processed += 1 try: task_id = task[0] gid = r.get("id_test") if gid.isdigit() and int(gid) == 0: LLOG.error("Invalid Id: %s", gid) elif not gid.isdigit(): LLOG.error("Invalid Id: %s", gid) if int(gid) < start_id: start_id = int(gid) if int(gid) > end_id: end_id = int(gid) n += 1 if n % 5000 == 0: LLOG.debug("Worker %04d: %020d", self.process_id, int(gid)) # LLOG.debug("Worker %04d: %020d", self.process_id, int(gid)) success_processed += 1 except Exception, e: LLOG.exception(e) else: break else: time.sleep(0.0001)
from __future__ import absolute_import import logging import constants import settings from logger import config_logging from vendored.lirc import Lirc import pubnub from pubnub.callbacks import SubscribeCallback from pubnub.enums import PNStatusCategory from pubnub.pnconfiguration import PNConfiguration from pubnub.pubnub import PubNub logger = config_logging(settings.LOG_LEVEL, settings.LOG_FILE) pubnub.set_stream_logger('pubnub', logging.WARNING) pnconfig = PNConfiguration() pnconfig.subscribe_key = settings.SUBSCRIBE_KEY pnconfig.publish_key = settings.PUBLISH_KEY pnconfig.ssl = True pubnub_client = PubNub(pnconfig) lirc = Lirc() class KeyPressSubscribeCallback(SubscribeCallback): def presence(self, pubnub, presence): pass # handle incoming presence data
def run(self): date_time = strftime("%Y%m%d_%H%M%S", localtime()) logger.config_logging(file_name = ("project_import_%s_" % self.process_id + '.log'), log_level = CONFIG['log_level'], dir_name = "logs", day_rotate = False, when = "D", interval = 1, max_size = 20, backup_count = 5, console = True) LOG.info("Start ProjectImportProcess(%s)", self.process_id) try: while True: try: command, project = self.pipe_client.recv() if command == "IMPORT": LOG.debug("ProjectImportProcess import %s[%s] to Process(%s)", project.sha1, project.project_name, self.process_id) # os.environ["GOPATH"] = "%s%s" % (os.environ["GOPATH"] + ":" if os.environ.has_key("GOPATH") else "", CONFIG["go_path"]) os.environ["GOPATH"] = project.go_path LOG.info("GOPATH: %s", os.environ["GOPATH"]) data_path = os.path.join(CONFIG["data_path"], "projects", project.project_name) flag = common_utils.make_callgraph_data(project.main_path, os.path.join(data_path, "data.callgraph")) if flag is True: LOG.debug("generate data.callgraph success") db_path = os.path.join(data_path, "table_calling.db") if os.path.exists(db_path) and os.path.isdir(db_path): shutil.rmtree(db_path) LOG.info("delete: %s", db_path) elif os.path.exists(db_path) and os.path.isfile(db_path): os.remove(db_path) LOG.info("delete: %s", db_path) db_path = os.path.join(data_path, "table_called.db") if os.path.exists(db_path) and os.path.isdir(db_path): shutil.rmtree(db_path) LOG.info("delete: %s", db_path) elif os.path.exists(db_path) and os.path.isfile(db_path): os.remove(db_path) LOG.info("delete: %s", db_path) finder = Finder(data_path, called = True) finder.build_finder() projects = Projects() ix = IX(projects = [v for v in projects.all().itervalues()]) LOG.debug("IX: %s", IX.IX_INDEXS) flag = index_all_func(db = finder.db, ix = ix.get(project.project_name)) if flag is True: finder = Finder(data_path, called = False) finder.build_finder() flag = index_all_func(db = finder.db, ix = ix.get(project.project_name)) else: LOG.error("Create data.callgraph failed!") if flag is True: self.pipe_client.send((command, True)) else: self.pipe_client.send((command, False)) elif command == "EXIT": LOG.info("ProjectImportProcess(%s) exit by EXIT command!", self.process_id) return except EOFError: LOG.error("EOFError ProjectImportProcess(%s) Write Thread exit!", self.process_id) return except Exception, e: LOG.exception(e) LOG.info("Leveldb Process(%s) exit!", self.process_id)
import logger LOG = logging.getLogger(__name__) home = str(Path.home()) if __name__ == "__main__": workspace, input_data = Action.get_input() logs_directory = os.path.join(workspace, "logs") logger.config_logging(file_name = "first.log", log_level = "DEBUG", dir_name = logs_directory, day_rotate = False, when = "D", interval = 1, max_size = 20, backup_count = 5, console = False) LOG.debug("test start") LOG.debug("input_data: %s", input_data) data = {"messages": []} i = 0 while True: now = datetime.datetime.now() message = "%s: hello world, tornado(%03d): %s" % (now, i, tornado.version) data["messages"].append(message) LOG.debug(message) time.sleep(1)
cwd = os.path.split(os.path.realpath(__file__))[0] sys.path.insert(0, os.path.split(cwd)[0]) from utils import common_utils from config import CONFIG import logger LOG = logging.getLogger(__name__) if __name__ == "__main__": os.environ["GOPATH"] = "%s" % "/home/breeze/Develop/IDGO" logger.config_logging(file_name = "test.log", log_level = CONFIG["log_level"], dir_name = "logs", day_rotate = False, when = "D", interval = 1, max_size = 20, backup_count = 5, console = True) LOG.info("Test Start") t = time.time() # common_utils.make_callgraph_data() r = common_utils.get_definition_from_guru("/home/breeze/Develop/IDGO/src/github.com/flike/idgo/server/command.go", 19, 15) LOG.debug("get_definition_from_guru: %s", r) tt = time.time() LOG.info("Use Time: %ss", tt - t) LOG.info("Test Exit!")
from config import CONFIG from utils.finder import Finder from utils import common_utils from utils.index_whoosh import IX, index_all_func import logger LOG = logging.getLogger(__name__) INF = logging.getLogger("info") INF.propagate = False if __name__ == "__main__": logger.config_logging(file_name="Build.log", log_level=CONFIG["log_level"], dir_name="logs", day_rotate=False, when="D", interval=1, max_size=50, backup_count=5, console=True) logger.config_logging(logger_name="info", file_name="Info.log", log_level="NOSET", dir_name="logs", day_rotate=False, when="D", interval=1, max_size=20, backup_count=10, console=True) LOG.info("Start Build")
break else: time.sleep(0.0001) LLOG.info("total processed: %s, total successed: %s", total_processed, success_processed) self.tasks.put("mission complete") self.results.put( (total_processed, success_processed, start_id, end_id)) if __name__ == "__main__": logger.config_logging(file_name="didgen_concurrent_test.log", log_level="DEBUG", dir_name="logs", day_rotate=False, when="D", interval=1, max_size=20, backup_count=5, console=True) LOG.info("Start didgen_concurrent_test Script") test_num = 1000000 + 2000 process_num = 4 mission_queue = Queue(10000) result_queue = Queue(process_num) r = redis.StrictRedis(host='localhost', port=6389, db=0) if r.exists("id_test"): r.delete("id_test") r.set("id_test", 0)