def __init__(self, argv=None): if argv is None: argv = sys.argv[1:] cur_path = os.path.abspath(os.path.curdir) self.product_path = cur_path options = self.parseArgs(argv) self.options = options if options.debug: level = logging.DEBUG else: level = logging.INFO initLogger(options.log_file, True, level) logger.debug('### RUN ProductMan from %s args: %s ' % (cur_path, ' '.join(argv)))
def main(): args, parser = args_parse() initLogger(logPath) start_time = getTimeInSec() myPrint(conf.minio_endpoint) try: prev_time = start_time if args.action == 'check_conn' or args.action == 'all': myPrint("Action: check minio connection", 1) m = MinioWrapper() myPrint(m.bucketExists("my-test-bucket")) prev_time, prstr = timeDiff(prev_time) myPrint("Time taken by Action check_conn: " + prstr, 11) if args.action == 'remove_bucket' or args.action == 'all': myPrint("Action: remove_bucket test", 1) m = MinioWrapper() myPrint(m.deleteBucket()) prev_time, prstr = timeDiff(prev_time) myPrint("Time taken by Action remove_bucket: " + prstr, 11) if args.action == 'check_hash' or args.action == 'all': myPrint("Action: check_hash test", 1) packet_names = getJsonFile(hashCheckPacketsPacket) for packet in packet_names: myPrint("Packet name: " + packet, 2) Migration().checkHash(packet) prev_time, prstr = timeDiff(prev_time) myPrint("Time taken by Action check_hash: " + prstr, 11) if args.action == 'check_records' or args.action == 'all': myPrint("Action: check_records", 1) total_buckets = getJsonFile(bucketListPath) total_packets = getJsonFile(packetListPath) total_ignored = getJsonFile(ignoredBucketListPath) total_migrated = getJsonFile(migratedPackets) myPrint("total_buckets: " + str(len(total_buckets))) myPrint("total_packets: " + str(len(total_packets))) myPrint("total_ignored: " + str(len(total_ignored))) myPrint("total_migrated: " + str(len(total_migrated))) myPrint(list(set(total_packets) - set(total_migrated))) prev_time, prstr = timeDiff(prev_time) myPrint("Time taken by Action check_records: " + prstr, 11) except: prev_time, prstr = timeDiff(start_time) myPrint("Total time taken by the script: " + prstr, 11) formatted_lines = traceback.format_exc() myPrint(formatted_lines, 13) sys.exit(1) prev_time, prstr = timeDiff(start_time) myPrint("Total time taken by the script: " + prstr, 11) return sys.exit(0)
def main(ctx, **kwargs): import pdb pdb.set_trace() # pdb.set_trace() site_name = kwargs['site_name'] task_type = kwargs['task_type'] thread_number = kwargs['thread_number'] server_ip = kwargs['server_ip'] server_port = kwargs['server_port'] proxy_type = kwargs['proxy_type'] log_level = log_dict.get(kwargs['log_level'], logging.INFO) register = kwargs['register'] login = kwargs['login'] import pdb pdb.set_trace() if site_name is None or task_type is None: click.echo(ctx.get_help()) sys.exit(-1) # 初始化logger log_dir = '%s/%s_%s.log' % (settings.logs_dir_home, site_name, task_type) logger = utils.initLogger(log_dir, log_level) # 开始执行爬虫程序 mainCrawler = MainCrawler(logger, site_name, task_type, thread_number, server_ip, server_port, proxy_type, register, login) import pdb pdb.set_trace() try: mainCrawler.run() except: traceback.print_exc() utils.send_email(traceback.format_exc())
def runWriter(config_file_location): with open(config_file_location, 'r') as f: config = yaml.load(f, Loader=yaml.SafeLoader) logger = initLogger('monitor', level=config['log_level']) # init database db = getPostgresDBCursor(config['target']['postgres_uri']) createDatabaseObjectsSafe(db, logger) websites_ids = initDictionaries(db, config['websites'], logger) threads = [] thread_id = 0 for w in config['websites']: x = threading.Thread( name='thread {}'.format(w['url']), target=processItem, args=(thread_id, Writer(kafka_consumer=getKafkaConsumer( kafka_connect=config['kafka_connect']), db_cursor=getPostgresDBCursor( config['target']['postgres_uri']), websites_ids=websites_ids.copy()), logger)) threads.append(x) x.start() thread_id += 1
def __init__(self, settings_path=None): """ settings -- full path to settings module """ if settings_path is not None: Settings(file_path=settings_path) elif os.environ.get("BIONIC_SETTINGS_FILE") is None: module_path = os.path.dirname(__file__) settings_path = os.path.join(module_path, "settings.yaml") os.environ["BIONIC_SETTINGS_FILE"] = settings_path Settings() else: Settings() initLogger() global logger logger = logging.getLogger("bionic")
def main(): args, parser = args_parse() initLogger(logPath) start_time = getTimeInSec() myPrint(conf.minio_endpoint) try: prev_time = start_time if args.action == 'get_buckets' or args.action == 'all': myPrint("Action: get_buckets", 1) GetBuckets().run() prev_time, prstr = timeDiff(prev_time) myPrint("Time taken by Action get_buckets: " + prstr, 11) if args.action == 'find_packets' or args.action == 'all': myPrint("Action: find_packets", 1) FindPackets().run() prev_time, prstr = timeDiff(prev_time) myPrint("Time taken by Action find_packets: " + prstr, 11) if args.action == 'migrate' or args.action == 'all': myPrint("Action: migrate", 1) Migration().run() prev_time, prstr = timeDiff(prev_time) myPrint("Time taken by Action migrate: " + prstr, 11) if args.action == 'get_records' or args.action == 'all': myPrint("Action: get_records", 1) m = MinioWrapper() objs = m.listObjects(conf.new_bucket_name, False) new_objs = [] for ob in objs: new_objs.append(ob.replace("/", "")) writeJsonFile(migratedPackets, new_objs) myPrint("Total objects level 1 " + str(len(new_objs))) prev_time, prstr = timeDiff(prev_time) myPrint("Time taken by Action get_records: " + prstr, 11) except: prev_time, prstr = timeDiff(start_time) myPrint("Total time taken by the script: " + prstr, 11) formatted_lines = traceback.format_exc() myPrint(formatted_lines, 13) sys.exit(1) prev_time, prstr = timeDiff(start_time) myPrint("Total time taken by the script: " + prstr, 11) return sys.exit(0)
def runMonitoring(config_file_location): with open(config_file_location, 'r') as f: config = yaml.load(f, Loader=yaml.SafeLoader) logger = initLogger('monitor', level=config['log_level']) threads = [] for w in config['websites']: x = threading.Thread( name='thread {}'.format(w['url']), target=processItem, args=(Checker( website=w, kafka_producer=getKafkaProducer( kafka_connect=config['kafka_connect']), kafka_topic=config['kafka_connect']['topic'], default_interval=config['check_every_seconds_default']), logger)) threads.append(x) x.start()
def __init__(self): self.users = {} write_queues = {} task_queues = {} count_queue = Queue(maxsize=settings.count_queue_size) mc = settings.getMCInstance(isTxMongo=False) self.logger = utils.initLogger('%s/run_data_server_main.log' % settings.logs_dir_home) self.db = DbManager.DbManager(self.logger, mc, write_queues, task_queues, count_queue) # 开一个线程定时清理 ts = [] t1 = threading.Thread(target=self.sched_cleanup, args=()) ts.append(t1) for t in ts: t.setDaemon(True) t.start() self.logger.info("__init__ finish")
from kafka import KafkaProducer from json import dumps from utils import initLogger, getConfigPath, readConfigFile from apiService import APIService import time LOGGER = initLogger("KAFKA_PRODUCER_LOG") CONFIG_FILE_PATH = getConfigPath() KAFKA_CONFIG_SETTINGS = readConfigFile(CONFIG_FILE_PATH, "kafka") PRODUCER_SETTINGS = readConfigFile(CONFIG_FILE_PATH, "producer.settings") KAFKA_TOPIC = PRODUCER_SETTINGS["topic"] URL = PRODUCER_SETTINGS["url"] SLEEP_DURATION = int(PRODUCER_SETTINGS["timeout.seconds"]) producer = KafkaProducer( value_serializer=lambda v: dumps(v).encode('utf-8'), bootstrap_servers=KAFKA_CONFIG_SETTINGS["bootstrap_servers"]) def produceMessage(filteredJson): for coinData in filteredJson: LOGGER.info(f"Message is : {coinData}") nameCoin = coinData["name_coin"] try: producer.send(topic=KAFKA_TOPIC, value=coinData) except Exception as e: LOGGER.error(f"Coin : {nameCoin} has exception : {e}") if __name__ == "__main__":
for sql in expireSql: while True: utils.logInfo("execute sql=%s" % sql) if logdb.update(sql) == 0: break utils.logInfo("all done!") # 关闭连接 logdb.disconnect() gmsdb.disconnect() if __name__ == "__main__": props = Properties() props.load(open('conf/conf.properties')) utils.initLogger(props["log.file.path"]) parser = OptionParser() parser.add_option("-b", "--btime", action="store", type="string", dest="baseTime", default=str(datetime.date.today()), help="基准时间") parser.add_option("-g", "--srvs", action="store", type="string", dest="srvs", default=props["srvs"],
def setup_module(module): logger = initLogger('monitor', level='DEBUG')
def main(argv): utils.initLogger(values.LOG_FILE, argv) utils.registerInterrupt(cleanup) #Initialize (values.PIN is board.PIN, gpiozero needs pin number, so call board.PIN.id) global rxThread, statusThread status_q = queue.Queue() statusThread = status.StatusThread(status_q, values.STATUS_PIN1.id, values.STATUS_PIN2.id, values.STATUS_PIN3.id) statusThread.start() #Initialize Navigation logger.info("Initializing Navigation") nav = autopilot.Navigation(status_q, values.TURN_DEBOUNCE_SECS) #Initialize Relays logger.info("Initializing Controls") controls = ctl.Control(status_q, values.MAX_TURN_TIME_SECS) controls.direction = ctl.Direction( values.TURN_MASTER_RELAY_PIN.id, values.TURN_POWER_RELAY_PIN.id, values.TURN_GROUND_RELAY_PIN.id) controls.speed = ctl.Speed(status_q, values.SPEED_MASTER_RELAY_PIN.id, values.SPEED_R1_RELAY_PIN.id, values.SPEED_R2_RELAY_PIN.id, values.SPEED_R3_RELAY_PIN.id, values.SPEED_R4_RELAY_PIN.id) controls.stop() #Initialize RF Remote logger.info("Initializing RF Receiver") q = queue.Queue() rxThread = rfcontrols.RxThread(status_q, values.RX_PIN.id, rfcontrols.Yosoo4ButtonRemote(), q) #Pause for hardware to initialize time.sleep(2) #Start Threads logger.info("Starting Remote Thread") rxThread.start() #Begin Keep Main Thread open for interrupt signal stopDownAt = 0 goDownAt = 0 while True: #Read Navigation Data nav.read() #Check for remote control button events try: m = q.get_nowait() logger.debug(f"Button Event: {m}") button = m["button"] if m["position"] == rfcontrols.BUTTON_DOWN: if button.id == rfcontrols.GO_BTN: goDownAt = time.time() stopDownAt = 0 controls.speed.bump() elif button.id == rfcontrols.STOP_BTN: goDownAt = 0 stopDownAt = time.time() controls.speed.bump(-1) elif button.id == rfcontrols.LEFT_BTN: controls.turnLeft() elif button.id == rfcontrols.RIGHT_BTN: controls.turnRight() else: logger.warning(f"Unsupported button {button.id}") else: if button.id == rfcontrols.GO_BTN: goDownAt = 0 elif button.id == rfcontrols.STOP_BTN: stopDownAt = 0 elif button.id == rfcontrols.LEFT_BTN: controls.stopTurn() elif button.id == rfcontrols.RIGHT_BTN: controls.stopTurn() else: logger.warning(f"Unsupported button {button.id}") except queue.Empty: pass #Check for long presses on remote control button events if goDownAt > 0: dur = time.time() - goDownAt if dur > 2: logger.info("Go Long Press") nav.setHeadingLock() if dur > 5: logger.info("Go Extended Long Press") controls.stop() nav.setAnchorLock() elif stopDownAt > 0: dur = time.time() - stopDownAt if dur > 2: logger.info("Stop Long Press") nav.setHeadingLock(False) nav.setAnchorLock(False) controls.stop() controls.speed.set(ctl.Speed.MIN) if dur > 10: logger.info("Stop Extended Long Press") controls.resetTurnHeading() #Check autopilot needs nav.applyCoarseCorrection(controls) #Check that turning is not stuck controls.checkTurn() time.sleep(0.25)
import requests from requests.exceptions import ConnectionError, Timeout, TooManyRedirects from utils import currentUnixTime, initLogger LOGGER = initLogger("API_SERVICE_LOG") class APIService: def getJson(self, url): try: response = requests.get(url) statusCode = response.status_code assert statusCode == 200, f"Response status code is : {statusCode}" LOGGER.info("Received JSON response") return response.json() except (AssertionError, ConnectionError, Timeout, TooManyRedirects) as e: LOGGER.error(f"Exception occurred : {e}") return {} def parseCoinData(self, coinData): try: return { "name_coin": coinData["name"], "symbol_coin": coinData["symbol"], "id": coinData["id"], "uuid": coinData["uuid"], "number_of_markets": coinData["numberOfMarkets"], "volume": coinData["volume"], "market_cap": coinData["marketCap"], "total_supply": coinData["totalSupply"],
import unittest from paths import envPath, logPath from dotenv import load_dotenv load_dotenv() # OR, the same with increased verbosity load_dotenv(verbose=True) load_dotenv(dotenv_path=envPath) from minioWrapper import MinioWrapper from utils import ridToCenterTimestamp, initLogger, myPrint initLogger(logPath) class MyTestCase(unittest.TestCase): def test_listBuckets(self): print("OKay") m = MinioWrapper() myPrint(m.listBuckets()) def test_bucketExists(self): m = MinioWrapper() myPrint(m.bucketExists("10001100010002420210223073024")) def test_getObject(self): m = MinioWrapper() objects = [ 'RESIDENT/RES_UPDATE/10001100010002420210223073024_evidence', 'RESIDENT/RES_UPDATE/10001100010002420210223073024_id', 'RESIDENT/RES_UPDATE/10001100010002420210223073024_optional'
# [(name, list(group)) for name, group in itertools.groupby(qs, lambda p:p['date'])] class test1: t1 = 2 t2 = "a" def aa(self): def f1(): print self.t1 f1() if __name__ == "__main__": utils.initLogger(None) parser = OptionParser() parser.add_option("-o", "--opt", action="store", type="string", dest="opt", default="hd", help="渠道标示") parser.add_option("-b", "--btime", action="store", type="string", dest="baseTime", default=str(datetime.date.today()), help="基准时间")
for k, v in managerEmails.items(): emailMatch[k] = {} emailMatch[k]['managerEmails'] = [] emailMatch[k]['agents'] = [] for obj in objects: if getattr(obj, 'managerNotify', None) is not None: if obj.managerNotify in emailMatch.keys(): emailMatch[obj.managerNotify]['managerEmails'].append([a for a in obj.managerEmails \ if a not in emailMatch[obj.managerNotify]['managerEmails']]) emailMatch[obj.managerNotify]['agents'].append(obj.name) continue return emailMatch logging = initLogger() def main(): # At several points we are going to turn these vars to false if there is an error or # unexpected results. Doing so will prevent any emails from going out. proceedToEmailMangers, proceedToEmailAgents = True, True proceedManagerReasons, proceedAgentReasons = [], [] # These events have already results in a warning email to the agent. The # event IDs were stored in a txt file from prior runs of the script. calendarEventsAlreadyNotified, retrieveCalFuncStatus = retrieveCalendarEventsAlreadyNotified( ) logging.info('Retrieved {} events from list of previously reminded PTO events.'.format(\ len(calendarEventsAlreadyNotified)))
def main(): args, parser = args_parse() initLogger(logPath) start_time = getTimeInSec() db = DatabaseSession(conf.db_host, conf.db_port, conf.db_user, conf.db_pass) try: prev_time = start_time if args.action == 'get_vids' or args.action == 'all': myPrint("Action: get_vids", 1) vids = [] vid_dicts = db.getVids() for vid_dict in vid_dicts: vids.append(vid_dict['vid']) writeJsonFile(vidListPath, vids) prev_time, prstr = timeDiff(prev_time) myPrint("Time taken by Action get_vids: " + prstr, 11) if args.action == 'fetch_info' or args.action == 'all': output = [] myPrint("Action: fetch_info", 1) ms = MosipSession(conf.server, conf.regproc_client_id, conf.regproc_secret_key, conf.regproc_app_id) vids = getJsonFile(vidListPath) for vid in vids: myPrint("Operating on VID " + vid, 3) res = ms.getUin(vid) uin = res['identity']['UIN'] if conf.debug: myPrint("UIN: " + uin) modulo = int(uin) % conf.idrepo_modulo myPrint("Modulo: " + str(modulo)) salt_row = db.getHash(modulo) if salt_row is not None: salt = salt_row['salt'] if conf.debug: myPrint("Salt: " + salt) uin_hash = hashlib.sha256(bytes(uin + salt, 'utf-8')).hexdigest() mod_uin_hash = str(modulo) + "_" + uin_hash.upper() rid_row = db.getRid(mod_uin_hash) if rid_row is not None: rid = rid_row['rid'] myPrint("RID found") if conf.debug: myPrint("RID: " + rid) center_id, timestamp = ridToCenterTimestamp(rid) output.append({ 'vid': vid, 'uin': uin, 'mod_uin_hash': mod_uin_hash, 'salt': salt, 'rid': rid, 'center_id': center_id, 'timestamp': timestamp }) else: raise RuntimeError( "RID not found for for mod_uin_hash: " + mod_uin_hash) else: raise RuntimeError("salt not found for for modulo: " + str(modulo)) writeJsonFile(credentialPreparedDataPath, output) prev_time, prstr = timeDiff(prev_time) myPrint("Time taken by Action fetch_info: " + prstr, 11) if args.action == 'reprint' or args.action == 'all': myPrint("Action: reprint", 1) output = [] vids = getJsonFile(credentialPreparedDataPath) ms = MosipSession(conf.server, conf.ida_client_id, conf.ida_secret_key, conf.ida_app_id) for vidInfo in vids: myPrint("VID: " + vidInfo['vid'], 3) data = { "id": vidInfo['vid'], "credentialType": conf.credential_type, "issuer": conf.partner_id, "recepiant": "", "user": "******", "encrypt": False, "encryptionKey": "", "sharableAttributes": [], "additionalData": { 'centerId': vidInfo['center_id'], 'creationDate': vidInfo['timestamp'], 'registrationId': vidInfo['rid'] } } json_data = json.dumps(data, separators=(',', ':')) myPrint(json_data) if db.checkRequestInCredentialTransaction(json_data) is None: resp = ms.credentialRequest(data) output.append(resp) else: myPrint("Skipping credential request", 11) writeJsonFile(vidRequestId, output) myPrint('Input VIDs: ' + str(len(vids)), 12) myPrint('Output RequestIds: ' + str(len(output)), 12) prev_time, prstr = timeDiff(prev_time) myPrint("Time taken by Action reprint: " + prstr, 11) db.closeAll() except: db.closeAll() prev_time, prstr = timeDiff(start_time) myPrint("Total time taken by the script: " + prstr, 11) formatted_lines = traceback.format_exc() myPrint(formatted_lines, 13) sys.exit(1) prev_time, prstr = timeDiff(start_time) myPrint("Total time taken by the script: " + prstr, 11) return sys.exit(0)